From a6f98eadb0dc2993a00e3110ab352e1f05f5121d Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 3 Sep 2026 23:30:39 +0500 Subject: [PATCH 01/26] IGNITE-29032 Improvement of the pageEviction mechanism for in-memory --- .../DataRegionConfiguration.java | 13 +- .../processors/cache/GridCacheEntryEx.java | 18 ++ .../processors/cache/GridCacheMapEntry.java | 46 +++- .../IgniteCacheDatabaseSharedManager.java | 181 ++++++++++++-- .../cache/persistence/RowStore.java | 15 ++ .../evict/PageAbstractEvictionTracker.java | 30 ++- ...eEvictionConcurrentWritesAbstractTest.java | 157 ++++++++++++ .../paged/PageEvictionGuardOomTest.java | 193 +++++++++++++++ .../paged/PageEvictionMetricTest.java | 31 +++ ...geEvictionPutLargeObjectsAbstractTest.java | 5 +- .../PageEvictionSizeAwareAbstractTest.java | 225 ++++++++++++++++++ ...eEvictionWithExpiryPolicyAbstractTest.java | 140 +++++++++++ ...m2LruPageEvictionConcurrentWritesTest.java | 32 +++ .../Random2LruPageEvictionSizeAwareTest.java | 32 +++ ...m2LruPageEvictionWithExpiryPolicyTest.java | 32 +++ ...omLruPageEvictionConcurrentWritesTest.java | 32 +++ .../RandomLruPageEvictionSizeAwareTest.java | 32 +++ ...omLruPageEvictionWithExpiryPolicyTest.java | 32 +++ .../IgniteCacheEvictionSelfTestSuite.java | 18 ++ 19 files changed, 1226 insertions(+), 38 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java index 1293ed7989486..9666295a5203b 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java @@ -18,7 +18,6 @@ import java.io.Serializable; import org.apache.ignite.DataRegionMetrics; -import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.mem.MemoryAllocator; import org.apache.ignite.mxbean.MetricsMxBean; @@ -346,9 +345,9 @@ public DataRegionConfiguration setEvictionThreshold(double evictionThreshold) { * Specifies the minimal number of empty pages to be present in reuse lists for this data region. * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of * (key, value) pair is slightly larger than page size / 2. - * Increase this parameter if cache can contain very big entries (total size of pages in this pool should be enough - * to contain largest cache entry). - * Increase this parameter if {@link IgniteOutOfMemoryException} occurred with enabled page eviction. + * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool, + * it is no longer required to increase this parameter up to the size of the largest cache entry; + * it may be kept at its default as the steady-state reserve of empty pages. * * @return Minimum number of empty pages in reuse list. */ @@ -360,9 +359,9 @@ public int getEmptyPagesPoolSize() { * Specifies the minimal number of empty pages to be present in reuse lists for this data region. * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of * (key, value) pair is slightly larger than page size / 2. - * Increase this parameter if cache can contain very big entries (total size of pages in this pool should be enough - * to contain largest cache entry). - * Increase this parameter if {@link IgniteOutOfMemoryException} occurred with enabled page eviction. + * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool, + * it is no longer required to increase this parameter up to the size of the largest cache entry; + * it may be kept at its default as the steady-state reserve of empty pages. * * @param emptyPagesPoolSize Empty pages pool size. * @return {@code this} for chaining. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index bda9d3fbdc198..0d07d70602c11 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -201,6 +201,24 @@ public interface GridCacheEntryEx { public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap) throws IgniteCheckedException; + /** + * Same as {@link #evictInternal(GridCacheVersion, CacheEntryPredicate[], boolean)}, but acquires the entry lock + * non-blockingly when {@code tryLock} is {@code true}, returning {@code false} (instead of blocking) if the entry + * lock is contended. Used by size-aware page eviction which may run while the current thread already holds other + * entry locks, to avoid a lock-ordering deadlock. The default implementation uses the blocking variant. + * + * @param obsoleteVer Version for eviction. + * @param filter Optional filter. + * @param evictOffheap Evict offheap value flag. + * @param tryLock {@code true} to acquire the entry lock non-blockingly (skip contended entries). + * @return {@code True} if entry could be evicted. + * @throws IgniteCheckedException In case of error. + */ + public default boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, + boolean evictOffheap, boolean tryLock) throws IgniteCheckedException { + return evictInternal(obsoleteVer, filter, evictOffheap); + } + /** * This method should be called each time entry is marked obsolete * other than by calling {@link #markObsolete(GridCacheVersion)}. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 021fdd0408d48..4e5eec7bd999c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3659,6 +3659,10 @@ protected void removeValue() throws IgniteCheckedException { */ private void ensureFreeSpace() throws IgniteCheckedException { // Deadlock alert: evicting data page causes removing (and locking) all entries on the page one by one. + // This entry-level eviction is invoked only while NOT holding this entry's lock (all call sites run before + // lockEntry()). The separately invoked size-aware path (see RowStore.addRow -> + // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs while the lock IS held, so it relies on + // the non-blocking tryLockEntry(ENTRY_LOCK_TIMEOUT) inside evictInternal to avoid a lock-ordering deadlock. assert !lock.isHeldByCurrentThread(); cctx.shared().database().ensureFreeSpace(cctx.dataRegion()); @@ -3687,11 +3691,28 @@ private CacheEntryImplEx wrapVersionedWithValue() { boolean evictOffheap) throws IgniteCheckedException { + return evictInternal(obsoleteVer, filter, evictOffheap, false); + } + + /** {@inheritDoc} */ + @Override public boolean evictInternal( + GridCacheVersion obsoleteVer, + @Nullable CacheEntryPredicate[] filter, + boolean evictOffheap, + boolean tryLock) + throws IgniteCheckedException { + boolean marked = false; try { if (F.isEmptyOrNulls(filter)) { - lockEntry(); + // With tryLock=true (size-aware eviction running while the current thread already holds entry locks) + // the lock is acquired non-blockingly: a contended entry is skipped (returning false) rather than + // blocking, which prevents a lock-ordering deadlock between concurrent evictions. The eviction tracker + // will then pick another page. For all other paths (tryLock=false) the original + // blocking lockEntry() is preserved. + if (!lockEntry(tryLock)) + return false; try { if (evictionDisabled()) { @@ -3728,7 +3749,8 @@ private CacheEntryImplEx wrapVersionedWithValue() { while (true) { GridCacheVersion v; - lockEntry(); + if (!lockEntry(tryLock)) + return false; try { v = ver; @@ -3740,7 +3762,8 @@ private CacheEntryImplEx wrapVersionedWithValue() { if (!cctx.isAll(/*version needed for sync evicts*/this, filter)) return false; - lockEntry(); + if (!lockEntry(tryLock)) + return false; try { if (evictionDisabled()) { @@ -4182,6 +4205,23 @@ private int extrasSize() { lock.lock(); } + /** + * Acquires the entry lock either blocking ({@code tryLock == false}) or non-blockingly with the configured + * {@link #ENTRY_LOCK_TIMEOUT} ({@code tryLock == true}). Used by {@link #evictInternal} to let size-aware + * eviction skip contended entries instead of blocking, avoiding a lock-ordering deadlock. + * + * @param tryLock {@code true} to acquire the lock non-blockingly. + * @return {@code true} if the lock was acquired (always {@code true} when {@code tryLock == false}). + */ + private boolean lockEntry(boolean tryLock) { + if (tryLock) + return tryLockEntry(ENTRY_LOCK_TIMEOUT); + + lockEntry(); + + return true; + } + /** {@inheritDoc} */ @Override public boolean tryLockEntry(long timeout) { try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index b22a6682957c4..4e6bc990281b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -63,6 +63,7 @@ import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress; import org.apache.ignite.internal.processors.cache.persistence.evict.FairFifoPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker; +import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.Random2LruPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker; @@ -1172,32 +1173,56 @@ public WALPointer latestWalPointerReservedForPreloading() { } /** - * Checks that the given {@code region} has enough space for putting a new entry. + * Checks that the given {@code region} has enough space for putting a new entry of {@code dataRowSize} bytes. * - * This method makes sense then and only then - * the data region is not persisted {@link DataRegionConfiguration#isPersistenceEnabled()} - * and page eviction is disabled {@link DataPageEvictionMode#DISABLED}. + * For a non-persistent region with page eviction disabled, verifies that the region reserves enough pages to + * support a free list {@link AbstractFreeList}. For example, removing a row from underlying store may require + * allocating a new data page in order to move a tracked page from one bucket to another one which does not have + * a free space for a new stripe. See {@link AbstractFreeList#removeDataRowByLink}. Therefore, inserting a new + * entry should be prevented in case of some threshold is exceeded. * - * The non-persistent region should reserve a number of pages to support a free list {@link AbstractFreeList}. - * For example, removing a row from underlying store may require allocating a new data page - * in order to move a tracked page from one bucket to another one which does not have a free space for a new stripe. - * See {@link AbstractFreeList#removeDataRowByLink}. - * Therefore, inserting a new entry should be prevented in case of some threshold is exceeded. + * For a non-persistent region with page eviction enabled, additionally performs size-aware eviction: when the + * row does not fit into the currently available page space, data pages are evicted until either enough space is + * freed or it becomes clear that the goal is unreachable (in which case an + * {@link IgniteOutOfMemoryException} is thrown). + * + * The size-aware reserve is required because page eviction by itself only keeps a steady-state pool of empty pages + * ({@link DataRegionConfiguration#getEmptyPagesPoolSize()}) and does not guarantee enough space for a single row + * larger than this pool. * * @param region Data region to be checked. * @param dataRowSize Size of data row to be inserted. - * @throws IgniteOutOfMemoryException In case of the given data region does not have enough free space - * for putting a new entry. + * @throws IgniteOutOfMemoryException In case the given data region does not have enough free space + * for putting a new entry, even after eviction. + * @throws IgniteCheckedException If failed to evict data pages. */ - public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws IgniteOutOfMemoryException { + public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) + throws IgniteOutOfMemoryException, IgniteCheckedException { if (region == null) return; DataRegionConfiguration regCfg = region.config(); - if (regCfg.getPageEvictionMode() != DataPageEvictionMode.DISABLED || regCfg.isPersistenceEnabled()) + if (regCfg.isPersistenceEnabled()) return; + if (regCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED) + checkOomThreshold(region, regCfg, dataRowSize); + else + ensureFreeSpaceForEviction(region, regCfg, dataRowSize); + } + + /** + * Checks that a non-persistent region with disabled page eviction has enough pages for a new row, taking into + * account the pages required to support the free list. + * + * @param region Data region. + * @param regCfg Data region configuration. + * @param dataRowSize Size of data row to be inserted. + * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. + */ + private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) + throws IgniteOutOfMemoryException { long memorySize = regCfg.getMaxSize(); PageMemory pageMem = region.pageMemory(); @@ -1216,24 +1241,128 @@ public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws boolean oomThreshold = (memorySize / pageMem.systemPageSize()) < ((double)dataRowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); - if (oomThreshold) { - IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + - "name=" + regCfg.getName() + - ", initSize=" + U.readableSize(regCfg.getInitialSize(), false) + - ", maxSize=" + U.readableSize(regCfg.getMaxSize(), false) + - ", persistenceEnabled=" + regCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + - " ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + - " ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + - " ^-- Enable eviction or expiration policies" - ); + if (oomThreshold) + throw outOfMemory(regCfg); + } + + /** + * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the region has enough + * available pages to accommodate the row, or throws {@link IgniteOutOfMemoryException} if the goal is + * unreachable / no progress can be made. + * + * @param region Data region. + * @param regCfg Data region configuration. + * @param dataRowSize Size of data row to be inserted. + * @throws IgniteOutOfMemoryException If the target cannot be reached (row too large for the region or eviction + * makes no progress). + * @throws IgniteCheckedException If failed to evict data pages. + */ + private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) + throws IgniteOutOfMemoryException, IgniteCheckedException { + PageMemory pageMem = region.pageMemory(); + + CacheFreeList freeList = freeListMap.get(regCfg.getName()); - if (cctx.kernalContext() != null) - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); + if (freeList == null) + return; + + long sysPageSize = pageMem.systemPageSize(); + long pageSize = pageMem.pageSize(); + + long totalPages = regCfg.getMaxSize() / sysPageSize; + + // Pages required to place the row (rounded up) plus a margin for the page header and fragmentation. + long requiredPages = (dataRowSize + pageSize - 1) / pageSize + 1; + + // If the row fits into the configured steady-state empty-pages pool, normal threshold eviction is enough. + if (requiredPages <= regCfg.getEmptyPagesPoolSize()) + return; - throw oom; + // The row fundamentally cannot fit into the whole region. + if (requiredPages > totalPages) + throw outOfMemory(regCfg); + + long availablePages = (totalPages - pageMem.loadedPages()) + freeList.emptyDataPages(); + + // Fast path: enough pages are already available, no eviction is needed. + if (availablePages >= requiredPages) + return; + + PageEvictionTracker evictionTracker = region.evictionTracker(); + + // Evict data pages until enough free space is available. Progress is measured against the overall available + // space, so pages freed concurrently (e.g. by TTL cleanup) also count as progress. The loop is bounded to + // avoid an infinite busy-spin when there is nothing more to evict. Eviction here runs while the current + // thread may already hold entry locks (single-row insertion), so entries whose locks are contended are + // skipped (non-blocking) rather than blocked upon, to avoid a lock-ordering deadlock. + final int maxAttemptsWithoutProgress = 300; + + long bestAvailable = availablePages; + int attemptsWithoutProgress = 0; + + while (bestAvailable < requiredPages) { + evictDataPageNonBlocking(evictionTracker); + + long curAvailable = (totalPages - pageMem.loadedPages()) + freeList.emptyDataPages(); + + // Progress is measured against the best available space observed so far. Concurrent inserts may + // temporarily reduce available (loadedPages grows) even while eviction is freeing pages, so a drop below + // the running best is not treated as "no progress". Only when available fails to exceed the best value + // over many attempts we conclude that no more space can be freed (e.g. all candidate entries are locked + // by other threads/transactions). + if (curAvailable > bestAvailable) { + bestAvailable = curAvailable; + + attemptsWithoutProgress = 0; + } + else if (curAvailable < bestAvailable) { + // A transient drop caused by concurrent activity: keep the best value, do not penalize. + } + else + attemptsWithoutProgress++; + + if (attemptsWithoutProgress >= maxAttemptsWithoutProgress) + throw outOfMemory(regCfg); } } + /** + * Invokes a single page eviction, acquiring entry locks non-blockingly so that contended entries are skipped. + * This is required when eviction runs while the current thread already holds entry locks (size-aware eviction + * from a single-row insertion) to avoid a lock-ordering deadlock. {@link NoOpPageEvictionTracker} + * (disabled eviction, never reaching this path) falls back to the plain {@code evictDataPage()}. + * + * @param evictionTracker Page eviction tracker. + * @throws IgniteCheckedException If failed to evict a data page. + */ + private void evictDataPageNonBlocking(PageEvictionTracker evictionTracker) throws IgniteCheckedException { + if (evictionTracker instanceof PageAbstractEvictionTracker) + ((PageAbstractEvictionTracker)evictionTracker).evictDataPageNonBlocking(); + else + evictionTracker.evictDataPage(); + } + + /** + * @param regCfg Data region configuration. + * @return New {@link IgniteOutOfMemoryException} (also reported as a critical failure) for the given region. + */ + private IgniteOutOfMemoryException outOfMemory(DataRegionConfiguration regCfg) { + IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + + "name=" + regCfg.getName() + + ", initSize=" + U.readableSize(regCfg.getInitialSize(), false) + + ", maxSize=" + U.readableSize(regCfg.getMaxSize(), false) + + ", persistenceEnabled=" + regCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + + " ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + + " ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + + " ^-- Enable eviction or expiration policies" + ); + + if (cctx.kernalContext() != null) + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); + + return oom; + } + /** * See {@code GridCacheMapEntry#ensureFreeSpace()} * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index cffcf9b1e5be0..86e5e94595d1d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.function.Supplier; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.internal.metric.IoStatisticsHolder; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; @@ -134,6 +135,20 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { + if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { + // Size-aware reserve for the largest row in the batch. Eviction performed here runs without entry locks + // (see AbstractFreeList#insertDataRows), so reserving space for any single large row is safe and keeps the + // batch path consistent with the single-row path. Smaller rows are covered by the regular + // threshold eviction loop inside insertDataRows. + int maxRowSize = 0; + + for (CacheDataRow row : rows) + maxRowSize = Math.max(maxRowSize, row.size()); + + if (maxRowSize > 0) + ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), maxRowSize); + } + assert ctx.database().checkpointLockIsHeldByThread(); freeList.insertDataRows(rows, statHolder); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index 2330c0942662d..fade3443b109c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -41,6 +41,13 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker /** Millis in day. */ private static final int DAY = 24 * 60 * 60 * 1000; + /** + * Thread-local marker that the current eviction is requested by size-aware eviction, which may run + * while the calling thread already holds entry locks. When set, entries whose locks are contended are skipped + * (via a non-blocking {@code evictInternal}) instead of blocking, avoiding a lock-ordering deadlock. + */ + private static final ThreadLocal EVICT_NON_BLOCKING = new ThreadLocal<>(); + /** Page memory. */ protected final PageMemoryNoStoreImpl pageMem; @@ -87,6 +94,26 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker return pageMem.loadedPages() > pagesThreshold && freeList.emptyDataPages() < regCfg.getEmptyPagesPoolSize(); } + /** + * Evicts a data page, acquiring entry locks in a non-blocking way so that contended entries are skipped instead + * of blocked upon. Used by size-aware eviction which may run while the calling thread already holds + * entry locks, to avoid a lock-ordering deadlock. + * + * @throws IgniteCheckedException If failed. + */ + public void evictDataPageNonBlocking() throws IgniteCheckedException { + Boolean prev = EVICT_NON_BLOCKING.get(); + + EVICT_NON_BLOCKING.set(Boolean.TRUE); + + try { + evictDataPage(); + } + finally { + EVICT_NON_BLOCKING.set(prev); + } + } + /** * @param pageIdx Page index. * @return true if at least one data row has been evicted @@ -144,7 +171,8 @@ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) : cacheCtx.cache().entryEx(dataRow.key()); - evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true); + evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, + Boolean.TRUE.equals(EVICT_NON_BLOCKING.get())); } return evictionDone; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java new file mode 100644 index 0000000000000..e6d5d518fb7e3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; + +/** + * Concurrent deadlock test for size-aware page eviction. + *

+ * The region is first filled with a large number of small entries (so there is plenty of evictable page space), then + * several threads concurrently insert large rows (larger than the empty-pages pool). Each large insert goes through + * the size-aware reserve and, for the single-row path, eviction under the new entry lock with the non-blocking + * {@code tryLockEntry}. The average data volume is kept within the region capacity, so eviction frees already-stored + * small entries rather than overrunning the free list. The test asserts that no deadlock occurs (all threads finish + * within a global deadline). + */ +public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommonAbstractTest { + /** Off-heap region size. */ + private static final int SIZE = 128 * 1024 * 1024; + + /** Partition count (kept low so that index-tree structures do not exhaust the region). */ + private static final int PARTITIONS = 32; + + /** Large record size (much larger than the empty-pages pool). */ + private static final int LARGE_RECORD_SIZE = 4 * 1024 * 1024; + + /** Small record size used to pre-fill the region with evictable data. */ + private static final int SMALL_RECORD_SIZE = 4096; + + /** Empty pages pool size. */ + private static final int POOL_SIZE = 100; + + /** Number of small pre-fill entries. */ + private static final int SMALL_ENTRIES = 10_000; + + /** Number of writer threads. */ + private static final int THREADS = 4; + + /** Large rows inserted per thread (moderate total, kept within region capacity after eviction of small rows). */ + private static final int LARGE_ROWS_PER_THREAD = 3; + + /** Global deadline for the whole test (protects against a deadlock/busy-spin hang). */ + private static final long DEADLINE = TimeUnit.MINUTES.toMillis(3); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return super.getConfiguration(gridName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(SIZE) + .setMaxSize(SIZE) + .setEmptyPagesPoolSize(POOL_SIZE) + ) + .setPageSize(DFLT_PAGE_SIZE) + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * @param ignite Ignite node. + * @return Cache with a small partition count (reduces structural page overhead). + */ + private IgniteCache createCache(IgniteEx ignite) { + return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS))); + } + + /** + * Concurrent large inserts into a region pre-filled with small entries must complete within the deadline without + * deadlock, and without corrupting the free list (eviction frees small entries rather than overrunning the region). + * + * @throws Exception If failed. + */ + @Test + public void testConcurrentLargeWritesNoDeadlock() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + // Pre-fill the region with many small entries so that eviction always has evictable pages to free. + for (int i = 0; i < SMALL_ENTRIES; i++) + cache.put(i, new byte[SMALL_RECORD_SIZE]); + + byte[] largeVal = new byte[LARGE_RECORD_SIZE]; + + AtomicLong errors = new AtomicLong(); + + CountDownLatch startLatch = new CountDownLatch(1); + + long deadline = System.currentTimeMillis() + DEADLINE; + + Thread[] threads = new Thread[THREADS]; + + for (int i = 0; i < THREADS; i++) { + final int threadIdx = i; + + threads[i] = new Thread(() -> { + try { + startLatch.await(); + + for (int k = 0; k < LARGE_ROWS_PER_THREAD; k++) + cache.put(SMALL_ENTRIES + threadIdx * LARGE_ROWS_PER_THREAD + k, largeVal); + } + catch (Throwable e) { + errors.incrementAndGet(); + + log.error("Unexpected error in writer thread", e); + } + }, "paged-writer-" + i); + + threads[i].start(); + } + + startLatch.countDown(); + + for (Thread t : threads) + t.join(Math.max(1, deadline - System.currentTimeMillis())); + + // The core assertion of this deadlock test: every writer must have completed (no thread is stuck waiting on + // an entry lock held by size-aware eviction running under another entry lock). + for (Thread t : threads) + assertFalse("Writer thread " + t.getName() + " did not finish (possible deadlock)", t.isAlive()); + + assertEquals("Writer threads reported errors", 0, errors.get()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java new file mode 100644 index 0000000000000..03af63f8ff0e6 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; + +/** + * Negative test for the size-aware eviction progress guard. + *

+ * When every resident entry is locked by another thread/transaction, page eviction cannot free any page: the guarded + * {@code tryLockEntry} in {@code evictInternal} fails for every candidate, so {@link + * IgniteCacheDatabaseSharedManager#ensureFreeSpaceForEviction} makes no progress and must fail with an + * {@code IgniteOutOfMemoryException} within bounded time instead of busy-spinning forever (deadlock). + *

+ * The lock timeout is reduced via {@code -DENTRY_LOCK_TIMEOUT=1} (applied through {@code @WithSystemProperty} before + * the node starts) so that each non-blocking lock attempt fails quickly and the whole guard run stays within a few + * seconds. The test is self-guarded by {@code @Test(timeout = ...)}: a deadlock or unbounded busy-spin would fail the + * deadline. + */ +public class PageEvictionGuardOomTest extends GridCommonAbstractTest { + /** Off-heap region size. */ + private static final int SIZE = 12 * 1024 * 1024; + + /** Partition count (kept low so that index-tree structures do not exhaust the region). */ + private static final int PARTITIONS = 32; + + /** Empty pages pool size. */ + private static final int POOL_SIZE = 100; + + /** Small record size chosen to occupy roughly one data page ({@link DFLT_PAGE_SIZE}) each. */ + private static final int FILL_VALUE_SIZE = 3_800; + + /** + * Number of resident entries (each ~one page) filling the region to ~55% of its capacity. This keeps the region + * comfortably below the eviction threshold (so the ordinary threshold-based {@code ensureFreeSpace} path is a + * no-op) while leaving less free space than a single large record needs, so the size-aware eviction guard is + * exercised. + */ + private static final int FILL_ENTRIES = 1_600; + + /** Large record size that does not fit into the remaining free space (requires eviction to be stored). */ + private static final int LARGE_RECORD_SIZE = 8 * 1024 * 1024; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return super.getConfiguration(gridName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(SIZE) + .setMaxSize(SIZE) + .setEmptyPagesPoolSize(POOL_SIZE) + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU) + ) + .setPageSize(DFLT_PAGE_SIZE) + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * @param ignite Ignite node. + * @return Cache with a small partition count (reduces structural page overhead). + */ + private IgniteCache createCache(IgniteEx ignite) { + // TRANSACTIONAL is required so that cache.lockAll(...) can hold entry locks (the root cause of the + // "no evictable page" scenario this test exercises). + return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + } + + /** + * Filling the region with locked entries and then writing a row that needs more free pages than remain must fail + * with OOM (bounded time), not hang: eviction cannot free any page because every candidate entry is locked. + * + * @throws Exception If failed. + */ + @Test(timeout = 180_000) + @WithSystemProperty(key = "ENTRY_LOCK_TIMEOUT", value = "1") + public void testGuardOomWhenAllEntriesLocked() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + // Pre-fill the region so that less than one large record of free space remains, without overflowing it. + byte[] fillVal = new byte[FILL_VALUE_SIZE]; + + for (int i = 1; i <= FILL_ENTRIES; i++) + cache.put(i, fillVal); + + Collection keys = new ArrayList<>(FILL_ENTRIES); + + for (int i = 1; i <= FILL_ENTRIES; i++) + keys.add(i); + + CountDownLatch ready = new CountDownLatch(1); + + CountDownLatch release = new CountDownLatch(1); + + AtomicReference lockerErr = new AtomicReference<>(); + + // Hold entry locks on every resident key from a background thread so that eviction has no evictable page. + Thread locker = new Thread(() -> { + try { + Lock lock = cache.lockAll(keys); + + lock.lock(); + + ready.countDown(); + + release.await(); + + lock.unlock(); + } + catch (Throwable e) { + lockerErr.set(e); + + ready.countDown(); + } + }, "size-aware-guard-locker"); + + locker.start(); + + assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); + + assertNull("Unexpected error while locking entries: " + lockerErr.get(), lockerErr.get()); + + try { + cache.put(FILL_ENTRIES + 1, new byte[LARGE_RECORD_SIZE]); + + fail("Expected out-of-memory because all resident entries are locked, but put succeeded"); + } + catch (Exception e) { + assertTrue("Expected an out-of-memory (progress guard) failure, but got: " + e, isOutOfMemory(e)); + } + finally { + release.countDown(); + + locker.join(TimeUnit.SECONDS.toMillis(10)); + } + } + + /** + * @param t Throwable. + * @return {@code True} if {@code t} or any of its causes is an out-of-memory. + */ + private static boolean isOutOfMemory(Throwable t) { + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof IgniteOutOfMemoryException) + return true; + } + + return false; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java index 9f40cf4958431..b7a86a2459611 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java @@ -49,6 +49,37 @@ public void testPageEvictionMetric() throws Exception { checkPageEvictionMetric(CacheAtomicityMode.ATOMIC); } + /** + * Regression: ordinary small records that keep the region below the eviction threshold must not trigger page + * eviction at all (eviction is not started, eviction rate stays zero). + * + * @throws Exception If failed. + */ + @Test + public void testNoEvictionBelowThreshold() throws Exception { + IgniteEx ignite = startGrid(0); + + DataRegionMetricsImpl metrics = + ignite.context().cache().context().database().dataRegion(null).metrics(); + + metrics.enableMetrics(); + + CacheConfiguration cfg = cacheConfig("no-evict-below-threshold", null, + CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC, CacheWriteSynchronizationMode.PRIMARY_SYNC); + + IgniteCache cache = ignite.getOrCreateCache(cfg); + + // A small number of records far below the eviction threshold and empty-pages pool pressure. + for (int i = 1; i <= 500; i++) + cache.put(i, new TestObject(PAGE_SIZE / 6)); + + assertFalse("Page eviction must not start while the region is below the eviction threshold", + metrics.isEvictionsStarted()); + + assertEquals("Eviction rate must be zero while the region is below the eviction threshold", + 0f, metrics.getEvictionRate(), 0f); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java index b141fa82bb357..4ece99be95f6e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java @@ -70,6 +70,9 @@ public void testPutLargeObjects() throws Exception { for (Integer key : primaryKeys(grid(1).cache(DEFAULT_CACHE_NAME), ENTRIES)) cache.put(key, val); - assertTrue(cache.size() < ENTRIES); + // With size-aware eviction the large records do not fail with OOM: older records are evicted to + // make room for the newer ones. The resident set must therefore be bounded well below the total written + // (50 x 80MB >> 1GB region) but stay non-empty (at least the most recently written entries survive). + assertTrue(cache.size() > 0 && cache.size() < ENTRIES); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java new file mode 100644 index 0000000000000..6731287fae957 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; + +/** + * Tests size-aware page eviction on in-memory (non-persistent) data regions. + * + * Verifies that a row larger than the configured {@code emptyPagesPoolSize} (in pages) is still written successfully + * when page eviction is enabled, by evicting old entries to free enough space. Also verifies that a row + * which fundamentally cannot fit into the region fails with OOM instead of hanging in an infinite eviction loop. + * + * Note: the atomic DHT batch path (putAll of many large rows overflowing a small region) is out of scope here — it is + * handled by a separate size-aware reserve in the batch store path and already fails on the original code. + */ +public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstractTest { + /** Off-heap region size (large enough to hold cache structural pages with the configured partition count). */ + private static final int SIZE = 128 * 1024 * 1024; + + /** Partition count (kept low so that index-tree structures do not exhaust the region). */ + private static final int PARTITIONS = 32; + + /** Record size: chosen to be much larger than {@code emptyPagesPoolSize} pages. */ + private static final int RECORD_SIZE = 4 * 1024 * 1024; + + /** Empty pages pool size. */ + private static final int POOL_SIZE = 100; + + /** Entry count to accumulate beyond the region capacity. */ + private static final int ENTRIES = 40; + + /** Small record size used to pre-fill the region with evictable data (for putAll tests). */ + private static final int SMALL_RECORD_SIZE = 4096; + + /** Small pre-fill entries count. */ + private static final int SMALL_ENTRIES = 8000; + + /** Large rows written via putAll. */ + private static final int PUT_ALL_LARGE_ROWS = 3; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return super.getConfiguration(gridName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(SIZE) + .setMaxSize(SIZE) + .setEmptyPagesPoolSize(POOL_SIZE) + ) + .setPageSize(DFLT_PAGE_SIZE) + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * @param ignite Ignite node. + * @return Cache with a small partition count (reduces structural page overhead). + */ + private IgniteCache createCache(IgniteEx ignite) { + return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS))); + } + + /** + * A large record (larger than the empty-pages pool) must be stored without OOM when there is evictable data, + * by evicting previously stored records to free enough space. + * + * @throws Exception If failed. + */ + @Test + public void testPutLargeObjectsDoesNotOom() throws Exception { + IgniteEx ignite = startGrids(2); + + IgniteCache cache = createCache(ignite); + + Object val = new byte[RECORD_SIZE]; + + // Total data (ENTRIES * RECORD_SIZE) exceeds the region size, so at least some records must be evicted. + for (Integer key : primaryKeys(grid(1).cache(DEFAULT_CACHE_NAME), ENTRIES)) + cache.put(key, val); + + // Eviction must have bounded the number of resident entries. + assertTrue("Expected some entries to be evicted, but cache.size()=" + cache.size(), + cache.size() > 0 && cache.size() < ENTRIES); + } + + /** + * A large record written must be readable right away (the just-written entry is the most recently used and is not + * a candidate for eviction before the write completes). + * + * @throws Exception If failed. + */ + @Test + public void testLargeObjectReadBack() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + byte[] val = new byte[RECORD_SIZE]; + + Arrays.fill(val, (byte)42); + + cache.put(1, val); + + byte[] read = (byte[])cache.get(1); + + assertNotNull("Large value must be readable after put", read); + + assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); + } + + /** + * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. + * + * @throws Exception If failed. + */ + @Test + public void testRecordLargerThanRegionOom() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + boolean rejected = false; + + try { + cache.put(1, new byte[SIZE * 2]); + } + catch (Exception e) { + // OOM (possibly wrapped) because the row cannot fit into the region. + rejected = true; + } + + assertTrue("Record larger than the region must be rejected (no hang), but put succeeded", rejected); + } + + /** + * A batch putAll of several large records (each larger than the empty-pages pool) must be stored successfully when + * page eviction is enabled. Exercises the size-aware reserve in the batch store path ({@code RowStore.addRows}). + * + * @throws Exception If failed. + */ + @Test + public void testPutAllLargeRows() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + // Pre-fill with small evictable entries so large rows below region capacity fit via the reserve path. + byte[] small = new byte[SMALL_RECORD_SIZE]; + + for (int i = 0; i < SMALL_ENTRIES; i++) + cache.put(SMALL_ENTRIES + i, small); + + Map large = new HashMap<>(); + + Object val = new byte[RECORD_SIZE]; + + for (int i = 0; i < PUT_ALL_LARGE_ROWS; i++) + large.put(i, val); + + cache.putAll(large); + + for (int i = 0; i < PUT_ALL_LARGE_ROWS; i++) + assertNotNull("Large row " + i + " must be readable after putAll", cache.get(i)); + } + + /** + * Updating a record from a small to a large value (larger than the empty-pages pool) must succeed with page + * eviction enabled: the update goes through the same size-aware reserve as an insert. + * + * @throws Exception If failed. + */ + @Test + public void testUpdateRowGrows() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite); + + cache.put(1, new byte[1024]); + + byte[] big = new byte[RECORD_SIZE]; + + Arrays.fill(big, (byte)7); + + cache.put(1, big); + + byte[] read = (byte[])cache.get(1); + + assertNotNull("Updated large value must be readable", read); + + assertTrue("Updated value must equal the stored value", Arrays.equals(big, read)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java new file mode 100644 index 0000000000000..c6dd438367de1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import javax.cache.expiry.CreatedExpiryPolicy; +import javax.cache.expiry.Duration; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; + +/** + * Tests the synergy between ExpiryPolicy (TTL cleanup) and size-aware page eviction on an in-memory data region. + * Verifies that concurrent TTL cleanup and eviction do not deadlock, that a large row larger than the + * empty-pages pool is still written when eviction is enabled, and that TTL-freed space is accounted for by eviction + * (a row that only fits after expired entries are removed is still written without OOM). + */ +public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommonAbstractTest { + /** Off-heap region size. */ + private static final int SIZE = 128 * 1024 * 1024; + + /** Partition count (kept low so that index-tree structures do not exhaust the region). */ + private static final int PARTITIONS = 32; + + /** Large record size (much larger than the empty-pages pool). */ + private static final int RECORD_SIZE = 8 * 1024 * 1024; + + /** Empty pages pool size. */ + private static final int POOL_SIZE = 100; + + /** Short TTL applied to some entries. */ + private static final long TTL = 1500; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return super.getConfiguration(gridName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(SIZE) + .setMaxSize(SIZE) + .setEmptyPagesPoolSize(POOL_SIZE) + ) + .setPageSize(DFLT_PAGE_SIZE) + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * @param ignite Ignite node. + * @param ttl TTL in milliseconds ({@code 0} for no expiry). + * @return Cache with a small partition count and, if {@code ttl > 0}, eager TTL expiry. + */ + private IgniteCache createCache(IgniteEx ignite, long ttl) { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)); + + if (ttl > 0) { + ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(MILLISECONDS, ttl))) + .setEagerTtl(true); + } + + return ignite.createCache(ccfg); + } + + /** + * Concurrent TTL cleanup and eviction must not deadlock, and a large record (larger than the empty-pages pool) + * must still be stored on a region with enabled eviction even in the presence of short-TTL entries. + * + * @throws Exception If failed. + */ + @Test + public void testLargePutWithExpiryNoDeadlock() throws Exception { + IgniteEx ignite = startGrid(1); + + // Short-TTL entries keep the TTL worker actively freeing pages while eviction runs. + IgniteCache cache = createCache(ignite, TTL); + + Object val = new byte[RECORD_SIZE]; + + // Writing more data than the region can hold forces eviction; concurrent expiry of short-TTL entries must not + // deadlock with it. The test itself is protected against a hang by the framework test timeout. + for (int i = 0; i < 30; i++) + cache.put(i, val); + + cache.get(0); + } + + /** + * Space freed by TTL cleanup must be taken into account by size-aware eviction: a large record written after some + * entries have expired must be accepted (no OOM) because their pages become available. + * + * @throws Exception If failed. + */ + @Test + public void testTtlFreedSpaceAccountedForByEviction() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite, TTL); + + // Fill the region up to its capacity with short-TTL large records. + Object val = new byte[RECORD_SIZE]; + + for (int i = 0; i < 10; i++) + cache.put(i, val); + + // Wait for the TTL worker to expire and free the short-TTL entries. + Thread.sleep(TTL + 1500); + + // A fresh large record must now be accepted (space freed by TTL counts as available for eviction). + cache.put(100, val); + + assertNotNull(cache.get(100)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java new file mode 100644 index 0000000000000..320d48cdde1cd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_2_LRU}. + */ +public class Random2LruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_2_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java new file mode 100644 index 0000000000000..f9b9e405905cf --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_2_LRU}. + */ +public class Random2LruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_2_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java new file mode 100644 index 0000000000000..cdbb5218c05b1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_2_LRU}. + */ +public class Random2LruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_2_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java new file mode 100644 index 0000000000000..d3dc5b99d0437 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_LRU}. + */ +public class RandomLruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java new file mode 100644 index 0000000000000..b2bb394b65f0e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_LRU}. + */ +public class RandomLruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java new file mode 100644 index 0000000000000..ff5a209da1eb5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; + +/** + * ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_LRU}. + */ +public class RandomLruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index 03d95e5fabbcb..0b240cf0f23dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -38,19 +38,26 @@ import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionGuardOomTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMetricTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionPagesRecyclingAndReusingTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruNearEnabledPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionConcurrentWritesTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionDataStreamerTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionMultinodeTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionPutLargeObjectsTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionSizeAwareTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionWithExpiryPolicyTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruNearEnabledPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionConcurrentWritesTest; import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionDataStreamerTest; import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionMultinodeTest; import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionPutLargeObjectsTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionSizeAwareTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithExpiryPolicyTest; import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; @@ -100,8 +107,19 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionPutLargeObjectsTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionPutLargeObjectsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionSizeAwareTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionSizeAwareTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionWithExpiryPolicyTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionWithExpiryPolicyTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionConcurrentWritesTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionConcurrentWritesTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionMetricTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionGuardOomTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionPagesRecyclingAndReusingTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DhtAndNearEvictionTest.class, ignoredTests); From b159894a0b692222e4c8a2c53e92c53b0f902871 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 4 Sep 2026 13:58:05 +0500 Subject: [PATCH 02/26] address copilot comments --- .../processors/cache/GridCacheMapEntry.java | 8 +-- .../IgniteCacheDatabaseSharedManager.java | 27 ++++++---- .../cache/persistence/RowStore.java | 22 ++++---- ...eEvictionConcurrentWritesAbstractTest.java | 13 +++-- .../paged/PageEvictionGuardOomTest.java | 45 ++++++---------- .../paged/PageEvictionMetricTest.java | 3 +- .../PageEvictionSizeAwareAbstractTest.java | 53 ++++++++++++++----- ...eEvictionWithExpiryPolicyAbstractTest.java | 48 ++++++++++++----- ...m2LruPageEvictionConcurrentWritesTest.java | 5 +- .../Random2LruPageEvictionSizeAwareTest.java | 5 +- ...m2LruPageEvictionWithExpiryPolicyTest.java | 5 +- ...omLruPageEvictionConcurrentWritesTest.java | 5 +- .../RandomLruPageEvictionSizeAwareTest.java | 5 +- ...omLruPageEvictionWithExpiryPolicyTest.java | 5 +- 14 files changed, 140 insertions(+), 109 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 4e5eec7bd999c..3f5abc02f9f22 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3662,7 +3662,7 @@ private void ensureFreeSpace() throws IgniteCheckedException { // This entry-level eviction is invoked only while NOT holding this entry's lock (all call sites run before // lockEntry()). The separately invoked size-aware path (see RowStore.addRow -> // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs while the lock IS held, so it relies on - // the non-blocking tryLockEntry(ENTRY_LOCK_TIMEOUT) inside evictInternal to avoid a lock-ordering deadlock. + // the non-blocking tryLockEntry(0) inside evictInternal to avoid a lock-ordering deadlock. assert !lock.isHeldByCurrentThread(); cctx.shared().database().ensureFreeSpace(cctx.dataRegion()); @@ -4206,8 +4206,8 @@ private int extrasSize() { } /** - * Acquires the entry lock either blocking ({@code tryLock == false}) or non-blockingly with the configured - * {@link #ENTRY_LOCK_TIMEOUT} ({@code tryLock == true}). Used by {@link #evictInternal} to let size-aware + * Acquires the entry lock either blocking ({@code tryLock == false}) or non-blockingly with an immediate + * {@code tryLock(0)} ({@code tryLock == true}). Used by {@link #evictInternal} to let size-aware * eviction skip contended entries instead of blocking, avoiding a lock-ordering deadlock. * * @param tryLock {@code true} to acquire the lock non-blockingly. @@ -4215,7 +4215,7 @@ private int extrasSize() { */ private boolean lockEntry(boolean tryLock) { if (tryLock) - return tryLockEntry(ENTRY_LOCK_TIMEOUT); + return !lock.isHeldByCurrentThread() && tryLockEntry(0); lockEntry(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 4e6bc990281b2..152b716dbe527 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -75,6 +75,7 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageReadWriteManager; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.warmup.WarmUpStrategy; @@ -1271,8 +1272,11 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati long totalPages = regCfg.getMaxSize() / sysPageSize; - // Pages required to place the row (rounded up) plus a margin for the page header and fragmentation. - long requiredPages = (dataRowSize + pageSize - 1) / pageSize + 1; + // Maximum payload bytes that a single data page can hold for a fragmented row. + long pagePayload = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; + + // Pages required to place the row, computed from the actual data-page payload capacity. + long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; // If the row fits into the configured steady-state empty-pages pool, normal threshold eviction is enough. if (requiredPages <= regCfg.getEmptyPagesPoolSize()) @@ -1301,23 +1305,26 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati int attemptsWithoutProgress = 0; while (bestAvailable < requiredPages) { + if (region.metrics().onPageEvictionsStarted()) { + U.warn(log, "Page-based evictions started." + + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); + } + evictDataPageNonBlocking(evictionTracker); + region.metrics().updateEvictionRate(); + long curAvailable = (totalPages - pageMem.loadedPages()) + freeList.emptyDataPages(); - // Progress is measured against the best available space observed so far. Concurrent inserts may - // temporarily reduce available (loadedPages grows) even while eviction is freeing pages, so a drop below - // the running best is not treated as "no progress". Only when available fails to exceed the best value - // over many attempts we conclude that no more space can be freed (e.g. all candidate entries are locked - // by other threads/transactions). + // Progress is measured against the best available space observed so far. Any iteration that does not + // establish a new best (including drops caused by concurrent inserts consuming pages) counts toward the + // no-progress guard, so the loop is bounded: if eviction cannot outpace concurrent consumption within + // a fixed number of attempts, an OOM is thrown rather than busy-spinning indefinitely. if (curAvailable > bestAvailable) { bestAvailable = curAvailable; attemptsWithoutProgress = 0; } - else if (curAvailable < bestAvailable) { - // A transient drop caused by concurrent activity: keep the best value, do not penalize. - } else attemptsWithoutProgress++; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 86e5e94595d1d..3a09efaae10e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -136,17 +136,17 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - // Size-aware reserve for the largest row in the batch. Eviction performed here runs without entry locks - // (see AbstractFreeList#insertDataRows), so reserving space for any single large row is safe and keeps the - // batch path consistent with the single-row path. Smaller rows are covered by the regular - // threshold eviction loop inside insertDataRows. - int maxRowSize = 0; - - for (CacheDataRow row : rows) - maxRowSize = Math.max(maxRowSize, row.size()); - - if (maxRowSize > 0) - ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), maxRowSize); + // Size-aware reserve for each row in the batch. Eviction performed here runs without entry locks + // (see AbstractFreeList#insertDataRows), so this is safe. Reserving only the largest row is insufficient: + // insertDataRows consumes the reserve while writing the first large row, and its per-row threshold loop + // only restores emptyPagesPoolSize, which is smaller than a large row. A later large row in the batch + // would therefore exhaust page memory. + for (CacheDataRow row : rows) { + int rowSize = row.size(); + + if (rowSize > 0) + ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), rowSize); + } } assert ctx.database().checkpointLockIsHeldByThread(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index e6d5d518fb7e3..3b2f58d836fad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import java.util.concurrent.CountDownLatch; @@ -49,7 +50,7 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo private static final int PARTITIONS = 32; /** Large record size (much larger than the empty-pages pool). */ - private static final int LARGE_RECORD_SIZE = 4 * 1024 * 1024; + private static final int LARGE_RECORD_SIZE = 16 * 1024 * 1024; /** Small record size used to pre-fill the region with evictable data. */ private static final int SMALL_RECORD_SIZE = 4096; @@ -58,13 +59,13 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo private static final int POOL_SIZE = 100; /** Number of small pre-fill entries. */ - private static final int SMALL_ENTRIES = 10_000; + private static final int SMALL_ENTRIES = 30_000; /** Number of writer threads. */ private static final int THREADS = 4; /** Large rows inserted per thread (moderate total, kept within region capacity after eviction of small rows). */ - private static final int LARGE_ROWS_PER_THREAD = 3; + private static final int LARGE_ROWS_PER_THREAD = 1; /** Global deadline for the whole test (protects against a deadlock/busy-spin hang). */ private static final long DEADLINE = TimeUnit.MINUTES.toMillis(3); @@ -76,10 +77,8 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE) - ) - .setPageSize(DFLT_PAGE_SIZE) - ); + .setEmptyPagesPoolSize(POOL_SIZE)) + .setPageSize(DFLT_PAGE_SIZE)); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 03af63f8ff0e6..51b36928a22ea 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import java.util.ArrayList; @@ -31,25 +32,22 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; -import org.apache.ignite.testframework.junits.WithSystemProperty; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; +import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionSizeAwareAbstractTest.isOutOfMemory; /** * Negative test for the size-aware eviction progress guard. *

* When every resident entry is locked by another thread/transaction, page eviction cannot free any page: the guarded - * {@code tryLockEntry} in {@code evictInternal} fails for every candidate, so {@link + * {@code tryLockEntry(0)} in {@code evictInternal} fails for every candidate, so {@link * IgniteCacheDatabaseSharedManager#ensureFreeSpaceForEviction} makes no progress and must fail with an * {@code IgniteOutOfMemoryException} within bounded time instead of busy-spinning forever (deadlock). *

- * The lock timeout is reduced via {@code -DENTRY_LOCK_TIMEOUT=1} (applied through {@code @WithSystemProperty} before - * the node starts) so that each non-blocking lock attempt fails quickly and the whole guard run stays within a few - * seconds. The test is self-guarded by {@code @Test(timeout = ...)}: a deadlock or unbounded busy-spin would fail the + * The test is self-guarded by {@code @Test(timeout = ...)}: a deadlock or unbounded busy-spin would fail the * deadline. */ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { @@ -84,8 +82,7 @@ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { .setInitialSize(SIZE) .setMaxSize(SIZE) .setEmptyPagesPoolSize(POOL_SIZE) - .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU) - ) + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)) .setPageSize(DFLT_PAGE_SIZE) ); } @@ -114,7 +111,6 @@ private IgniteCache createCache(IgniteEx ignite) { * @throws Exception If failed. */ @Test(timeout = 180_000) - @WithSystemProperty(key = "ENTRY_LOCK_TIMEOUT", value = "1") public void testGuardOomWhenAllEntriesLocked() throws Exception { IgniteEx ignite = startGrid(1); @@ -159,17 +155,19 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { locker.start(); - assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); + try { + assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); - assertNull("Unexpected error while locking entries: " + lockerErr.get(), lockerErr.get()); + assertNull("Unexpected error while locking entries: " + lockerErr.get(), lockerErr.get()); - try { - cache.put(FILL_ENTRIES + 1, new byte[LARGE_RECORD_SIZE]); + try { + cache.put(FILL_ENTRIES + 1, new byte[LARGE_RECORD_SIZE]); - fail("Expected out-of-memory because all resident entries are locked, but put succeeded"); - } - catch (Exception e) { - assertTrue("Expected an out-of-memory (progress guard) failure, but got: " + e, isOutOfMemory(e)); + fail("Expected out-of-memory because all resident entries are locked, but put succeeded"); + } + catch (Exception e) { + assertTrue("Expected an out-of-memory (progress guard) failure, but got: " + e, isOutOfMemory(e)); + } } finally { release.countDown(); @@ -177,17 +175,4 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { locker.join(TimeUnit.SECONDS.toMillis(10)); } } - - /** - * @param t Throwable. - * @return {@code True} if {@code t} or any of its causes is an out-of-memory. - */ - private static boolean isOutOfMemory(Throwable t) { - for (Throwable cur = t; cur != null; cur = cur.getCause()) { - if (cur instanceof IgniteOutOfMemoryException) - return true; - } - - return false; - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java index b7a86a2459611..6efd5b6bffbd0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java @@ -59,8 +59,7 @@ public void testPageEvictionMetric() throws Exception { public void testNoEvictionBelowThreshold() throws Exception { IgniteEx ignite = startGrid(0); - DataRegionMetricsImpl metrics = - ignite.context().cache().context().database().dataRegion(null).metrics(); + DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); metrics.enableMetrics(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 6731287fae957..a611ffb377071 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import java.util.Arrays; @@ -26,6 +27,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -33,13 +35,11 @@ /** * Tests size-aware page eviction on in-memory (non-persistent) data regions. - * + *

* Verifies that a row larger than the configured {@code emptyPagesPoolSize} (in pages) is still written successfully * when page eviction is enabled, by evicting old entries to free enough space. Also verifies that a row * which fundamentally cannot fit into the region fails with OOM instead of hanging in an infinite eviction loop. - * - * Note: the atomic DHT batch path (putAll of many large rows overflowing a small region) is out of scope here — it is - * handled by a separate size-aware reserve in the batch store path and already fails on the original code. + * The batch path ({@code putAll} of large rows) and the update path (growing a row) are covered as well. */ public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstractTest { /** Off-heap region size (large enough to hold cache structural pages with the configured partition count). */ @@ -60,8 +60,12 @@ public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstra /** Small record size used to pre-fill the region with evictable data (for putAll tests). */ private static final int SMALL_RECORD_SIZE = 4096; - /** Small pre-fill entries count. */ - private static final int SMALL_ENTRIES = 8000; + /** + * Small pre-fill entries count. Chosen to fill the 128 MiB region close to capacity so that less than one large + * record ({@link #RECORD_SIZE}) remains available, forcing {@code ensureFreeSpaceForInsert} to actually evict + * prefilled entries rather than taking its fast path. + */ + private static final int SMALL_ENTRIES = 28_000; /** Large rows written via putAll. */ private static final int PUT_ALL_LARGE_ROWS = 3; @@ -73,10 +77,8 @@ public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstra .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE) - ) - .setPageSize(DFLT_PAGE_SIZE) - ); + .setEmptyPagesPoolSize(POOL_SIZE)) + .setPageSize(DFLT_PAGE_SIZE)); } /** {@inheritDoc} */ @@ -158,7 +160,9 @@ public void testRecordLargerThanRegionOom() throws Exception { cache.put(1, new byte[SIZE * 2]); } catch (Exception e) { - // OOM (possibly wrapped) because the row cannot fit into the region. + assertTrue("Expected IgniteOutOfMemoryException because the row cannot fit into the region, but got: " + e, + isOutOfMemory(e)); + rejected = true; } @@ -177,7 +181,8 @@ public void testPutAllLargeRows() throws Exception { IgniteCache cache = createCache(ignite); - // Pre-fill with small evictable entries so large rows below region capacity fit via the reserve path. + // Pre-fill the region to near capacity with small evictable entries so that less than one large row remains + // available. This forces ensureFreeSpaceForInsert to evict prefilled entries rather than taking its fast path. byte[] small = new byte[SMALL_RECORD_SIZE]; for (int i = 0; i < SMALL_ENTRIES; i++) @@ -198,7 +203,8 @@ public void testPutAllLargeRows() throws Exception { /** * Updating a record from a small to a large value (larger than the empty-pages pool) must succeed with page - * eviction enabled: the update goes through the same size-aware reserve as an insert. + * eviction enabled: the update goes through the same size-aware reserve as an insert. The region is pre-filled + * to near capacity so that the grown value cannot fit without evicting prefilled entries. * * @throws Exception If failed. */ @@ -208,6 +214,14 @@ public void testUpdateRowGrows() throws Exception { IgniteCache cache = createCache(ignite); + // Pre-fill the region to near capacity with small evictable entries so that the grown value below cannot + // fit without eviction. + byte[] small = new byte[SMALL_RECORD_SIZE]; + + for (int i = 0; i < SMALL_ENTRIES; i++) + cache.put(SMALL_ENTRIES + i, small); + + // Insert key 1 with a small value, then update it to a large value that requires size-aware eviction. cache.put(1, new byte[1024]); byte[] big = new byte[RECORD_SIZE]; @@ -222,4 +236,17 @@ public void testUpdateRowGrows() throws Exception { assertTrue("Updated value must equal the stored value", Arrays.equals(big, read)); } + + /** + * @param t Throwable. + * @return {@code True} if {@code t} or any of its causes is an out-of-memory. + */ + static boolean isOutOfMemory(Throwable t) { + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof IgniteOutOfMemoryException) + return true; + } + + return false; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index c6dd438367de1..974f6adfe6604 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import javax.cache.expiry.CreatedExpiryPolicy; @@ -60,10 +61,8 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE) - ) - .setPageSize(DFLT_PAGE_SIZE) - ); + .setEmptyPagesPoolSize(POOL_SIZE)) + .setPageSize(DFLT_PAGE_SIZE)); } /** {@inheritDoc} */ @@ -73,11 +72,12 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo /** * @param ignite Ignite node. + * @param cacheName Cache name. * @param ttl TTL in milliseconds ({@code 0} for no expiry). * @return Cache with a small partition count and, if {@code ttl > 0}, eager TTL expiry. */ - private IgniteCache createCache(IgniteEx ignite, long ttl) { - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + private IgniteCache createCache(IgniteEx ignite, String cacheName, long ttl) { + CacheConfiguration ccfg = new CacheConfiguration(cacheName) .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)); if (ttl > 0) { @@ -99,7 +99,7 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { IgniteEx ignite = startGrid(1); // Short-TTL entries keep the TTL worker actively freeing pages while eviction runs. - IgniteCache cache = createCache(ignite, TTL); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, TTL); Object val = new byte[RECORD_SIZE]; @@ -114,6 +114,10 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { /** * Space freed by TTL cleanup must be taken into account by size-aware eviction: a large record written after some * entries have expired must be accepted (no OOM) because their pages become available. + *

+ * The region is filled to near capacity with non-expiring small entries, then large short-TTL entries are added + * (evicting some small entries). After the TTL entries expire and their pages are freed, a fresh large record is + * written — it can only fit if size-aware eviction accounts for the space freed by TTL. * * @throws Exception If failed. */ @@ -121,20 +125,36 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { public void testTtlFreedSpaceAccountedForByEviction() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite, TTL); + // Non-expiring cache for prefill and the final large put. + IgniteCache plainCache = createCache(ignite, "plain-cache", 0); + + // Short-TTL cache for entries that will expire and free pages. + IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL); + + // Pre-fill the region to near capacity with small non-expiring entries. + byte[] small = new byte[4096]; + + for (int i = 0; i < 28_000; i++) + plainCache.put(i, small); - // Fill the region up to its capacity with short-TTL large records. + // Add large short-TTL entries that occupy significant space and will expire. Object val = new byte[RECORD_SIZE]; - for (int i = 0; i < 10; i++) - cache.put(i, val); + for (int i = 0; i < 4; i++) + ttlCache.put(i, val); + + // Verify the TTL entries are present before expiry. + assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); // Wait for the TTL worker to expire and free the short-TTL entries. Thread.sleep(TTL + 1500); - // A fresh large record must now be accepted (space freed by TTL counts as available for eviction). - cache.put(100, val); + // Verify the TTL entries have expired. + assertNull("TTL entry must be expired", ttlCache.get(0)); + + // A fresh large record must now be accepted: space freed by TTL counts as available for eviction. + plainCache.put(100, val); - assertNotNull(cache.get(100)); + assertNotNull(plainCache.get(100)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java index 320d48cdde1cd..347c724007c3f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_2_LRU}. - */ +/** Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java index f9b9e405905cf..8feebf712572a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_2_LRU}. - */ +/** Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java index cdbb5218c05b1..128549bfa67cb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_2_LRU}. - */ +/** ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java index d3dc5b99d0437..8b2fd795e8485 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_LRU}. - */ +/** Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java index b2bb394b65f0e..eac6af507b879 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_LRU}. - */ +/** Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java index ff5a209da1eb5..0709c75eafee3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.processors.cache.eviction.paged; import org.apache.ignite.configuration.DataPageEvictionMode; @@ -21,9 +22,7 @@ import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; -/** - * ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_LRU}. - */ +/** ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { From 22a93d4dd433aa2fc579ab0c8bcefe6ee7a287a1 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 7 Sep 2026 13:15:59 +0500 Subject: [PATCH 03/26] stabilize tests --- .../IgniteCacheDatabaseSharedManager.java | 6 +-- .../cache/persistence/RowStore.java | 3 +- ...eEvictionConcurrentWritesAbstractTest.java | 19 +++++---- .../paged/PageEvictionGuardOomTest.java | 3 +- .../PageEvictionSizeAwareAbstractTest.java | 7 +++- ...eEvictionWithExpiryPolicyAbstractTest.java | 40 ++++++++++++------- 6 files changed, 48 insertions(+), 30 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 152b716dbe527..3cc4c3761b32f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1175,18 +1175,18 @@ public WALPointer latestWalPointerReservedForPreloading() { /** * Checks that the given {@code region} has enough space for putting a new entry of {@code dataRowSize} bytes. - * + *

* For a non-persistent region with page eviction disabled, verifies that the region reserves enough pages to * support a free list {@link AbstractFreeList}. For example, removing a row from underlying store may require * allocating a new data page in order to move a tracked page from one bucket to another one which does not have * a free space for a new stripe. See {@link AbstractFreeList#removeDataRowByLink}. Therefore, inserting a new * entry should be prevented in case of some threshold is exceeded. - * + *

* For a non-persistent region with page eviction enabled, additionally performs size-aware eviction: when the * row does not fit into the currently available page space, data pages are evicted until either enough space is * freed or it becomes clear that the goal is unreachable (in which case an * {@link IgniteOutOfMemoryException} is thrown). - * + *

* The size-aware reserve is required because page eviction by itself only keeps a steady-state pool of empty pages * ({@link DataRegionConfiguration#getEmptyPagesPoolSize()}) and does not guarantee enough space for a single row * larger than this pool. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 3a09efaae10e1..0c7eb18a09076 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -133,8 +133,7 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. */ - public void addRows(Collection rows, - IoStatisticsHolder statHolder) throws IgniteCheckedException { + public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { // Size-aware reserve for each row in the batch. Eviction performed here runs without entry locks // (see AbstractFreeList#insertDataRows), so this is safe. Reserving only the largest row is insufficient: diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index 3b2f58d836fad..ec1fc7a3a9b3f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -44,13 +44,13 @@ */ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommonAbstractTest { /** Off-heap region size. */ - private static final int SIZE = 128 * 1024 * 1024; + private static final int SIZE = 256 * 1024 * 1024; /** Partition count (kept low so that index-tree structures do not exhaust the region). */ private static final int PARTITIONS = 32; - /** Large record size (much larger than the empty-pages pool). */ - private static final int LARGE_RECORD_SIZE = 16 * 1024 * 1024; + /** Large record size (larger than the empty-pages pool so that each write is size-aware). */ + private static final int LARGE_RECORD_SIZE = 2 * 1024 * 1024; /** Small record size used to pre-fill the region with evictable data. */ private static final int SMALL_RECORD_SIZE = 4096; @@ -58,14 +58,17 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo /** Empty pages pool size. */ private static final int POOL_SIZE = 100; - /** Number of small pre-fill entries. */ - private static final int SMALL_ENTRIES = 30_000; + /** Number of small pre-fill entries, leaving a buffer that is exceeded by the total of the large writes, so that + * the last of them can only be stored by freeing pages via size-aware eviction. The large records are small + * enough that concurrent size-aware eviction reliably frees the required pages (no spurious guard OOM). */ + private static final int SMALL_ENTRIES = 48_000; /** Number of writer threads. */ - private static final int THREADS = 4; + private static final int THREADS = 2; - /** Large rows inserted per thread (moderate total, kept within region capacity after eviction of small rows). */ - private static final int LARGE_ROWS_PER_THREAD = 1; + /** Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the + * last large writes overflow the region and require size-aware eviction to free small entry pages. */ + private static final int LARGE_ROWS_PER_THREAD = 20; /** Global deadline for the whole test (protects against a deadlock/busy-spin hang). */ private static final long DEADLINE = TimeUnit.MINUTES.toMillis(3); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 51b36928a22ea..979041f2f891e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -83,8 +83,7 @@ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { .setMaxSize(SIZE) .setEmptyPagesPoolSize(POOL_SIZE) .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)) - .setPageSize(DFLT_PAGE_SIZE) - ); + .setPageSize(DFLT_PAGE_SIZE)); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index a611ffb377071..e5346890545fb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -145,6 +146,8 @@ public void testLargeObjectReadBack() throws Exception { /** * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. + * Uses a transactional cache so that the size-aware OOM propagates directly (in an atomic cache it is wrapped in + * a {@code CachePartialUpdateException} and would not be detectable as the specific OOM). * * @throws Exception If failed. */ @@ -152,7 +155,9 @@ public void testLargeObjectReadBack() throws Exception { public void testRecordLargerThanRegionOom() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite); + IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); boolean rejected = false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 974f6adfe6604..7be31d93ea4ba 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -52,7 +53,7 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo private static final int POOL_SIZE = 100; /** Short TTL applied to some entries. */ - private static final long TTL = 1500; + private static final long TTL = 8000; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -112,12 +113,13 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { } /** - * Space freed by TTL cleanup must be taken into account by size-aware eviction: a large record written after some + * Space freed by TTL cleanup must be taken into account by size-aware eviction: large records written after some * entries have expired must be accepted (no OOM) because their pages become available. *

- * The region is filled to near capacity with non-expiring small entries, then large short-TTL entries are added - * (evicting some small entries). After the TTL entries expire and their pages are freed, a fresh large record is - * written — it can only fit if size-aware eviction accounts for the space freed by TTL. + * The region is pre-filled with non-expiring small entries and large short-TTL entries that later expire and free + * their pages. After expiry, fresh large records are written — totalling more than the space freed by TTL, so that + * the pre-fill plus the fresh records exceed the region size. The writes can only succeed because size-aware + * eviction accounts for the TTL-freed pages (as available) and frees further pages for the rest. * * @throws Exception If failed. */ @@ -131,30 +133,40 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { // Short-TTL cache for entries that will expire and free pages. IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL); - // Pre-fill the region to near capacity with small non-expiring entries. + // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries to be + // written without evicting them (so they are guaranteed to be present until they expire). byte[] small = new byte[4096]; - for (int i = 0; i < 28_000; i++) + for (int i = 0; i < 22_000; i++) plainCache.put(i, small); - // Add large short-TTL entries that occupy significant space and will expire. + // Add large short-TTL entries that occupy significant space and will expire. They fit in the remaining free + // space, and being the freshest entries they are not evicted while they are being stored. Object val = new byte[RECORD_SIZE]; - for (int i = 0; i < 4; i++) + for (int i = 0; i < 2; i++) ttlCache.put(i, val); // Verify the TTL entries are present before expiry. assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); // Wait for the TTL worker to expire and free the short-TTL entries. - Thread.sleep(TTL + 1500); + U.sleep(TTL + 1500); // Verify the TTL entries have expired. assertNull("TTL entry must be expired", ttlCache.get(0)); - // A fresh large record must now be accepted: space freed by TTL counts as available for eviction. - plainCache.put(100, val); - - assertNotNull(plainCache.get(100)); + // Now write large records to the non-expiring cache, totalling more than the space freed by TTL (the prefill + // plus the fresh large records exceed the region size). The put can only succeed because size-aware eviction + // accounts for the pages freed by TTL (as available) and frees further pages for the rest; this verifies the + // synergy between TTL cleanup and size-aware eviction — a row that only fits once expired entries are removed + // is written without OOM. + for (int i = 0; i < 6; i++) + plainCache.put(100 + i, val); + + // The most recently written large record is still present: it was accepted without OOM thanks to the space + // freed by TTL and the additional pages freed by size-aware eviction. (Earlier large records may already have + // been evicted to make room for the subsequent ones, so only the freshest is asserted.) + assertNotNull("Fresh large record must be present after TTL-assisted eviction", plainCache.get(105)); } } From 4d6ca55ce5d3599eab2f7e19c38cbad586161f23 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 7 Sep 2026 15:42:21 +0500 Subject: [PATCH 04/26] fix TOCTOU race --- .../freelist/AbstractFreeList.java | 78 ++++++++++++++++++- ...eEvictionConcurrentWritesAbstractTest.java | 19 ++++- .../PageEvictionSizeAwareAbstractTest.java | 7 +- ...eEvictionWithExpiryPolicyAbstractTest.java | 40 ++++++++-- 4 files changed, 132 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 62452095631cf..3ab9df294e485 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; +import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; @@ -98,6 +99,25 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; + /** + * Upper bound on consecutive eviction attempts that free no page before falling back to raw page allocation + * (which throws an {@link org.apache.ignite.internal.mem.IgniteOutOfMemoryException} when the region is full). + * Guards against an unbounded busy-spin when eviction cannot free anything (e.g. all entries are locked). + */ + private static final int MAX_CONSECUTIVE_DEMAND_EVICTIONS = 100; + + /** + * Region capacity in pages ({@code maxSize / systemPageSize}) scaled by the eviction threshold. While the number + * of loaded pages is below this bound the region is comfortably under-utilized, so a need for a fresh page is + * satisfied by growing the region (allocating a new page) instead of evicting a live entry. Once the region is at + * or above the eviction-threshold point it is considered effectively full for a single "new page" demand, and the + * free list falls back to eviction to recycle existing pages (closing the TOCTOU gap between a size-aware reserve + * in RowStore.addRow and the actual page consumption). A raw compare against the full capacity is insufficient + * because a small fraction of pages is consumed by non-data structures, so the region cannot grow all the way to + * {@code maxSize / systemPageSize}. + */ + private final long maxGrowPages; + /** Page list cache limit. */ private final AtomicLong pageListCacheLimit; @@ -465,6 +485,9 @@ public AbstractFreeList( this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); + maxGrowPages = (long)(dataRegion.config().getMaxSize() / (double)pageMem.systemPageSize() + * dataRegion.config().getEvictionThreshold()); + assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize; assert U.isPow2(BUCKETS); assert BUCKETS <= pageSize : pageSize; @@ -706,7 +729,18 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePage(row.size() - written, row, statHolder); if (pageId == 0L) { - pageId = allocateDataPage(row.partition()); + // The steady-state pool of empty pages is exhausted. If the region is comfortably below the eviction + // threshold, satisfy the demand by growing the region (allocating a fresh page) rather than evicting a live + // entry: eviction would otherwise thrash live entries merely because the steady-state pool is momentarily + // drained (e.g. while initially filling the region). Once the region reaches the eviction-threshold point, + // it is effectively full for a single fresh page, so we fall back to (non-blocking) eviction and retake, + // which closes the TOCTOU gap between a size-aware reserve performed in RowStore.addRow and the actual + // consumption of pages here: pages freed by a concurrent eviction can then be reused instead of a spurious + // raw OOM. + if (pageMem.loadedPages() < maxGrowPages) + pageId = allocateDataPage(row.partition()); + else + pageId = evictAndTakePage(row, row.size() - written, statHolder); initIo = row.ioVersions().latest(); } @@ -718,6 +752,48 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t return written; } + /** + * Attempts to free a page through eviction and retake a page of the given size from the free list, repeating up + * to {@link #MAX_CONSECUTIVE_DEMAND_EVICTIONS} times. Falls back to a raw page allocation (which throws an + * {@link org.apache.ignite.internal.mem.IgniteOutOfMemoryException} when the region is over capacity) once eviction + * stops freeing pages. + * + * @param row Row to write. + * @param size Required free space on the page. + * @param statHolder Statistics holder to track IO operations. + * @return Page ID. + * @throws IgniteCheckedException If failed. + */ + private long evictAndTakePage(T row, int size, IoStatisticsHolder statHolder) throws IgniteCheckedException { + for (int i = 0; i < MAX_CONSECUTIVE_DEMAND_EVICTIONS; i++) { + evictDataPageSafe(); + + memMetrics.updateEvictionRate(); + + long pageId = takePage(size, row, statHolder); + + if (pageId != 0L) + return pageId; + } + + return allocateDataPage(row.partition()); + } + + /** + * Evicts a single data page, acquiring entry locks non-blockingly when the tracker supports it. The size-aware + * single-row insert path can invoke this while the current thread already holds the entry lock of the row being + * inserted, so a blocking eviction of another entry would risk a lock-ordering deadlock. Mirrors the non-blocking + * eviction used by {@code IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert}. + * + * @throws IgniteCheckedException If failed. + */ + private void evictDataPageSafe() throws IgniteCheckedException { + if (evictionTracker instanceof PageAbstractEvictionTracker) + ((PageAbstractEvictionTracker)evictionTracker).evictDataPageNonBlocking(); + else + evictionTracker.evictDataPage(); + } + /** * Take page from free list. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index ec1fc7a3a9b3f..b25114b08dd2b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -20,6 +20,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; @@ -118,6 +119,8 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { AtomicLong errors = new AtomicLong(); + AtomicReference firstErr = new AtomicReference<>(); + CountDownLatch startLatch = new CountDownLatch(1); long deadline = System.currentTimeMillis() + DEADLINE; @@ -137,6 +140,8 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { catch (Throwable e) { errors.incrementAndGet(); + firstErr.compareAndSet(null, e); + log.error("Unexpected error in writer thread", e); } }, "paged-writer-" + i); @@ -146,14 +151,26 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { startLatch.countDown(); + long start = System.currentTimeMillis(); + for (Thread t : threads) t.join(Math.max(1, deadline - System.currentTimeMillis())); // The core assertion of this deadlock test: every writer must have completed (no thread is stuck waiting on // an entry lock held by size-aware eviction running under another entry lock). + for (Thread t : threads) { + if (t.isAlive()) { + log.error("Writer thread " + t.getName() + " is still alive after " + + (System.currentTimeMillis() - start) + "ms, state=" + t.getState()); + + for (StackTraceElement frame : t.getStackTrace()) + log.error(" at " + frame); + } + } + for (Thread t : threads) assertFalse("Writer thread " + t.getName() + " did not finish (possible deadlock)", t.isAlive()); - assertEquals("Writer threads reported errors", 0, errors.get()); + assertEquals("Writer threads reported errors, reason: " + firstErr.get(), 0, errors.get()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index e5346890545fb..8acd1124d94a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -49,8 +49,11 @@ public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstra /** Partition count (kept low so that index-tree structures do not exhaust the region). */ private static final int PARTITIONS = 32; - /** Record size: chosen to be much larger than {@code emptyPagesPoolSize} pages. */ - private static final int RECORD_SIZE = 4 * 1024 * 1024; + /** Record size: chosen so that a single row requires more pages than are left free when the region is kept at the + * eviction threshold ({@code (1 - threshold) * totalPages}). This guarantees a large put cannot take the fast + * path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve + * ({@code ensureFreeSpaceForEviction}), which is the scenario these tests are meant to cover. */ + private static final int RECORD_SIZE = 32 * 1024 * 1024; /** Empty pages pool size. */ private static final int POOL_SIZE = 100; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 7be31d93ea4ba..3e5b2e933fc70 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; /** @@ -46,12 +47,27 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo /** Partition count (kept low so that index-tree structures do not exhaust the region). */ private static final int PARTITIONS = 32; - /** Large record size (much larger than the empty-pages pool). */ - private static final int RECORD_SIZE = 8 * 1024 * 1024; + /** Large record size (much larger than the empty-pages pool, and larger than the space left free when the region + * is held at the eviction threshold {@code (1 - threshold) * totalPages}) so that a large put cannot take the + * fast path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve. */ + private static final int RECORD_SIZE = 32 * 1024 * 1024; /** Empty pages pool size. */ private static final int POOL_SIZE = 100; + /** Size of a single non-expiring pre-fill record (well below a data-page payload so each record surely occupies + * exactly one data page). */ + private static final int SMALL_RECORD_SIZE = 1024; + + /** Number of non-expiring pre-fill records. Deliberately conservative: the pre-fill keeps a comfortable margin to + * the region capacity so that the large short-TTL records are guaranteed not to be evicted before they expire. */ + private static final int SMALL_ENTRIES = 6_000; + + /** Fresh large records written after TTL expiry. Together with the pre-fill (6000 pages) plus the index structures + * they exceed the region capacity (~32768 pages), forcing size-aware eviction that accounts for the TTL-freed + * space. */ + private static final int FRESH_RECORDS = 12; + /** Short TTL applied to some entries. */ private static final long TTL = 8000; @@ -135,11 +151,14 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries to be // written without evicting them (so they are guaranteed to be present until they expire). - byte[] small = new byte[4096]; + byte[] small = new byte[SMALL_RECORD_SIZE]; - for (int i = 0; i < 22_000; i++) + for (int i = 0; i < SMALL_ENTRIES; i++) plainCache.put(i, small); + info("Pre-fill done [entries=" + SMALL_ENTRIES + ", plainSize=" + plainCache.size() + + ", loadedPages=" + ignite.dataRegionMetrics(DFLT_DATA_REG_DEFAULT_NAME).getTotalAllocatedPages() + ']'); + // Add large short-TTL entries that occupy significant space and will expire. They fit in the remaining free // space, and being the freshest entries they are not evicted while they are being stored. Object val = new byte[RECORD_SIZE]; @@ -150,8 +169,12 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { // Verify the TTL entries are present before expiry. assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); - // Wait for the TTL worker to expire and free the short-TTL entries. - U.sleep(TTL + 1500); + // Wait for the TTL worker to expire and free the short-TTL entries. Polled instead of a fixed sleep so that a + // slow CI machine does not proceed before the entries have actually expired. + long expiryDeadline = System.currentTimeMillis() + 30_000; + + while (ttlCache.get(0) != null && System.currentTimeMillis() < expiryDeadline) + U.sleep(200); // Verify the TTL entries have expired. assertNull("TTL entry must be expired", ttlCache.get(0)); @@ -161,12 +184,13 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { // accounts for the pages freed by TTL (as available) and frees further pages for the rest; this verifies the // synergy between TTL cleanup and size-aware eviction — a row that only fits once expired entries are removed // is written without OOM. - for (int i = 0; i < 6; i++) + for (int i = 0; i < FRESH_RECORDS; i++) plainCache.put(100 + i, val); // The most recently written large record is still present: it was accepted without OOM thanks to the space // freed by TTL and the additional pages freed by size-aware eviction. (Earlier large records may already have // been evicted to make room for the subsequent ones, so only the freshest is asserted.) - assertNotNull("Fresh large record must be present after TTL-assisted eviction", plainCache.get(105)); + assertNotNull("Fresh large record must be present after TTL-assisted eviction", + plainCache.get(100 + FRESH_RECORDS - 1)); } } From 430c51a75447f7484192ea1c176089b8daf024ef Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 8 Sep 2026 01:18:57 +0500 Subject: [PATCH 05/26] fix PDS tests --- .../freelist/AbstractFreeList.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 3ab9df294e485..3cf357b11e23e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -729,18 +729,18 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePage(row.size() - written, row, statHolder); if (pageId == 0L) { - // The steady-state pool of empty pages is exhausted. If the region is comfortably below the eviction - // threshold, satisfy the demand by growing the region (allocating a fresh page) rather than evicting a live - // entry: eviction would otherwise thrash live entries merely because the steady-state pool is momentarily - // drained (e.g. while initially filling the region). Once the region reaches the eviction-threshold point, - // it is effectively full for a single fresh page, so we fall back to (non-blocking) eviction and retake, - // which closes the TOCTOU gap between a size-aware reserve performed in RowStore.addRow and the actual - // consumption of pages here: pages freed by a concurrent eviction can then be reused instead of a spurious - // raw OOM. - if (pageMem.loadedPages() < maxGrowPages) - pageId = allocateDataPage(row.partition()); - else + // The steady-state pool of empty pages is exhausted. The demand-eviction fallback only applies to + // regions with an active page eviction tracker (in-memory regions with eviction enabled). Once the region + // reaches the eviction-threshold point, it is effectively full for a single fresh page, so we fall back to + // (non-blocking) eviction and retake, which closes the TOCTOU gap between a size-aware reserve performed in + // RowStore.addRow and the actual consumption of pages here: pages freed by a concurrent eviction can then + // be reused instead of a spurious raw OOM. Below the threshold (or in the absence of a real eviction + // tracker - persistent regions and regions with eviction disabled use a NoOp tracker) we keep the original + // behaviour and simply grow the region by allocating a fresh page. + if (evictionTracker instanceof PageAbstractEvictionTracker && pageMem.loadedPages() >= maxGrowPages) pageId = evictAndTakePage(row, row.size() - written, statHolder); + else + pageId = allocateDataPage(row.partition()); initIo = row.ioVersions().latest(); } From 622b749230bec5839d504129de56db5052eb7f3f Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 8 Sep 2026 08:52:39 +0500 Subject: [PATCH 06/26] fix broken tests --- .../IgniteCacheDatabaseSharedManager.java | 63 +++++++++----- .../freelist/AbstractFreeList.java | 84 ++----------------- 2 files changed, 51 insertions(+), 96 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 3cc4c3761b32f..3325db7d6b77e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1275,7 +1275,8 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati // Maximum payload bytes that a single data page can hold for a fragmented row. long pagePayload = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - // Pages required to place the row, computed from the actual data-page payload capacity. + // Pages the row will actually occupy once written, and which the free list must be able to hand out on + // demand during the fragmented write. long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; // If the row fits into the configured steady-state empty-pages pool, normal threshold eviction is enough. @@ -1286,25 +1287,45 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati if (requiredPages > totalPages) throw outOfMemory(regCfg); - long availablePages = (totalPages - pageMem.loadedPages()) + freeList.emptyDataPages(); - - // Fast path: enough pages are already available, no eviction is needed. - if (availablePages >= requiredPages) + // The reserve must guarantee that the free list actually holds `requiredPages` REAL empty pages, not merely + // that the region "as a whole" has apparent headroom. The apparent headroom (totalPages - loadedPages) is a + // shared, non-exclusive resource: two concurrent inserts may both count on it and then both run out of pages + // while the fragmented write is in progress (TOCTOU // raw OOM), because a fresh allocation cannot grow the + // region beyond its capacity. Real empty pages already sitting in the free list are the only resource + // writeSinglePage can reliably consume once the region is effectively full, so the loop below accumulates them + // until it has enough. + long emptyPages = freeList.emptyDataPages(); + + long headroom = totalPages - pageMem.loadedPages(); + + // The region is "under pressure" once its loaded pages reach the eviction threshold: beyond that point a fresh + // allocation can no longer be trusted to grow the region, because concurrent writers may exhaust the shared + // headroom before this thread's fragmented write runs (TOCTOU raw OOM). Below the threshold the region has + // genuine slack, so a row that fits into the combined spare space is satisfied by growing the region rather + // than entering the eviction loop, which would otherwise destroy evictable entries (e.g. live short-TTL + // entries that must survive) merely to accumulate empty pages that the slack could have absorbed. + long pagesThreshold = (long)(totalPages * regCfg.getEvictionThreshold()); + + boolean underPressure = pageMem.loadedPages() >= pagesThreshold; + + // Fast path: the row can be satisfied without eviction when (a) the free list already holds enough real empty + // pages, or (b) the region is not under pressure and has enough spare space to grow into. + if (emptyPages >= requiredPages || (!underPressure && emptyPages + headroom >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); - // Evict data pages until enough free space is available. Progress is measured against the overall available - // space, so pages freed concurrently (e.g. by TTL cleanup) also count as progress. The loop is bounded to - // avoid an infinite busy-spin when there is nothing more to evict. Eviction here runs while the current - // thread may already hold entry locks (single-row insertion), so entries whose locks are contended are + // Evict data pages until the free list holds enough real empty pages. Progress is measured against the + // number of empty pages, so pages freed concurrently (e.g. by TTL cleanup) also count as progress. The loop + // is bounded to avoid an infinite busy-spin when there is nothing more to evict. Eviction here runs while the + // current thread may already hold entry locks (single-row insertion), so entries whose locks are contended are // skipped (non-blocking) rather than blocked upon, to avoid a lock-ordering deadlock. final int maxAttemptsWithoutProgress = 300; - long bestAvailable = availablePages; + long bestEmptyPages = emptyPages; int attemptsWithoutProgress = 0; - while (bestAvailable < requiredPages) { + while (bestEmptyPages < requiredPages) { if (region.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); @@ -1314,22 +1335,26 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati region.metrics().updateEvictionRate(); - long curAvailable = (totalPages - pageMem.loadedPages()) + freeList.emptyDataPages(); + long curEmptyPages = freeList.emptyDataPages(); - // Progress is measured against the best available space observed so far. Any iteration that does not - // establish a new best (including drops caused by concurrent inserts consuming pages) counts toward the - // no-progress guard, so the loop is bounded: if eviction cannot outpace concurrent consumption within - // a fixed number of attempts, an OOM is thrown rather than busy-spinning indefinitely. - if (curAvailable > bestAvailable) { - bestAvailable = curAvailable; + // Progress is measured against the best number of empty pages observed so far. Any iteration that does + // not establish a new best (including drops caused by concurrent inserts consuming pages) counts toward + // the no-progress guard, so the loop is bounded: if eviction cannot outpace concurrent consumption within + // a fixed number of attempts, we stop rather than busy-spinning indefinitely. + if (curEmptyPages > bestEmptyPages) { + bestEmptyPages = curEmptyPages; attemptsWithoutProgress = 0; } else attemptsWithoutProgress++; - if (attemptsWithoutProgress >= maxAttemptsWithoutProgress) + if (attemptsWithoutProgress >= maxAttemptsWithoutProgress) { + // No progress after many eviction attempts. The region is under pressure (fast path was already + // insufficient: real empty pages are below requiredPages), and eviction can neither outpace concurrent + // consumption nor free enough space. Throw OOM rather than busy-spinning indefinitely. throw outOfMemory(regCfg); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 3cf357b11e23e..5dd972c5689ab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -38,7 +38,6 @@ import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; -import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; @@ -99,25 +98,6 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; - /** - * Upper bound on consecutive eviction attempts that free no page before falling back to raw page allocation - * (which throws an {@link org.apache.ignite.internal.mem.IgniteOutOfMemoryException} when the region is full). - * Guards against an unbounded busy-spin when eviction cannot free anything (e.g. all entries are locked). - */ - private static final int MAX_CONSECUTIVE_DEMAND_EVICTIONS = 100; - - /** - * Region capacity in pages ({@code maxSize / systemPageSize}) scaled by the eviction threshold. While the number - * of loaded pages is below this bound the region is comfortably under-utilized, so a need for a fresh page is - * satisfied by growing the region (allocating a new page) instead of evicting a live entry. Once the region is at - * or above the eviction-threshold point it is considered effectively full for a single "new page" demand, and the - * free list falls back to eviction to recycle existing pages (closing the TOCTOU gap between a size-aware reserve - * in RowStore.addRow and the actual page consumption). A raw compare against the full capacity is insufficient - * because a small fraction of pages is consumed by non-data structures, so the region cannot grow all the way to - * {@code maxSize / systemPageSize}. - */ - private final long maxGrowPages; - /** Page list cache limit. */ private final AtomicLong pageListCacheLimit; @@ -485,9 +465,6 @@ public AbstractFreeList( this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); - maxGrowPages = (long)(dataRegion.config().getMaxSize() / (double)pageMem.systemPageSize() - * dataRegion.config().getEvictionThreshold()); - assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize; assert U.isPow2(BUCKETS); assert BUCKETS <= pageSize : pageSize; @@ -729,18 +706,13 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePage(row.size() - written, row, statHolder); if (pageId == 0L) { - // The steady-state pool of empty pages is exhausted. The demand-eviction fallback only applies to - // regions with an active page eviction tracker (in-memory regions with eviction enabled). Once the region - // reaches the eviction-threshold point, it is effectively full for a single fresh page, so we fall back to - // (non-blocking) eviction and retake, which closes the TOCTOU gap between a size-aware reserve performed in - // RowStore.addRow and the actual consumption of pages here: pages freed by a concurrent eviction can then - // be reused instead of a spurious raw OOM. Below the threshold (or in the absence of a real eviction - // tracker - persistent regions and regions with eviction disabled use a NoOp tracker) we keep the original - // behaviour and simply grow the region by allocating a fresh page. - if (evictionTracker instanceof PageAbstractEvictionTracker && pageMem.loadedPages() >= maxGrowPages) - pageId = evictAndTakePage(row, row.size() - written, statHolder); - else - pageId = allocateDataPage(row.partition()); + // The steady-state pool of empty pages is exhausted; allocate a fresh page. This path must NOT perform + // eviction (evictAndTakePage): it is reached while the row insert is already inside the cache data tree + // (BPlusTree.invoke on createRow), so evicting another entry here would recursively modify the same tree + // structure concurrently and corrupt it. Size-aware space management for eviction-enabled regions is + // handled separately and safely (before the tree operation) in + // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert and the batch insertDataRows loop. + pageId = allocateDataPage(row.partition()); initIo = row.ioVersions().latest(); } @@ -752,48 +724,6 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t return written; } - /** - * Attempts to free a page through eviction and retake a page of the given size from the free list, repeating up - * to {@link #MAX_CONSECUTIVE_DEMAND_EVICTIONS} times. Falls back to a raw page allocation (which throws an - * {@link org.apache.ignite.internal.mem.IgniteOutOfMemoryException} when the region is over capacity) once eviction - * stops freeing pages. - * - * @param row Row to write. - * @param size Required free space on the page. - * @param statHolder Statistics holder to track IO operations. - * @return Page ID. - * @throws IgniteCheckedException If failed. - */ - private long evictAndTakePage(T row, int size, IoStatisticsHolder statHolder) throws IgniteCheckedException { - for (int i = 0; i < MAX_CONSECUTIVE_DEMAND_EVICTIONS; i++) { - evictDataPageSafe(); - - memMetrics.updateEvictionRate(); - - long pageId = takePage(size, row, statHolder); - - if (pageId != 0L) - return pageId; - } - - return allocateDataPage(row.partition()); - } - - /** - * Evicts a single data page, acquiring entry locks non-blockingly when the tracker supports it. The size-aware - * single-row insert path can invoke this while the current thread already holds the entry lock of the row being - * inserted, so a blocking eviction of another entry would risk a lock-ordering deadlock. Mirrors the non-blocking - * eviction used by {@code IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert}. - * - * @throws IgniteCheckedException If failed. - */ - private void evictDataPageSafe() throws IgniteCheckedException { - if (evictionTracker instanceof PageAbstractEvictionTracker) - ((PageAbstractEvictionTracker)evictionTracker).evictDataPageNonBlocking(); - else - evictionTracker.evictDataPage(); - } - /** * Take page from free list. * From ad837725edf2dda71953179b300d5336ff65e9b0 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 8 Sep 2026 10:08:32 +0500 Subject: [PATCH 07/26] minor --- .../processors/cache/GridCacheEntryEx.java | 8 ++++++-- .../processors/cache/GridCacheMapEntry.java | 5 ++--- .../IgniteCacheDatabaseSharedManager.java | 14 +++++++++----- .../persistence/freelist/AbstractFreeList.java | 6 ------ 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index 0d07d70602c11..f6063a1708d6a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -214,8 +214,12 @@ public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryP * @return {@code True} if entry could be evicted. * @throws IgniteCheckedException In case of error. */ - public default boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, - boolean evictOffheap, boolean tryLock) throws IgniteCheckedException { + public default boolean evictInternal( + GridCacheVersion obsoleteVer, + @Nullable CacheEntryPredicate[] filter, + boolean evictOffheap, + boolean tryLock + ) throws IgniteCheckedException { return evictInternal(obsoleteVer, filter, evictOffheap); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 3f5abc02f9f22..99ecf9b496ee1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3699,9 +3699,8 @@ private CacheEntryImplEx wrapVersionedWithValue() { GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap, - boolean tryLock) - throws IgniteCheckedException { - + boolean tryLock + ) throws IgniteCheckedException { boolean marked = false; try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 3325db7d6b77e..f02efc6fb90be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1247,9 +1247,12 @@ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg } /** - * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the region has enough - * available pages to accommodate the row, or throws {@link IgniteOutOfMemoryException} if the goal is - * unreachable / no progress can be made. + * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the free list holds + * enough real empty pages to accommodate the row, or throws {@link IgniteOutOfMemoryException} if the goal is + * unreachable / no progress can be made. Progress is measured against the number of empty pages in the free list + * (the only resource a subsequent fragmented write can reliably consume once the region is effectively full); the + * region's spare capacity (headroom) is only trusted in the fast path while the region is below the eviction + * threshold. * * @param region Data region. * @param regCfg Data region configuration. @@ -1290,10 +1293,11 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati // The reserve must guarantee that the free list actually holds `requiredPages` REAL empty pages, not merely // that the region "as a whole" has apparent headroom. The apparent headroom (totalPages - loadedPages) is a // shared, non-exclusive resource: two concurrent inserts may both count on it and then both run out of pages - // while the fragmented write is in progress (TOCTOU // raw OOM), because a fresh allocation cannot grow the + // while the fragmented write is in progress (TOCTOU / raw OOM), because a fresh allocation cannot grow the // region beyond its capacity. Real empty pages already sitting in the free list are the only resource // writeSinglePage can reliably consume once the region is effectively full, so the loop below accumulates them - // until it has enough. + // until it has enough. (The headroom is nonetheless trusted in the fast path below, but only while the region + // is below the eviction threshold, i.e. where enough slack is guaranteed not to be exhausted by contention.) long emptyPages = freeList.emptyDataPages(); long headroom = totalPages - pageMem.loadedPages(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 5dd972c5689ab..62452095631cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -706,12 +706,6 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePage(row.size() - written, row, statHolder); if (pageId == 0L) { - // The steady-state pool of empty pages is exhausted; allocate a fresh page. This path must NOT perform - // eviction (evictAndTakePage): it is reached while the row insert is already inside the cache data tree - // (BPlusTree.invoke on createRow), so evicting another entry here would recursively modify the same tree - // structure concurrently and corrupt it. Size-aware space management for eviction-enabled regions is - // handled separately and safely (before the tree operation) in - // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert and the batch insertDataRows loop. pageId = allocateDataPage(row.partition()); initIo = row.ioVersions().latest(); From 53a2c3578e23ec6b9be74914cff4b963c212c5d9 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 8 Sep 2026 10:48:38 +0500 Subject: [PATCH 08/26] add benchmark --- .../jmh/cache/JmhPageEvictionBenchmark.java | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java new file mode 100644 index 0000000000000..854ff8025301c --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.cache; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Measures the impact of size-aware page eviction on an in-memory (non-persistent) data region. + *

+ * Two benchmark methods: + *

    + *
  • {@link #putSmall()} - puts of small values (well below the empty-pages pool, so the size-aware reserve + * in {@code RowStore.addRow} hits its fast path). This is the hot path whose per-operation cost the patch + * adds on every put, and is the primary A/B metric for detecting a performance regression between the + * unpatched baseline and this branch.
    • + *
  • {@link #putLarge()} - puts of large values (larger than the empty-pages pool) against a region that has + * been pre-filled to near capacity, so that each large put must actually run the size-aware eviction loop. + * This exercises the new eviction behavior; on an unpatched build such a put fails with an out-of-memory + * error, so this benchmark only runs meaningfully on the patched build.
    • + *
+ */ +@State(Scope.Benchmark) +@Fork(1) +@Threads(4) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 5, time = 10) +public class JmhPageEvictionBenchmark { + /** Default cache name. */ + private static final String CACHE_NAME = "default"; + + /** Empty pages pool size (kept low so that the LARGE scenario reliably exceeds it). */ + private static final int POOL_SIZE = 100; + + /** Small value size (bytes): a single data page, far below the empty-pages pool. */ + private static final int SMALL_VALUE_SIZE = 1024; + + /** Large value size (bytes): larger than the empty-pages pool in page terms. */ + private static final int LARGE_VALUE_SIZE = 2 * 1024 * 1024; + + /** Number of pre-fill small entries for the LARGE scenario (fills the region close to capacity). */ + private static final int PRE_FILL_ENTRIES = 48_000; + + /** Benchmark scenario: selects the value size and the pre-fill strategy. */ + @Param({"SMALL", "LARGE"}) + private String scenario; + + /** Ignite cache. */ + private IgniteCache cache; + + /** Pre-allocated small value (reused to avoid allocation noise in the hot path). */ + private final byte[] smallVal = new byte[SMALL_VALUE_SIZE]; + + /** Pre-allocated large value (reused to avoid allocation noise in the hot path). */ + private final byte[] largeVal = new byte[LARGE_VALUE_SIZE]; + + /** Monotonic key source for overwrite-style puts. */ + private final AtomicInteger keyGen = new AtomicInteger(); + + /** Checked once whether the region currently evicts (to warn about the LARGE scenario on an unpatched build). */ + private volatile String evictionMode; + + /** {@inheritDoc} */ + public JmhPageEvictionBenchmark() { + keyGen.set(0); + } + + /** Put of a small value (hot path, size-aware reserve takes its fast path). */ + @Benchmark + public void putSmall() { + int key = keyGen.incrementAndGet(); + + cache.put(key, smallVal); + } + + /** Put of a large value against a nearly-full region (runs the size-aware eviction loop). */ + @Benchmark + public void putLarge() { + int key = keyGen.incrementAndGet(); + + cache.put(key, largeVal); + } + + /** Starts Ignite with an in-memory, eviction-enabled data region and pre-fills it for the LARGE scenario. */ + @Setup(Level.Trial) + public void setup() { + DataPageEvictionMode evictionMode = DataPageEvictionMode.RANDOM_LRU; + + this.evictionMode = evictionMode.name(); + + long regionSize = 256 * 1024L * 1024L; + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(false) + .setMaxSize(regionSize) + .setEmptyPagesPoolSize(POOL_SIZE) + .setPageEvictionMode(evictionMode)); + + IgniteConfiguration cfg = new IgniteConfiguration() + .setIgniteInstanceName("test") + .setLocalHost("127.0.0.1") + .setDataStorageConfiguration(dsCfg); + + Ignite ignite = Ignition.start(cfg); + + cache = ignite.getOrCreateCache(new CacheConfiguration(CACHE_NAME).setBackups(0)); + + // Pre-fill the region with small entries for the LARGE scenario so that a large put cannot fit into the + // remaining headroom and must actually evict. + if ("LARGE".equalsIgnoreCase(scenario)) { + try (IgniteDataStreamer ldr = ignite.dataStreamer(CACHE_NAME)) { + ldr.perNodeBufferSize(1024); + + for (int i = 0; i < PRE_FILL_ENTRIES; i++) + ldr.addData(i, smallVal); + } + } + } + + /** @return Test data. */ + @Override public String toString() { + return "JmhPageEvictionBenchmark[scenario=" + scenario + ", evictionMode=" + evictionMode + ']'; + } + + /** Stops all Ignite instances started by this benchmark. */ + @TearDown + public void tearDown() { + Ignition.stopAll(true); + } + + /** + * Runs this benchmark over both {@code SMALL} and {@code LARGE} scenarios (configured by {@code @Param}). + * + * @param args Ignored. + * @throws Exception If failed. + */ + public static void main(String[] args) throws Exception { + JmhIdeBenchmarkRunner.create() + .benchmarks(JmhPageEvictionBenchmark.class.getSimpleName()) + .run(); + } +} From fb5b61ef1af0bf23d923b63e2dbd8e57b43eab6a Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 8 Sep 2026 22:40:56 +0500 Subject: [PATCH 09/26] self-review --- .../jmh/cache/JmhPageEvictionBenchmark.java | 71 ++++++--- .../processors/cache/GridCacheEntryEx.java | 14 +- .../processors/cache/GridCacheMapEntry.java | 15 +- .../IgniteCacheDatabaseSharedManager.java | 135 +++++++++++------- .../cache/persistence/RowStore.java | 14 +- .../evict/PageAbstractEvictionTracker.java | 5 +- .../freelist/AbstractFreeList.java | 31 ++++ .../paged/PageEvictionGuardOomTest.java | 5 +- .../PageEvictionSizeAwareAbstractTest.java | 4 + ...eEvictionWithExpiryPolicyAbstractTest.java | 11 +- 10 files changed, 204 insertions(+), 101 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java index 854ff8025301c..29145f07d4ea5 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java @@ -50,9 +50,9 @@ * Two benchmark methods: *
    *
  • {@link #putSmall()} - puts of small values (well below the empty-pages pool, so the size-aware reserve - * in {@code RowStore.addRow} hits its fast path). This is the hot path whose per-operation cost the patch - * adds on every put, and is the primary A/B metric for detecting a performance regression between the - * unpatched baseline and this branch.
    • + * in {@code RowStore.addRow} hits its fast path) within a bounded key range that keeps the region below the + * eviction threshold. This is the hot path whose per-operation cost the patch adds on every put, and is the + * primary A/B metric for detecting a performance regression between the unpatched baseline and this branch. *
  • {@link #putLarge()} - puts of large values (larger than the empty-pages pool) against a region that has * been pre-filled to near capacity, so that each large put must actually run the size-aware eviction loop. * This exercises the new eviction behavior; on an unpatched build such a put fails with an out-of-memory @@ -79,8 +79,24 @@ public class JmhPageEvictionBenchmark { /** Large value size (bytes): larger than the empty-pages pool in page terms. */ private static final int LARGE_VALUE_SIZE = 2 * 1024 * 1024; - /** Number of pre-fill small entries for the LARGE scenario (fills the region close to capacity). */ - private static final int PRE_FILL_ENTRIES = 48_000; + /** + * Number of pre-fill small entries for the LARGE scenario. Chosen so that the total written data + * (400k x 1 KiB) far exceeds the region capacity: threshold eviction then pins the region at the eviction + * threshold (default ~90% of {@code maxSize}), leaving the free list with only its empty-pages pool. At that + * point a {@link #LARGE_VALUE_SIZE} put cannot take the fast path and must actually run the size-aware eviction + * loop. (A modest pre-fill such as 48k x 1 KiB would leave the region only ~19% full and let every large put + * fit into the headroom via the fast path, so it would never exercise the code under measurement.) + */ + private static final int PRE_FILL_ENTRIES = 400_000; + + /** + * Bounded key range for {@link #putSmall()}. Each {@link #SMALL_VALUE_SIZE} value occupies one data page, so a + * working set of this many resident keys (~32k x 4 KiB ~ 128 MiB) stays comfortably below the eviction + * threshold (~90% of the 256 MiB region). Overwriting within this bounded range (instead of append-style fresh + * keys) keeps the region from filling up and drifting into steady-state threshold eviction during measurement, + * so the run isolates the per-put cost of the size-aware-reserve fast path. + */ + private static final int SMALL_KEY_RANGE = 32_000; /** Benchmark scenario: selects the value size and the pre-fill strategy. */ @Param({"SMALL", "LARGE"}) @@ -95,27 +111,35 @@ public class JmhPageEvictionBenchmark { /** Pre-allocated large value (reused to avoid allocation noise in the hot path). */ private final byte[] largeVal = new byte[LARGE_VALUE_SIZE]; - /** Monotonic key source for overwrite-style puts. */ + /** Monotonic key source: bounded (mod {@link #SMALL_KEY_RANGE}) for {@link #putSmall()} to keep the region below + * the eviction threshold, and unbounded (append-style) for {@link #putLarge()} to avoid overwriting entries. */ private final AtomicInteger keyGen = new AtomicInteger(); - /** Checked once whether the region currently evicts (to warn about the LARGE scenario on an unpatched build). */ - private volatile String evictionMode; - - /** {@inheritDoc} */ - public JmhPageEvictionBenchmark() { - keyGen.set(0); - } + /** Page eviction mode used for the data region. */ + @Param("RANDOM_LRU") + private String evictionMode; - /** Put of a small value (hot path, size-aware reserve takes its fast path). */ + /** Put of a small value (hot path, size-aware reserve takes its fast path). Keys are wrapped within a bounded + * range ({@link #SMALL_KEY_RANGE}) so the resident working set stays below the eviction threshold and the run + * isolates the fast-path cost instead of drifting into steady-state threshold eviction. */ @Benchmark public void putSmall() { - int key = keyGen.incrementAndGet(); + int key = keyGen.incrementAndGet() % SMALL_KEY_RANGE; cache.put(key, smallVal); } - /** Put of a large value against a nearly-full region (runs the size-aware eviction loop). */ + /** + * Put of a large value against a nearly-full region (runs the size-aware eviction loop). + *

    + * Pinned to a single thread: the size-aware reserve accumulates {@code requiredPages} real empty pages + * in the shared free list before writing, and with multiple concurrent writers those free pages are consumed + * by rivals as fast as they are freed, so no thread ever accumulates enough and the loop exhausts its + * no-progress budget into an out-of-memory. At one thread the free-page count grows monotonically and the + * reserve completes, measuring the honest per-put cost of eviction. + */ @Benchmark + @Threads(1) public void putLarge() { int key = keyGen.incrementAndGet(); @@ -125,10 +149,6 @@ public void putLarge() { /** Starts Ignite with an in-memory, eviction-enabled data region and pre-fills it for the LARGE scenario. */ @Setup(Level.Trial) public void setup() { - DataPageEvictionMode evictionMode = DataPageEvictionMode.RANDOM_LRU; - - this.evictionMode = evictionMode.name(); - long regionSize = 256 * 1024L * 1024L; DataStorageConfiguration dsCfg = new DataStorageConfiguration() @@ -136,7 +156,7 @@ public void setup() { .setPersistenceEnabled(false) .setMaxSize(regionSize) .setEmptyPagesPoolSize(POOL_SIZE) - .setPageEvictionMode(evictionMode)); + .setPageEvictionMode(DataPageEvictionMode.valueOf(evictionMode))); IgniteConfiguration cfg = new IgniteConfiguration() .setIgniteInstanceName("test") @@ -147,8 +167,8 @@ public void setup() { cache = ignite.getOrCreateCache(new CacheConfiguration(CACHE_NAME).setBackups(0)); - // Pre-fill the region with small entries for the LARGE scenario so that a large put cannot fit into the - // remaining headroom and must actually evict. + // Pre-fill the region with small entries for the LARGE scenario until threshold eviction pins it at the + // eviction threshold, so that a large put has no headroom to grow into and must actually evict. if ("LARGE".equalsIgnoreCase(scenario)) { try (IgniteDataStreamer ldr = ignite.dataStreamer(CACHE_NAME)) { ldr.perNodeBufferSize(1024); @@ -156,6 +176,11 @@ public void setup() { for (int i = 0; i < PRE_FILL_ENTRIES; i++) ldr.addData(i, smallVal); } + + // The pre-fill consumed keys [0, PRE_FILL_ENTRIES). Start large puts after that range so they write + // brand-new keys (true append), leaving the pre-filled small entries in place to be the eviction + // candidates, instead of overwriting them in place. + keyGen.set(PRE_FILL_ENTRIES); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index f6063a1708d6a..c0885b7803898 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -202,19 +202,21 @@ public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryP boolean evictOffheap) throws IgniteCheckedException; /** - * Same as {@link #evictInternal(GridCacheVersion, CacheEntryPredicate[], boolean)}, but acquires the entry lock - * non-blockingly when {@code tryLock} is {@code true}, returning {@code false} (instead of blocking) if the entry - * lock is contended. Used by size-aware page eviction which may run while the current thread already holds other - * entry locks, to avoid a lock-ordering deadlock. The default implementation uses the blocking variant. + * Same as {@link #evictInternal(GridCacheVersion, CacheEntryPredicate[], boolean)}, but when {@code tryLock} is + * {@code true} the entry lock is acquired non-blockingly: the entry is skipped (this method returns {@code false}) + * whenever its lock is contended or already held by the current thread (the self-hold case), instead of blocking. + * Used by size-aware page eviction which may run while the current thread already holds other entry locks, to + * avoid a lock-ordering deadlock. The default implementation ignores {@code tryLock} and uses the blocking + * variant. * * @param obsoleteVer Version for eviction. * @param filter Optional filter. * @param evictOffheap Evict offheap value flag. - * @param tryLock {@code true} to acquire the entry lock non-blockingly (skip contended entries). + * @param tryLock {@code true} to acquire the entry lock non-blockingly (skip contended or self-held entries). * @return {@code True} if entry could be evicted. * @throws IgniteCheckedException In case of error. */ - public default boolean evictInternal( + default boolean evictInternal( GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 99ecf9b496ee1..93bc2aacc303e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3658,10 +3658,10 @@ protected void removeValue() throws IgniteCheckedException { * Evicts necessary number of data pages if per-page eviction is configured in current {@link DataRegion}. */ private void ensureFreeSpace() throws IgniteCheckedException { - // Deadlock alert: evicting data page causes removing (and locking) all entries on the page one by one. - // This entry-level eviction is invoked only while NOT holding this entry's lock (all call sites run before - // lockEntry()). The separately invoked size-aware path (see RowStore.addRow -> - // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs while the lock IS held, so it relies on + // Deadlock alert: evicting a data page removes (and locks) all entries on the page one by one, so this + // entry-level eviction must only run while NOT holding this entry's lock (all call sites run before + // lockEntry()). The separate size-aware path (RowStore.addRow -> + // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs under the lock and instead relies on // the non-blocking tryLockEntry(0) inside evictInternal to avoid a lock-ordering deadlock. assert !lock.isHeldByCurrentThread(); @@ -3706,10 +3706,9 @@ private CacheEntryImplEx wrapVersionedWithValue() { try { if (F.isEmptyOrNulls(filter)) { // With tryLock=true (size-aware eviction running while the current thread already holds entry locks) - // the lock is acquired non-blockingly: a contended entry is skipped (returning false) rather than - // blocking, which prevents a lock-ordering deadlock between concurrent evictions. The eviction tracker - // will then pick another page. For all other paths (tryLock=false) the original - // blocking lockEntry() is preserved. + // the lock is taken non-blockingly and a contended entry is skipped (returns false) to avoid a + // lock-ordering deadlock; the tracker then picks another page. All other paths (tryLock=false) keep + // the original blocking lockEntry(). if (!lockEntry(tryLock)) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index f02efc6fb90be..d3e057992116a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -139,6 +141,26 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** Maximum initial size on 32-bit JVM */ private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024; + /** Initial backoff (nanoseconds) between consecutive no-progress eviction attempts. */ + private static final long EVICTION_BACKOFF_START_NANOS = 50_000L; + + /** Upper bound (nanoseconds) for the backoff between no-progress eviction attempts. */ + private static final long EVICTION_BACKOFF_MAX_NANOS = 1_000_000L; + + /** + * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without establishing a new + * highest number of empty pages before failing with an out-of-memory. Being time-based (measured from the last + * progress) rather than a fixed attempt count means a slow-but-progressing eviction is never torn down, while a + * genuinely stuck eviction (nothing evictable, or contenders that never release their locks) still terminates in + * bounded time instead of busy-spinning forever. + *

    + * This value also caps the worst-case time the size-aware reserve can hold the caller's entry lock: on the + * single-row path the eviction loop runs while the entry being written is locked, so in the pathological case + * (region under pressure and every evictable entry locked by a long-running transaction) a put can hold the + * entry lock for up to this duration before failing with an out-of-memory. + */ + private static final long EVICTION_NO_PROGRESS_TIMEOUT_MILLIS = 1_000L; + /** {@code True} to reuse memory on deactive. */ protected final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); @@ -1190,6 +1212,12 @@ public WALPointer latestWalPointerReservedForPreloading() { * The size-aware reserve is required because page eviction by itself only keeps a steady-state pool of empty pages * ({@link DataRegionConfiguration#getEmptyPagesPoolSize()}) and does not guarantee enough space for a single row * larger than this pool. + *

    + * Worst case: when called while the entry being written is locked (single-row insertion), the eviction loop can + * hold that lock for up to {@link #EVICTION_NO_PROGRESS_TIMEOUT_MILLIS} — only if no evictable entry releases + * its lock within that time (e.g. a long-running transaction holding all evictable entries), after which the + * call fails with {@link IgniteOutOfMemoryException} (reported as a critical failure to the configured failure + * handler). * * @param region Data region to be checked. * @param dataRowSize Size of data row to be inserted. @@ -1265,75 +1293,79 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati throws IgniteOutOfMemoryException, IgniteCheckedException { PageMemory pageMem = region.pageMemory(); - CacheFreeList freeList = freeListMap.get(regCfg.getName()); - - if (freeList == null) - return; - - long sysPageSize = pageMem.systemPageSize(); long pageSize = pageMem.pageSize(); - long totalPages = regCfg.getMaxSize() / sysPageSize; - // Maximum payload bytes that a single data page can hold for a fragmented row. long pagePayload = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - // Pages the row will actually occupy once written, and which the free list must be able to hand out on - // demand during the fragmented write. - long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; + // A row that fits into the steady-state empty-pages pool is satisfied by normal threshold eviction, so the + // fast path is a single comparison (no page computation, free-list lookup or page-memory reads on the hot + // small-put path). + long maxFastRowBytes = regCfg.getEmptyPagesPoolSize() * pagePayload; + + if (dataRowSize <= maxFastRowBytes) + return; + + CacheFreeList freeList = freeListMap.get(regCfg.getName()); - // If the row fits into the configured steady-state empty-pages pool, normal threshold eviction is enough. - if (requiredPages <= regCfg.getEmptyPagesPoolSize()) + if (freeList == null) return; + long totalPages = regCfg.getMaxSize() / pageMem.systemPageSize(); + + // Pages the row will actually occupy once written, and which the free list must hand out on demand during + // the fragmented write. + long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; + // The row fundamentally cannot fit into the whole region. if (requiredPages > totalPages) throw outOfMemory(regCfg); - // The reserve must guarantee that the free list actually holds `requiredPages` REAL empty pages, not merely - // that the region "as a whole" has apparent headroom. The apparent headroom (totalPages - loadedPages) is a - // shared, non-exclusive resource: two concurrent inserts may both count on it and then both run out of pages - // while the fragmented write is in progress (TOCTOU / raw OOM), because a fresh allocation cannot grow the - // region beyond its capacity. Real empty pages already sitting in the free list are the only resource - // writeSinglePage can reliably consume once the region is effectively full, so the loop below accumulates them - // until it has enough. (The headroom is nonetheless trusted in the fast path below, but only while the region - // is below the eviction threshold, i.e. where enough slack is guaranteed not to be exhausted by contention.) + // The reserve must guarantee that the free list holds `requiredPages` REAL empty pages, not merely that the + // region has apparent headroom. Apparent headroom (totalPages - loadedPages) is shared and non-exclusive: + // concurrent inserts can both count on it and then both run out of pages mid-write (TOCTOU / raw OOM), since + // a fresh allocation cannot grow the region beyond capacity. Once the region is effectively full, real empty + // pages already in the free list are the only resource the fragmented write can reliably consume, so the loop + // below accumulates them. (Headroom is trusted in the fast path only while the region is below the eviction + // threshold, i.e. where contention cannot exhaust the slack.) long emptyPages = freeList.emptyDataPages(); long headroom = totalPages - pageMem.loadedPages(); - // The region is "under pressure" once its loaded pages reach the eviction threshold: beyond that point a fresh - // allocation can no longer be trusted to grow the region, because concurrent writers may exhaust the shared - // headroom before this thread's fragmented write runs (TOCTOU raw OOM). Below the threshold the region has - // genuine slack, so a row that fits into the combined spare space is satisfied by growing the region rather - // than entering the eviction loop, which would otherwise destroy evictable entries (e.g. live short-TTL - // entries that must survive) merely to accumulate empty pages that the slack could have absorbed. + // The region is "under pressure" once loaded pages reach the eviction threshold; below it a fresh allocation + // can safely grow the region, so a row that fits into the combined spare space is satisfied without eviction + // (which would otherwise destroy evictable, e.g. short-TTL, entries just to accumulate empty pages the slack + // could have absorbed). long pagesThreshold = (long)(totalPages * regCfg.getEvictionThreshold()); boolean underPressure = pageMem.loadedPages() >= pagesThreshold; - // Fast path: the row can be satisfied without eviction when (a) the free list already holds enough real empty + // Fast path: the row is satisfiable without eviction when (a) the free list already holds enough real empty // pages, or (b) the region is not under pressure and has enough spare space to grow into. if (emptyPages >= requiredPages || (!underPressure && emptyPages + headroom >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); - // Evict data pages until the free list holds enough real empty pages. Progress is measured against the - // number of empty pages, so pages freed concurrently (e.g. by TTL cleanup) also count as progress. The loop - // is bounded to avoid an infinite busy-spin when there is nothing more to evict. Eviction here runs while the - // current thread may already hold entry locks (single-row insertion), so entries whose locks are contended are - // skipped (non-blocking) rather than blocked upon, to avoid a lock-ordering deadlock. - final int maxAttemptsWithoutProgress = 300; - + // Evict data pages until the free list holds enough real empty pages. Progress is measured against the count + // of empty pages, so pages freed concurrently (e.g. by TTL cleanup) also count. Eviction may run while the + // current thread already holds entry locks (single-row insertion), so contended entries are skipped + // (non-blocking) rather than blocked on, avoiding a lock-ordering deadlock. + // + // The guard is time-based (no progress for EVICTION_NO_PROGRESS_TIMEOUT_MILLIS) rather than attempt-count: + // a fixed budget could exhaust in milliseconds under contention/lock-holders and turn a slow-but-progressing + // eviction into a premature OOM. On each stalled iteration the thread backs off (rather than busy-spinning) + // both to save CPU and to let a lock holder run and release it. long bestEmptyPages = emptyPages; - int attemptsWithoutProgress = 0; + + long lastProgressNanos = System.nanoTime(); + + long backoffNanos = EVICTION_BACKOFF_START_NANOS; while (bestEmptyPages < requiredPages) { - if (region.metrics().onPageEvictionsStarted()) { + if (region.metrics().onPageEvictionsStarted()) U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); - } evictDataPageNonBlocking(evictionTracker); @@ -1341,24 +1373,27 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati long curEmptyPages = freeList.emptyDataPages(); - // Progress is measured against the best number of empty pages observed so far. Any iteration that does - // not establish a new best (including drops caused by concurrent inserts consuming pages) counts toward - // the no-progress guard, so the loop is bounded: if eviction cannot outpace concurrent consumption within - // a fixed number of attempts, we stop rather than busy-spinning indefinitely. + // Only an iteration that establishes a new highest empty-pages count counts as progress (drops caused by + // concurrent inserts consuming pages do not). As long as there is progress the loop continues; on a + // stalled iteration it backs off rather than busy-spinning. if (curEmptyPages > bestEmptyPages) { bestEmptyPages = curEmptyPages; - attemptsWithoutProgress = 0; + lastProgressNanos = System.nanoTime(); + + backoffNanos = EVICTION_BACKOFF_START_NANOS; } - else - attemptsWithoutProgress++; + else { + LockSupport.parkNanos(backoffNanos); - if (attemptsWithoutProgress >= maxAttemptsWithoutProgress) { - // No progress after many eviction attempts. The region is under pressure (fast path was already - // insufficient: real empty pages are below requiredPages), and eviction can neither outpace concurrent - // consumption nor free enough space. Throw OOM rather than busy-spinning indefinitely. - throw outOfMemory(regCfg); + backoffNanos = Math.min(backoffNanos << 1, EVICTION_BACKOFF_MAX_NANOS); } + + // Fail with OOM only after a sustained period without any progress: this bounds a genuinely stuck eviction + // (nothing evictable, or contenders never releasing their locks) without tearing down a slow-but- + // progressing one. The region is already under pressure (fast path failed), so OOM is correct here. + if (System.nanoTime() - lastProgressNanos > TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS)) + throw outOfMemory(regCfg); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 0c7eb18a09076..e48bbb9bdfa32 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -135,11 +135,15 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - // Size-aware reserve for each row in the batch. Eviction performed here runs without entry locks - // (see AbstractFreeList#insertDataRows), so this is safe. Reserving only the largest row is insufficient: - // insertDataRows consumes the reserve while writing the first large row, and its per-row threshold loop - // only restores emptyPagesPoolSize, which is smaller than a large row. A later large row in the batch - // would therefore exhaust page memory. + // Size-aware reserve for each row in the batch (reserving only the largest is insufficient: a later large + // row can still exhaust page memory mid-write). The reserve/consume TOCTOU race and the "second large row + // in a batch" case are both closed by the lazy re-reserve in AbstractFreeList#writeSinglePage, which + // re-runs the reserve on the row remainder when a fragmented write cannot take a page (a raw OOM there + // would otherwise be wrapped by insertDataRows into CorruptedFreeListException and reported as corruption). + // + // The reserve evicts non-blockingly even though the batch path holds no entry locks (so blocking would be + // deadlock-safe and more effective here): the same reserve path is shared with single-row insertion, + // which runs under an entry lock and must not block. for (CacheDataRow row : rows) { int rowSize = row.size(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index fade3443b109c..6a038272c054a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -162,6 +162,8 @@ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { boolean evictionDone = false; + boolean nonBlocking = Boolean.TRUE.equals(EVICT_NON_BLOCKING.get()); + for (CacheDataRowAdapter dataRow : rowsToEvict) { GridCacheContext cacheCtx = sharedCtx.cacheContext(dataRow.cacheId()); @@ -171,8 +173,7 @@ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) : cacheCtx.cache().entryEx(dataRow.key()); - evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, - Boolean.TRUE.equals(EVICT_NON_BLOCKING.get())); + evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, nonBlocking); } return evictionDone; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 62452095631cf..29ada4d8541a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; @@ -98,6 +99,12 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; + /** Data region this free list belongs to (used for lazy size-aware re-reserve on fragmented writes). */ + private final DataRegion dataRegion; + + /** Database shared manager (used for lazy size-aware re-reserve on fragmented writes). */ + private final IgniteCacheDatabaseSharedManager dbMgr; + /** Page list cache limit. */ private final AtomicLong pageListCacheLimit; @@ -462,6 +469,11 @@ public AbstractFreeList( rmvRow = new RemoveRowHandler(cacheGrpId == 0); this.evictionTracker = dataRegion.evictionTracker(); + this.dataRegion = dataRegion; + // The database manager is only needed for the on-demand re-reserve (an eviction-enabled, in-memory region), + // and is looked up lazily/null-safely because free lists can be built in unit tests against a kernal context + // without a cache processor (in which case eviction is disabled and the re-reserve never fires). + dbMgr = ctx.cache() == null ? null : ctx.cache().context().database(); this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); @@ -701,10 +713,29 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC * @throws IgniteCheckedException If failed. */ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { + // TOCTOU closure: the size-aware reserve (ensureFreeSpaceForInsert, invoked from RowStore.addRow/addRows + // before this write) accumulates enough real empty pages but does not pin them to this thread - a concurrent + // writer can consume them between the reserve and this allocation. When the free list cannot hand out a page, + // re-reserve on the remaining size and retry before allocating a brand-new page; otherwise the race surfaces + // as a raw IgniteOutOfMemoryException (wrapped into CorruptedFreeListException in the batch path). + // + // The re-reserve is an inline demand-eviction: reached from the BPlusTree.invoke row-creation closure, it may + // re-entrantly remove other entries from the same data tree. That is safe because the closure runs with no + // data-tree page locks held (page read lock released before it runs, leaf write lock taken after), and the + // outer operation revalidates via the page tag / triangle / removeId protocols. The key being written is + // skipped (its entry lock is held, so tryLock fails for it), so there is no self-eviction or lock-ordering + // deadlock; like the initial reserve, the re-reserve throws OOM if the row genuinely cannot fit. AbstractDataPageIO initIo = null; long pageId = takePage(row.size() - written, row, statHolder); + if (pageId == 0L) { + if (dbMgr != null) + dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); + + pageId = takePage(row.size() - written, row, statHolder); + } + if (pageId == 0L) { pageId = allocateDataPage(row.partition()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 979041f2f891e..4c706d7347f2b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -32,7 +32,6 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -43,8 +42,8 @@ * Negative test for the size-aware eviction progress guard. *

    * When every resident entry is locked by another thread/transaction, page eviction cannot free any page: the guarded - * {@code tryLockEntry(0)} in {@code evictInternal} fails for every candidate, so {@link - * IgniteCacheDatabaseSharedManager#ensureFreeSpaceForEviction} makes no progress and must fail with an + * {@code tryLockEntry(0)} in {@code evictInternal} fails for every candidate, so {@code ensureFreeSpaceForEviction} + * makes no progress and must fail with an * {@code IgniteOutOfMemoryException} within bounded time instead of busy-spinning forever (deadlock). *

    * The test is self-guarded by {@code @Test(timeout = ...)}: a deadlock or unbounded busy-spin would fail the diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 8acd1124d94a8..b3ea9d0e87df8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -102,6 +102,10 @@ private IgniteCache createCache(IgniteEx ignite) { /** * A large record (larger than the empty-pages pool) must be stored without OOM when there is evictable data, * by evicting previously stored records to free enough space. + *

    + * This scenario intentionally overlaps with + * {@link PageEvictionPutLargeObjectsAbstractTest#testPutLargeObjects}; the added value of this class is the + * additional coverage below (putAll, update-growth, read-back and the larger-than-region OOM case). * * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 3e5b2e933fc70..82c05e4a05e66 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -63,9 +63,9 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo * the region capacity so that the large short-TTL records are guaranteed not to be evicted before they expire. */ private static final int SMALL_ENTRIES = 6_000; - /** Fresh large records written after TTL expiry. Together with the pre-fill (6000 pages) plus the index structures - * they exceed the region capacity (~32768 pages), forcing size-aware eviction that accounts for the TTL-freed - * space. */ + /** Fresh large records written after TTL expiry. Together with the pre-fill ({@link #SMALL_ENTRIES} small entries, + * each occupying roughly one data page) plus the index structures, they exceed the region capacity (~32768 pages + * for a 128 MiB region with 4 KiB pages), forcing size-aware eviction that accounts for the TTL-freed space. */ private static final int FRESH_RECORDS = 12; /** Short TTL applied to some entries. */ @@ -125,7 +125,10 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { for (int i = 0; i < 30; i++) cache.put(i, val); - cache.get(0); + // The most recently written entry cannot have expired yet (TTL is far larger than this read), so a non-null + // read both verifies the cache is responsive after concurrent expiry/eviction (the test's goal) and is not + // racy. Reading the first written key would be racy (it may already have expired under the TTL). + assertNotNull("Cache must remain responsive after concurrent expiry and eviction", cache.get(29)); } /** From 965c9a8d83e7cc2826de986ccc96eb923bc5a4e7 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 11 Sep 2026 14:59:05 +0500 Subject: [PATCH 10/26] address review comments --- .../IgniteCacheDatabaseSharedManager.java | 89 ++++++++++--------- .../cache/persistence/RowStore.java | 24 +++-- .../evict/FairFifoPageEvictionTracker.java | 4 +- .../evict/NoOpPageEvictionTracker.java | 2 +- .../evict/PageAbstractEvictionTracker.java | 39 ++------ .../evict/PageEvictionTracker.java | 17 +++- .../evict/Random2LruPageEvictionTracker.java | 4 +- .../evict/RandomLruPageEvictionTracker.java | 4 +- .../freelist/AbstractFreeList.java | 47 ++++++++-- 9 files changed, 137 insertions(+), 93 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index d3e057992116a..91196ed1a50dc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -65,7 +65,6 @@ import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress; import org.apache.ignite.internal.processors.cache.persistence.evict.FairFifoPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker; -import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.Random2LruPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker; @@ -1225,8 +1224,7 @@ public WALPointer latestWalPointerReservedForPreloading() { * for putting a new entry, even after eviction. * @throws IgniteCheckedException If failed to evict data pages. */ - public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) - throws IgniteOutOfMemoryException, IgniteCheckedException { + public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { if (region == null) return; @@ -1250,8 +1248,7 @@ public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) * @param dataRowSize Size of data row to be inserted. * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. */ - private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) - throws IgniteOutOfMemoryException { + private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) throws IgniteOutOfMemoryException { long memorySize = regCfg.getMaxSize(); PageMemory pageMem = region.pageMemory(); @@ -1289,8 +1286,11 @@ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg * makes no progress). * @throws IgniteCheckedException If failed to evict data pages. */ - private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) - throws IgniteOutOfMemoryException, IgniteCheckedException { + private void ensureFreeSpaceForEviction( + DataRegion region, + DataRegionConfiguration regCfg, + int dataRowSize + ) throws IgniteOutOfMemoryException, IgniteCheckedException { PageMemory pageMem = region.pageMemory(); long pageSize = pageMem.pageSize(); @@ -1322,27 +1322,48 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati throw outOfMemory(regCfg); // The reserve must guarantee that the free list holds `requiredPages` REAL empty pages, not merely that the - // region has apparent headroom. Apparent headroom (totalPages - loadedPages) is shared and non-exclusive: - // concurrent inserts can both count on it and then both run out of pages mid-write (TOCTOU / raw OOM), since - // a fresh allocation cannot grow the region beyond capacity. Once the region is effectively full, real empty - // pages already in the free list are the only resource the fragmented write can reliably consume, so the loop - // below accumulates them. (Headroom is trusted in the fast path only while the region is below the eviction - // threshold, i.e. where contention cannot exhaust the slack.) + // region has apparent headroom. Both resources are shared and non-exclusive: emptyDataPages() is just a + // snapshot of a common counter, and any concurrent writer can consume these pages before this thread's write. + // The two resources differ in how they behave once the region is effectively full (loadedPages == totalPages): + // headroom (totalPages - loadedPages) can no longer grow the region - a fresh allocateDataPage would fail with + // a raw OOM - whereas empty pages already in the reuse bucket remain reachable through takePage(), and so are + // the only resource the fragmented write can then consume. This makes empty pages a necessary condition for + // the write to proceed on a full region, not a reservation pinned to this thread. The TOCTOU between this + // reserve and the actual write is closed by the lazy re-reserve in AbstractFreeList#writeSinglePage, which + // re-runs the reserve on the row remainder before allocating a new page; if the region genuinely cannot fit + // the concurrent writes, that path raises a proper OOM rather than corrupting the free list. (Headroom is + // trusted in the fast path only while the region is below the eviction threshold, i.e. where contention cannot + // exhaust the slack.) long emptyPages = freeList.emptyDataPages(); long headroom = totalPages - pageMem.loadedPages(); - // The region is "under pressure" once loaded pages reach the eviction threshold; below it a fresh allocation - // can safely grow the region, so a row that fits into the combined spare space is satisfied without eviction - // (which would otherwise destroy evictable, e.g. short-TTL, entries just to accumulate empty pages the slack - // could have absorbed). long pagesThreshold = (long)(totalPages * regCfg.getEvictionThreshold()); - boolean underPressure = pageMem.loadedPages() >= pagesThreshold; - - // Fast path: the row is satisfiable without eviction when (a) the free list already holds enough real empty - // pages, or (b) the region is not under pressure and has enough spare space to grow into. - if (emptyPages >= requiredPages || (!underPressure && emptyPages + headroom >= requiredPages)) + // The gate below reuses regCfg.evictionThreshold as a regime boundary, not as "when to start eviction" (that + // role belongs to evictionRequired(), which raises the same threshold but stops on emptyPages >= poolSize, so + // on a filling region loadedPages routinely exceeds it and no last 10% of page memory is left unused). + // + // loadedPages() counts every allocated page, including empty ones sitting in the reuse bucket: evicting a page + // moves it to that bucket but does not lower loadedPages. So on a filling in-memory region evictionRegime is the + // normal steady state once the region has materialized up to its capacity, not a rare anomaly. + // + // The two regimes: + // - below the threshold: the region still has real slack, so a row fitting into the combined spare space is + // satisfied without eviction - the region grows and live entries (e.g. short-TTL ones before expiry) are + // not evicted just to accumulate empty pages the slack could absorb; + // - at/above the threshold: headroom is no longer trustworthy (concurrent size-aware writers could commit + // the same headroom and both hit capacity mid-write - the TOCTOU case), so only real empty pages already + // in the free list are counted and eviction is driven below. + boolean evictionRegime = pageMem.loadedPages() >= pagesThreshold; + + // Fast path: heuristics to skip eviction when the row is likely satisfiable without it - (a) the free list + // already holds enough real empty pages, or (b) the region is below the eviction regime and has enough spare + // space to grow into. This is evaluated on a snapshot and is a necessary, not sufficient, condition: under heavy + // contention two size-aware writers can both pass it and then both consume the same empty pages. That race is + // recovered in AbstractFreeList#writeSinglePage (lazy re-reserve before allocation) and, if the region truly + // cannot fit the concurrent writes, surfaces as a proper OOM rather than mid-write corruption. + if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + headroom >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); @@ -1363,11 +1384,15 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati long backoffNanos = EVICTION_BACKOFF_START_NANOS; while (bestEmptyPages < requiredPages) { - if (region.metrics().onPageEvictionsStarted()) + if (region.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); + } - evictDataPageNonBlocking(evictionTracker); + // tryLock=true: eviction runs non-blockingly (contended or already-held entries are skipped) because the + // size-aware reserve can run while the current thread already holds entry locks (single-row insertion); + // blocking here would risk a lock-ordering deadlock. + evictionTracker.evictDataPage(true); region.metrics().updateEvictionRate(); @@ -1397,22 +1422,6 @@ private void ensureFreeSpaceForEviction(DataRegion region, DataRegionConfigurati } } - /** - * Invokes a single page eviction, acquiring entry locks non-blockingly so that contended entries are skipped. - * This is required when eviction runs while the current thread already holds entry locks (size-aware eviction - * from a single-row insertion) to avoid a lock-ordering deadlock. {@link NoOpPageEvictionTracker} - * (disabled eviction, never reaching this path) falls back to the plain {@code evictDataPage()}. - * - * @param evictionTracker Page eviction tracker. - * @throws IgniteCheckedException If failed to evict a data page. - */ - private void evictDataPageNonBlocking(PageEvictionTracker evictionTracker) throws IgniteCheckedException { - if (evictionTracker instanceof PageAbstractEvictionTracker) - ((PageAbstractEvictionTracker)evictionTracker).evictDataPageNonBlocking(); - else - evictionTracker.evictDataPage(); - } - /** * @param regCfg Data region configuration. * @return New {@link IgniteOutOfMemoryException} (also reported as a critical failure) for the given region. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index e48bbb9bdfa32..09b5cec52d25e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -135,21 +135,31 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - // Size-aware reserve for each row in the batch (reserving only the largest is insufficient: a later large - // row can still exhaust page memory mid-write). The reserve/consume TOCTOU race and the "second large row - // in a batch" case are both closed by the lazy re-reserve in AbstractFreeList#writeSinglePage, which - // re-runs the reserve on the row remainder when a fragmented write cannot take a page (a raw OOM there - // would otherwise be wrapped by insertDataRows into CorruptedFreeListException and reported as corruption). + // Size-aware pre-reserve for the rebalance batch (regular single puts get the same guarantee via the + // per-put reserve in addRow). Reserving "for each row" would collapse to reserving for the largest one: + // every reserve runs before any insert and only enforces a lower bound on the shared empty-pages counter, + // so the final guarantee is max(row sizes). A single reserve for the largest row is therefore equivalent + // and is what is done here. + // + // The batch insert path (insertDataRows) mirrors writeSinglePage: it runs its own lazy re-reserve on the + // trailing fragment when takePage fails, and rethrows IgniteOutOfMemoryException as-is rather than + // wrapping it into CorruptedFreeListException (see AbstractFreeList). This pre-reserve here is sized for + // the largest row in the batch and runs before any insert, so it bounds the empty-pages counter up front. // // The reserve evicts non-blockingly even though the batch path holds no entry locks (so blocking would be // deadlock-safe and more effective here): the same reserve path is shared with single-row insertion, // which runs under an entry lock and must not block. + int maxRowSize = 0; + for (CacheDataRow row : rows) { int rowSize = row.size(); - if (rowSize > 0) - ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), rowSize); + if (rowSize > maxRowSize) + maxRowSize = rowSize; } + + if (maxRowSize > 0) + ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), maxRowSize); } assert ctx.database().checkpointLockIsHeldByThread(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java index 04454d05acbac..a20c150db5f0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java @@ -65,8 +65,8 @@ public FairFifoPageEvictionTracker( } /** {@inheritDoc} */ - @Override public synchronized void evictDataPage() throws IgniteCheckedException { - evictDataPage(pageUsageList.pollFirst()); + @Override public synchronized void evictDataPage(boolean tryLock) throws IgniteCheckedException { + evictDataPage(pageUsageList.pollFirst(), tryLock); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java index 4409e35e7446b..9d1ba2a5fce6f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java @@ -38,7 +38,7 @@ public class NoOpPageEvictionTracker implements PageEvictionTracker { } /** {@inheritDoc} */ - @Override public void evictDataPage() { + @Override public void evictDataPage(boolean tryLock) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index 6a038272c054a..9f0bd9bbbcd42 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -32,7 +32,8 @@ import org.apache.ignite.internal.util.typedef.internal.U; /** - * + * Base for page eviction trackers sharing the data-page eviction logic + * ({@link #evictDataPage(int, boolean)}). */ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker { /** This number of least significant bits is dropped from timestamp. */ @@ -41,13 +42,6 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker /** Millis in day. */ private static final int DAY = 24 * 60 * 60 * 1000; - /** - * Thread-local marker that the current eviction is requested by size-aware eviction, which may run - * while the calling thread already holds entry locks. When set, entries whose locks are contended are skipped - * (via a non-blocking {@code evictInternal}) instead of blocking, avoiding a lock-ordering deadlock. - */ - private static final ThreadLocal EVICT_NON_BLOCKING = new ThreadLocal<>(); - /** Page memory. */ protected final PageMemoryNoStoreImpl pageMem; @@ -94,32 +88,15 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker return pageMem.loadedPages() > pagesThreshold && freeList.emptyDataPages() < regCfg.getEmptyPagesPoolSize(); } - /** - * Evicts a data page, acquiring entry locks in a non-blocking way so that contended entries are skipped instead - * of blocked upon. Used by size-aware eviction which may run while the calling thread already holds - * entry locks, to avoid a lock-ordering deadlock. - * - * @throws IgniteCheckedException If failed. - */ - public void evictDataPageNonBlocking() throws IgniteCheckedException { - Boolean prev = EVICT_NON_BLOCKING.get(); - - EVICT_NON_BLOCKING.set(Boolean.TRUE); - - try { - evictDataPage(); - } - finally { - EVICT_NON_BLOCKING.set(prev); - } - } - /** * @param pageIdx Page index. + * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries + * (e.g. when size-aware eviction runs while the current thread already holds entry locks), avoiding a + * lock-ordering deadlock. * @return true if at least one data row has been evicted * @throws IgniteCheckedException If failed. */ - final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { + final boolean evictDataPage(int pageIdx, boolean tryLock) throws IgniteCheckedException { long fakePageId = PageIdUtils.pageId(0, (byte)0, pageIdx); long page = pageMem.acquirePage(0, fakePageId); @@ -162,8 +139,6 @@ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { boolean evictionDone = false; - boolean nonBlocking = Boolean.TRUE.equals(EVICT_NON_BLOCKING.get()); - for (CacheDataRowAdapter dataRow : rowsToEvict) { GridCacheContext cacheCtx = sharedCtx.cacheContext(dataRow.cacheId()); @@ -173,7 +148,7 @@ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) : cacheCtx.cache().entryEx(dataRow.key()); - evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, nonBlocking); + evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, tryLock); } return evictionDone; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java index 82970fb52521c..553cc36c37096 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java @@ -38,15 +38,30 @@ public interface PageEvictionTracker extends LifecycleAware { */ public boolean evictionRequired(); + /** + * Evicts one data page (blocking flavor, equivalent to {@link #evictDataPage(boolean) evictDataPage(false)}). + * In most cases, all entries will be removed from the page. + * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be + * not possible, as some of them can be used by active transactions. + * + * @throws IgniteCheckedException In case of page memory error. + */ + public default void evictDataPage() throws IgniteCheckedException { + evictDataPage(false); + } + /** * Evicts one data page. * In most cases, all entries will be removed from the page. * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be * not possible, as some of them can be used by active transactions. * + * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries + * instead of blocking on them. Used by size-aware eviction that may run while the current thread already + * holds other entry locks, to avoid a lock-ordering deadlock. * @throws IgniteCheckedException In case of page memory error. */ - public void evictDataPage() throws IgniteCheckedException; + public void evictDataPage(boolean tryLock) throws IgniteCheckedException; /** * Call this method when last entry is removed from data page. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index 01300ffb73fa0..e0a837964a41a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -123,7 +123,7 @@ public Random2LruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public void evictDataPage() throws IgniteCheckedException { + @Override public void evictDataPage(boolean tryLock) throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -191,7 +191,7 @@ public Random2LruPageEvictionTracker( } } - if (evictDataPage(pageIdx(lruTrackingIdx))) + if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) return; evictAttemptsCnt++; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index b940681403c15..0e2df2e5da4d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -109,7 +109,7 @@ public RandomLruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public void evictDataPage() throws IgniteCheckedException { + @Override public void evictDataPage(boolean tryLock) throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -165,7 +165,7 @@ public RandomLruPageEvictionTracker( } } - if (evictDataPage(pageIdx(lruTrackingIdx))) + if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) return; evictAttemptsCnt++; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 29ada4d8541a9..b05d2591e4680 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -24,6 +24,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.metric.IoStatisticsHolder; import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; import org.apache.ignite.internal.pagemem.PageIdAllocator; @@ -619,6 +620,11 @@ private long allocateDataPage(int part) throws IgniteCheckedException { catch (IgniteCheckedException | Error e) { throw e; } + catch (IgniteOutOfMemoryException e) { + // A genuine capacity exhaustion (region cannot grow, eviction cannot help) is reported as OOM/critical + // failure by the reserve path; do not mislabel it as free-list corruption. + throw e; + } catch (Throwable t) { throw new CorruptedFreeListException("Failed to insert data row", t, grpId); } @@ -662,6 +668,21 @@ private long allocateDataPage(int part) throws IgniteCheckedException { long pageId = takePage(row.size() - written, row, statHolder); + // Symmetric TOCTOU closure to writeSinglePage for the trailing fragment of a row (full-page fragments + // go through writeWholePages -> writeSinglePage, which already has this re-reserve). The batch-level + // pre-reserve in RowStore.addRows covers only the largest row and runs before any insert, so a + // concurrent writer can consume those pages before this remainder is written; if the free list cannot + // hand out a page, re-reserve on the remaining size and retry before allocating a brand-new page. + // Unlike the single-row path this is not reached from a BPlusTree.invoke closure (the data tree is + // updated separately, after the batch), so demand-eviction here is safe; the batch also already runs an + // evictionRequired() loop above. + if (pageId == 0L) { + if (dbMgr != null) + dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); + + pageId = takePage(row.size() - written, row, statHolder); + } + if (pageId == 0L) { pageId = allocateDataPage(row.partition()); @@ -673,6 +694,11 @@ private long allocateDataPage(int part) throws IgniteCheckedException { assert written != FAIL_I; // We can't fail here. } } + catch (IgniteOutOfMemoryException e) { + // A genuine capacity exhaustion is reported as OOM/critical failure by the reserve path; do not mislabel + // it as free-list corruption. + throw e; + } catch (RuntimeException e) { throw new CorruptedFreeListException("Failed to insert data rows", e, grpId); } @@ -717,14 +743,23 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t // before this write) accumulates enough real empty pages but does not pin them to this thread - a concurrent // writer can consume them between the reserve and this allocation. When the free list cannot hand out a page, // re-reserve on the remaining size and retry before allocating a brand-new page; otherwise the race surfaces - // as a raw IgniteOutOfMemoryException (wrapped into CorruptedFreeListException in the batch path). + // as a raw IgniteOutOfMemoryException (propagated as-is, not mislabelled as corruption - see insertDataRow / + // insertDataRows catch clauses). The same re-reserve is applied in insertDataRows for the trailing fragment + // of a batch row. // // The re-reserve is an inline demand-eviction: reached from the BPlusTree.invoke row-creation closure, it may - // re-entrantly remove other entries from the same data tree. That is safe because the closure runs with no - // data-tree page locks held (page read lock released before it runs, leaf write lock taken after), and the - // outer operation revalidates via the page tag / triangle / removeId protocols. The key being written is - // skipped (its entry lock is held, so tryLock fails for it), so there is no self-eviction or lock-ordering - // deadlock; like the initial reserve, the re-reserve throws OOM if the row genuinely cannot fit. + // re-entrantly remove other entries from the same data tree. The closure runs with no data-tree page locks + // held: the search returns (releasing the page read lock) before the closure is invoked, and the leaf write + // lock is only taken afterwards, in tryInsert/tryReplace/tryRemoveFromLeaf (see BPlusTree.invokeDown); the + // outer operation revalidates via the page tag / triangle / removeId protocols. + // + // The non-blocking entry tryLock (evictDataPage(true) -> evictInternal(..., tryLock=true)) guarantees + // entry-level ordering only: the key being written is skipped (its entry lock is held by this thread, so + // tryLock fails for it), and a contended entry is skipped instead of blocked on. It does NOT make the + // eviction page-level lock-free: removing a row still takes blocking data-tree (and, for TTL entries, + // pending-tree) write locks. A cross-tree data->pending vs pending->data ordering inversion with the TTL + // expiration worker is therefore a known residual risk, not excluded by the non-blocking entry lock. Like + // the initial reserve, the re-reserve throws OOM if the row genuinely cannot fit. AbstractDataPageIO initIo = null; long pageId = takePage(row.size() - written, row, statHolder); From db9de4d33c0604f75b4c5e873eebdbce54c690c1 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 11 Sep 2026 23:14:40 +0500 Subject: [PATCH 11/26] improvement --- .../processors/cache/GridCacheMapEntry.java | 15 +-- .../IgniteCacheDatabaseSharedManager.java | 127 ++++++++---------- .../cache/persistence/RowStore.java | 20 +-- .../evict/FairFifoPageEvictionTracker.java | 6 +- .../evict/NoOpPageEvictionTracker.java | 5 +- .../evict/PageEvictionTracker.java | 8 +- .../evict/Random2LruPageEvictionTracker.java | 8 +- .../evict/RandomLruPageEvictionTracker.java | 8 +- .../freelist/AbstractFreeList.java | 110 ++++++++------- 9 files changed, 150 insertions(+), 157 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 93bc2aacc303e..5e8908cd32300 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3658,11 +3658,10 @@ protected void removeValue() throws IgniteCheckedException { * Evicts necessary number of data pages if per-page eviction is configured in current {@link DataRegion}. */ private void ensureFreeSpace() throws IgniteCheckedException { - // Deadlock alert: evicting a data page removes (and locks) all entries on the page one by one, so this - // entry-level eviction must only run while NOT holding this entry's lock (all call sites run before - // lockEntry()). The separate size-aware path (RowStore.addRow -> - // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs under the lock and instead relies on - // the non-blocking tryLockEntry(0) inside evictInternal to avoid a lock-ordering deadlock. + // Deadlock alert: evicting a data page removes (and locks) all entries on the page one by one, so this must + // only run while NOT holding this entry's lock (all call sites run before lockEntry()). The size-aware path + // (RowStore.addRow -> ensureFreeSpaceForInsert) runs under the lock and instead uses the non-blocking + // tryLockEntry(0) inside evictInternal. assert !lock.isHeldByCurrentThread(); cctx.shared().database().ensureFreeSpace(cctx.dataRegion()); @@ -3705,10 +3704,8 @@ private CacheEntryImplEx wrapVersionedWithValue() { try { if (F.isEmptyOrNulls(filter)) { - // With tryLock=true (size-aware eviction running while the current thread already holds entry locks) - // the lock is taken non-blockingly and a contended entry is skipped (returns false) to avoid a - // lock-ordering deadlock; the tracker then picks another page. All other paths (tryLock=false) keep - // the original blocking lockEntry(). + // With tryLock=true the lock is taken non-blockingly and a contended/self-held entry is skipped + // (returns false) to avoid a lock-ordering deadlock; the tracker then picks another page. if (!lockEntry(tryLock)) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 91196ed1a50dc..68fb07ba0d6e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -147,19 +147,26 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap private static final long EVICTION_BACKOFF_MAX_NANOS = 1_000_000L; /** - * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without establishing a new - * highest number of empty pages before failing with an out-of-memory. Being time-based (measured from the last + * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without either an eviction or a + * new highest empty-pages count before failing with an out-of-memory. Being time-based (measured from the last * progress) rather than a fixed attempt count means a slow-but-progressing eviction is never torn down, while a * genuinely stuck eviction (nothing evictable, or contenders that never release their locks) still terminates in - * bounded time instead of busy-spinning forever. - *

    - * This value also caps the worst-case time the size-aware reserve can hold the caller's entry lock: on the - * single-row path the eviction loop runs while the entry being written is locked, so in the pathological case - * (region under pressure and every evictable entry locked by a long-running transaction) a put can hold the - * entry lock for up to this duration before failing with an out-of-memory. + * bounded time instead of busy-spinning forever. The overall cycle is additionally capped by + * {@link #EVICTION_MAX_CYCLE_TIME_MILLIS}. */ private static final long EVICTION_NO_PROGRESS_TIMEOUT_MILLIS = 1_000L; + /** + * Hard upper bound (milliseconds) on the total duration of one size-aware eviction cycle, regardless of + * per-iteration progress. {@link #EVICTION_NO_PROGRESS_TIMEOUT_MILLIS} bounds the period with no progress, but + * eviction that keeps making partial progress (e.g. evicting a contended page's available entries one-by-one + * without ever emptying a page, so the empty-pages count never reaches the target) could otherwise extend the + * loop indefinitely; this absolute deadline guarantees the cycle still fails with an out-of-memory in bounded + * time while the per-progress timeout above allows a genuinely progressing eviction to run to its natural end. + * It also caps the worst-case time the reserve can hold the caller's entry lock. + */ + private static final long EVICTION_MAX_CYCLE_TIME_MILLIS = 5 * EVICTION_NO_PROGRESS_TIMEOUT_MILLIS; + /** {@code True} to reuse memory on deactive. */ protected final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); @@ -1213,10 +1220,10 @@ public WALPointer latestWalPointerReservedForPreloading() { * larger than this pool. *

    * Worst case: when called while the entry being written is locked (single-row insertion), the eviction loop can - * hold that lock for up to {@link #EVICTION_NO_PROGRESS_TIMEOUT_MILLIS} — only if no evictable entry releases - * its lock within that time (e.g. a long-running transaction holding all evictable entries), after which the - * call fails with {@link IgniteOutOfMemoryException} (reported as a critical failure to the configured failure - * handler). + * hold that lock for up to {@link #EVICTION_MAX_CYCLE_TIME_MILLIS} — only if eviction cannot consolidate enough + * empty pages within that time (e.g. a long-running transaction holding all evictable entries, or only partial + * progress), after which the call fails with {@link IgniteOutOfMemoryException} (reported as a critical failure + * to the configured failure handler). * * @param region Data region to be checked. * @param dataRowSize Size of data row to be inserted. @@ -1321,88 +1328,67 @@ private void ensureFreeSpaceForEviction( if (requiredPages > totalPages) throw outOfMemory(regCfg); - // The reserve must guarantee that the free list holds `requiredPages` REAL empty pages, not merely that the - // region has apparent headroom. Both resources are shared and non-exclusive: emptyDataPages() is just a - // snapshot of a common counter, and any concurrent writer can consume these pages before this thread's write. - // The two resources differ in how they behave once the region is effectively full (loadedPages == totalPages): - // headroom (totalPages - loadedPages) can no longer grow the region - a fresh allocateDataPage would fail with - // a raw OOM - whereas empty pages already in the reuse bucket remain reachable through takePage(), and so are - // the only resource the fragmented write can then consume. This makes empty pages a necessary condition for - // the write to proceed on a full region, not a reservation pinned to this thread. The TOCTOU between this - // reserve and the actual write is closed by the lazy re-reserve in AbstractFreeList#writeSinglePage, which - // re-runs the reserve on the row remainder before allocating a new page; if the region genuinely cannot fit - // the concurrent writes, that path raises a proper OOM rather than corrupting the free list. (Headroom is - // trusted in the fast path only while the region is below the eviction threshold, i.e. where contention cannot - // exhaust the slack.) + // The reserve must guarantee `requiredPages` REAL empty pages, not just apparent headroom. Both are shared and + // non-exclusive (emptyDataPages() is a snapshot; any writer can consume them), but once the region is full + // (loadedPages == totalPages) headroom can no longer grow it (fresh allocateDataPage -> raw OOM), while empty + // pages in the reuse bucket stay reachable via takePage(). So empty pages are the only resource the fragmented + // write can consume on a full region. The TOCTOU between this reserve and the actual write is closed by the + // lazy re-reserve in AbstractFreeList#writeSinglePage. long emptyPages = freeList.emptyDataPages(); long headroom = totalPages - pageMem.loadedPages(); long pagesThreshold = (long)(totalPages * regCfg.getEvictionThreshold()); - // The gate below reuses regCfg.evictionThreshold as a regime boundary, not as "when to start eviction" (that - // role belongs to evictionRequired(), which raises the same threshold but stops on emptyPages >= poolSize, so - // on a filling region loadedPages routinely exceeds it and no last 10% of page memory is left unused). - // - // loadedPages() counts every allocated page, including empty ones sitting in the reuse bucket: evicting a page - // moves it to that bucket but does not lower loadedPages. So on a filling in-memory region evictionRegime is the - // normal steady state once the region has materialized up to its capacity, not a rare anomaly. - // - // The two regimes: - // - below the threshold: the region still has real slack, so a row fitting into the combined spare space is - // satisfied without eviction - the region grows and live entries (e.g. short-TTL ones before expiry) are - // not evicted just to accumulate empty pages the slack could absorb; - // - at/above the threshold: headroom is no longer trustworthy (concurrent size-aware writers could commit - // the same headroom and both hit capacity mid-write - the TOCTOU case), so only real empty pages already - // in the free list are counted and eviction is driven below. + // The gate reuses evictionThreshold as a regime boundary, not as "when to start eviction" (evictionRequired() + // does that, stopping on emptyPages >= poolSize; no last 10% of page memory is left unusable). Below the + // threshold the region has real slack, so a row fitting into the combined spare space is satisfied without + // eviction (live, e.g. short-TTL, entries are not evicted just to accumulate empty pages). At/above it headroom + // is no longer trustworthy (concurrent writers could commit the same headroom - TOCTOU), so only real empty + // pages are counted and eviction is driven below. boolean evictionRegime = pageMem.loadedPages() >= pagesThreshold; - // Fast path: heuristics to skip eviction when the row is likely satisfiable without it - (a) the free list - // already holds enough real empty pages, or (b) the region is below the eviction regime and has enough spare - // space to grow into. This is evaluated on a snapshot and is a necessary, not sufficient, condition: under heavy - // contention two size-aware writers can both pass it and then both consume the same empty pages. That race is - // recovered in AbstractFreeList#writeSinglePage (lazy re-reserve before allocation) and, if the region truly - // cannot fit the concurrent writes, surfaces as a proper OOM rather than mid-write corruption. + // Fast path: skip eviction when (a) enough real empty pages exist, or (b) below the regime with enough spare + // space to grow into. This is a snapshot and only necessary, not sufficient: under contention two writers can + // both pass and consume the same pages - recovered by the lazy re-reserve in AbstractFreeList#writeSinglePage. if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + headroom >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); - // Evict data pages until the free list holds enough real empty pages. Progress is measured against the count - // of empty pages, so pages freed concurrently (e.g. by TTL cleanup) also count. Eviction may run while the - // current thread already holds entry locks (single-row insertion), so contended entries are skipped - // (non-blocking) rather than blocked on, avoiding a lock-ordering deadlock. - // - // The guard is time-based (no progress for EVICTION_NO_PROGRESS_TIMEOUT_MILLIS) rather than attempt-count: - // a fixed budget could exhaust in milliseconds under contention/lock-holders and turn a slow-but-progressing - // eviction into a premature OOM. On each stalled iteration the thread backs off (rather than busy-spinning) - // both to save CPU and to let a lock holder run and release it. + // Evict until the free list holds enough real empty pages. Progress counts as either an actual eviction this + // iteration (evictDataPage returned true) or a new high-water empty-pages count. The count alone is unreliable + // under concurrency: writers consume the shared counter as fast as eviction frees pages, so a thread may never + // see its high-water exceeded while eviction still makes real global progress - tearing down there would be a + // false OOM. Counting the actual eviction prevents that; when eviction genuinely cannot proceed the method + // returns false, so the loop still fails with OOM in bounded time. long bestEmptyPages = emptyPages; long lastProgressNanos = System.nanoTime(); + long cycleDeadlineNanos = lastProgressNanos + TimeUnit.MILLISECONDS.toNanos(EVICTION_MAX_CYCLE_TIME_MILLIS); + long backoffNanos = EVICTION_BACKOFF_START_NANOS; - while (bestEmptyPages < requiredPages) { + while (freeList.emptyDataPages() < requiredPages) { if (region.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); } - // tryLock=true: eviction runs non-blockingly (contended or already-held entries are skipped) because the - // size-aware reserve can run while the current thread already holds entry locks (single-row insertion); - // blocking here would risk a lock-ordering deadlock. - evictionTracker.evictDataPage(true); + // tryLock=true: skip contended/self-held entries (the reserve can run while the current thread already + // holds entry locks on the single-row path), avoiding a lock-ordering deadlock. + boolean evicted = evictionTracker.evictDataPage(true); region.metrics().updateEvictionRate(); long curEmptyPages = freeList.emptyDataPages(); - // Only an iteration that establishes a new highest empty-pages count counts as progress (drops caused by - // concurrent inserts consuming pages do not). As long as there is progress the loop continues; on a - // stalled iteration it backs off rather than busy-spinning. - if (curEmptyPages > bestEmptyPages) { - bestEmptyPages = curEmptyPages; + // A new high-water count or an actual eviction re-arms the no-progress timeout; a stalled iteration backs + // off rather than busy-spinning. + if (evicted || curEmptyPages > bestEmptyPages) { + if (curEmptyPages > bestEmptyPages) + bestEmptyPages = curEmptyPages; lastProgressNanos = System.nanoTime(); @@ -1414,10 +1400,13 @@ private void ensureFreeSpaceForEviction( backoffNanos = Math.min(backoffNanos << 1, EVICTION_BACKOFF_MAX_NANOS); } - // Fail with OOM only after a sustained period without any progress: this bounds a genuinely stuck eviction - // (nothing evictable, or contenders never releasing their locks) without tearing down a slow-but- - // progressing one. The region is already under pressure (fast path failed), so OOM is correct here. - if (System.nanoTime() - lastProgressNanos > TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS)) + // OOM when there was no eviction/count growth for EVICTION_NO_PROGRESS_TIMEOUT_MILLIS (stuck eviction) or + // the whole cycle exceeds EVICTION_MAX_CYCLE_TIME_MILLIS (only partial progress, target never reached). + long nowNanos = System.nanoTime(); + + long maxNoProgress = TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS); + + if (nowNanos - lastProgressNanos > maxNoProgress || nowNanos > cycleDeadlineNanos) throw outOfMemory(regCfg); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 09b5cec52d25e..77ab3436c05d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -135,20 +135,12 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - // Size-aware pre-reserve for the rebalance batch (regular single puts get the same guarantee via the - // per-put reserve in addRow). Reserving "for each row" would collapse to reserving for the largest one: - // every reserve runs before any insert and only enforces a lower bound on the shared empty-pages counter, - // so the final guarantee is max(row sizes). A single reserve for the largest row is therefore equivalent - // and is what is done here. - // - // The batch insert path (insertDataRows) mirrors writeSinglePage: it runs its own lazy re-reserve on the - // trailing fragment when takePage fails, and rethrows IgniteOutOfMemoryException as-is rather than - // wrapping it into CorruptedFreeListException (see AbstractFreeList). This pre-reserve here is sized for - // the largest row in the batch and runs before any insert, so it bounds the empty-pages counter up front. - // - // The reserve evicts non-blockingly even though the batch path holds no entry locks (so blocking would be - // deadlock-safe and more effective here): the same reserve path is shared with single-row insertion, - // which runs under an entry lock and must not block. + // Size-aware pre-reserve for the rebalance batch (single puts get the same guarantee via the per-put + // reserve in addRow). Reserving "for each row" collapses to reserving for the largest one (all reserves run + // before any insert and only enforce a lower bound on the shared empty-pages counter), so a single reserve + // for the max row is equivalent and is what is done here. The reserve evicts non-blockingly even though the + // batch path holds no entry locks (blocking would be safe here): the path is shared with single-row + // insertion, which runs under an entry lock and must not block. int maxRowSize = 0; for (CacheDataRow row : rows) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java index a20c150db5f0c..26035641d5539 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java @@ -65,8 +65,10 @@ public FairFifoPageEvictionTracker( } /** {@inheritDoc} */ - @Override public synchronized void evictDataPage(boolean tryLock) throws IgniteCheckedException { - evictDataPage(pageUsageList.pollFirst(), tryLock); + @Override public synchronized boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { + Integer pageIdx = pageUsageList.pollFirst(); + + return pageIdx != null && evictDataPage(pageIdx, tryLock); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java index 9d1ba2a5fce6f..137c331d5ade9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java @@ -38,8 +38,9 @@ public class NoOpPageEvictionTracker implements PageEvictionTracker { } /** {@inheritDoc} */ - @Override public void evictDataPage(boolean tryLock) { - // No-op. + @Override public boolean evictDataPage(boolean tryLock) { + // No-op: eviction is disabled, nothing is ever evicted. + return false; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java index 553cc36c37096..8d90fda724303 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java @@ -44,10 +44,11 @@ public interface PageEvictionTracker extends LifecycleAware { * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be * not possible, as some of them can be used by active transactions. * + * @return {@code true} if at least one data row has been evicted. * @throws IgniteCheckedException In case of page memory error. */ - public default void evictDataPage() throws IgniteCheckedException { - evictDataPage(false); + public default boolean evictDataPage() throws IgniteCheckedException { + return evictDataPage(false); } /** @@ -59,9 +60,10 @@ public default void evictDataPage() throws IgniteCheckedException { * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries * instead of blocking on them. Used by size-aware eviction that may run while the current thread already * holds other entry locks, to avoid a lock-ordering deadlock. + * @return {@code true} if at least one data row has been evicted. * @throws IgniteCheckedException In case of page memory error. */ - public void evictDataPage(boolean tryLock) throws IgniteCheckedException; + public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException; /** * Call this method when last entry is removed from data page. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index e0a837964a41a..ac361fcb7b10d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -123,7 +123,7 @@ public Random2LruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public void evictDataPage(boolean tryLock) throws IgniteCheckedException { + @Override public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -187,17 +187,19 @@ public Random2LruPageEvictionTracker( if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT); - return; + return false; } } if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) - return; + return true; evictAttemptsCnt++; } LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT); + + return false; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index 0e2df2e5da4d6..ff94ac714b3e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -109,7 +109,7 @@ public RandomLruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public void evictDataPage(boolean tryLock) throws IgniteCheckedException { + @Override public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -161,17 +161,19 @@ public RandomLruPageEvictionTracker( if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT); - return; + return false; } } if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) - return; + return true; evictAttemptsCnt++; } LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT); + + return false; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index b05d2591e4680..dc44d3873dad6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -76,6 +76,13 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private static final int MIN_PAGE_FREE_SPACE = 8; + /** + * Bounded number of lazy size-aware re-reserve attempts on a fragmented write before falling back to + * allocating a brand-new page. Each attempt runs size-aware eviction (which itself fails with a clean OOM when + * eviction cannot progress), so this bounds the retry even under heavy contention. + */ + private static final int RE_RESERVE_ATTEMPTS = 4; + /** * Step between buckets in free list, measured in powers of two. * For example, for page size 4096 and 256 buckets, shift is 4 and step is 16 bytes. @@ -471,9 +478,8 @@ public AbstractFreeList( this.evictionTracker = dataRegion.evictionTracker(); this.dataRegion = dataRegion; - // The database manager is only needed for the on-demand re-reserve (an eviction-enabled, in-memory region), - // and is looked up lazily/null-safely because free lists can be built in unit tests against a kernal context - // without a cache processor (in which case eviction is disabled and the re-reserve never fires). + // dbMgr is needed only for the on-demand re-reserve (eviction-enabled in-memory region); null in unit tests + // without a cache processor (where eviction is disabled and the re-reserve never fires). dbMgr = ctx.cache() == null ? null : ctx.cache().context().database(); this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); @@ -604,6 +610,44 @@ private long allocateDataPage(int part) throws IgniteCheckedException { return pageMem.allocatePage(grpId, part, FLAG_DATA); } + /** + * @return {@code true} when the region has effectively no headroom left (allocated pages reached the configured + * max), so a fresh {@code allocateDataPage} could no longer grow it. + */ + private boolean regionEffectivelyFull() { + return pageMem.loadedPages() >= dataRegion.config().getMaxSize() / pageMem.systemPageSize(); + } + + /** + * Take a page, and if the free list cannot hand one out, re-run the size-aware reserve and retry. The reserve only + * bounds the shared empty-pages counter and does not pin pages to this thread, so a concurrent writer may consume + * them before this allocation; retrying closes that TOCTOU instead of falling straight to a raw + * {@code allocateDataPage}. How hard to retry depends on whether the region can still grow: on an effectively-full + * region re-reserving is bounded (each attempt itself fails with a clean OOM when eviction cannot progress), while + * with headroom a single re-reserve suffices and the subsequent {@code allocateDataPage} grows the region. + * + * @param size Free space required on the page. + * @param row Row to write. + * @param statHolder Statistics holder to track IO operations. + * @return Page identifier or 0 if no page could be obtained after re-reserving. + * @throws IgniteCheckedException If failed. + */ + private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { + long pageId = takePage(size, row, statHolder); + + if (pageId == 0L && dbMgr != null) { + int reReserveAttempts = regionEffectivelyFull() ? RE_RESERVE_ATTEMPTS : 1; + + for (int i = 0; pageId == 0L && i < reReserveAttempts; i++) { + dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + + pageId = takePage(size, row, statHolder); + } + } + + return pageId; + } + /** {@inheritDoc} */ @Override public void insertDataRow(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { int written = 0; @@ -621,8 +665,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { throw e; } catch (IgniteOutOfMemoryException e) { - // A genuine capacity exhaustion (region cannot grow, eviction cannot help) is reported as OOM/critical - // failure by the reserve path; do not mislabel it as free-list corruption. + // OOM (reserve path reports it as a critical failure) must not be mislabelled as free-list corruption. throw e; } catch (Throwable t) { @@ -666,22 +709,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { AbstractDataPageIO initIo = null; - long pageId = takePage(row.size() - written, row, statHolder); - - // Symmetric TOCTOU closure to writeSinglePage for the trailing fragment of a row (full-page fragments - // go through writeWholePages -> writeSinglePage, which already has this re-reserve). The batch-level - // pre-reserve in RowStore.addRows covers only the largest row and runs before any insert, so a - // concurrent writer can consume those pages before this remainder is written; if the free list cannot - // hand out a page, re-reserve on the remaining size and retry before allocating a brand-new page. - // Unlike the single-row path this is not reached from a BPlusTree.invoke closure (the data tree is - // updated separately, after the batch), so demand-eviction here is safe; the batch also already runs an - // evictionRequired() loop above. - if (pageId == 0L) { - if (dbMgr != null) - dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); - - pageId = takePage(row.size() - written, row, statHolder); - } + long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { pageId = allocateDataPage(row.partition()); @@ -695,8 +723,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { } } catch (IgniteOutOfMemoryException e) { - // A genuine capacity exhaustion is reported as OOM/critical failure by the reserve path; do not mislabel - // it as free-list corruption. + // OOM (reserve path reports it as a critical failure) must not be mislabelled as free-list corruption. throw e; } catch (RuntimeException e) { @@ -739,37 +766,16 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC * @throws IgniteCheckedException If failed. */ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { - // TOCTOU closure: the size-aware reserve (ensureFreeSpaceForInsert, invoked from RowStore.addRow/addRows - // before this write) accumulates enough real empty pages but does not pin them to this thread - a concurrent - // writer can consume them between the reserve and this allocation. When the free list cannot hand out a page, - // re-reserve on the remaining size and retry before allocating a brand-new page; otherwise the race surfaces - // as a raw IgniteOutOfMemoryException (propagated as-is, not mislabelled as corruption - see insertDataRow / - // insertDataRows catch clauses). The same re-reserve is applied in insertDataRows for the trailing fragment - // of a batch row. - // - // The re-reserve is an inline demand-eviction: reached from the BPlusTree.invoke row-creation closure, it may - // re-entrantly remove other entries from the same data tree. The closure runs with no data-tree page locks - // held: the search returns (releasing the page read lock) before the closure is invoked, and the leaf write - // lock is only taken afterwards, in tryInsert/tryReplace/tryRemoveFromLeaf (see BPlusTree.invokeDown); the - // outer operation revalidates via the page tag / triangle / removeId protocols. - // - // The non-blocking entry tryLock (evictDataPage(true) -> evictInternal(..., tryLock=true)) guarantees - // entry-level ordering only: the key being written is skipped (its entry lock is held by this thread, so - // tryLock fails for it), and a contended entry is skipped instead of blocked on. It does NOT make the - // eviction page-level lock-free: removing a row still takes blocking data-tree (and, for TTL entries, - // pending-tree) write locks. A cross-tree data->pending vs pending->data ordering inversion with the TTL - // expiration worker is therefore a known residual risk, not excluded by the non-blocking entry lock. Like - // the initial reserve, the re-reserve throws OOM if the row genuinely cannot fit. + // Lazy re-reserve: the size-aware reserve (RowStore.addRow/addRows) only bounds the shared empty-pages counter + // and does not pin pages to this thread, so a concurrent writer may consume them before this allocation. The + // re-reserve below is an inline demand-eviction; reached from the BPlusTree.invoke row-creation closure, it + // removes entries with no data-tree page locks held (search releases the read lock before the closure, and the + // leaf write lock is taken only afterwards - see BPlusTree.invokeDown). Entry-level tryLock only skips + // contended/self-held entries and never blocks; the residual cross-tree data->pending vs pending->data + // lock-ordering risk with the TTL expiration worker is documented (known limitation, see design notes). AbstractDataPageIO initIo = null; - long pageId = takePage(row.size() - written, row, statHolder); - - if (pageId == 0L) { - if (dbMgr != null) - dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); - - pageId = takePage(row.size() - written, row, statHolder); - } + long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { pageId = allocateDataPage(row.partition()); From 3e810dc3663d7104b1303bdf9c91f71123ebd752 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 14 Sep 2026 16:26:01 +0500 Subject: [PATCH 12/26] minor --- .../processors/cache/GridCacheEntryEx.java | 2 +- .../processors/cache/GridCacheMapEntry.java | 5 ++-- .../IgniteCacheDatabaseSharedManager.java | 24 +++++-------------- .../cache/persistence/RowStore.java | 13 +++++----- .../evict/NoOpPageEvictionTracker.java | 1 - .../evict/PageAbstractEvictionTracker.java | 4 ++-- .../evict/PageEvictionTracker.java | 4 ++-- .../freelist/AbstractFreeList.java | 20 ++++++++-------- 8 files changed, 30 insertions(+), 43 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index c0885b7803898..042e9f78a323d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -216,7 +216,7 @@ public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryP * @return {@code True} if entry could be evicted. * @throws IgniteCheckedException In case of error. */ - default boolean evictInternal( + public default boolean evictInternal( GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 5e8908cd32300..dc34161f9b3ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3687,9 +3687,8 @@ private CacheEntryImplEx wrapVersionedWithValue() { @Override public boolean evictInternal( GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, - boolean evictOffheap) - throws IgniteCheckedException { - + boolean evictOffheap + ) throws IgniteCheckedException { return evictInternal(obsoleteVer, filter, evictOffheap, false); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 68fb07ba0d6e1..5139d9f263c72 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1256,13 +1256,9 @@ public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. */ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) throws IgniteOutOfMemoryException { - long memorySize = regCfg.getMaxSize(); - PageMemory pageMem = region.pageMemory(); - CacheFreeList freeList = freeListMap.get(regCfg.getName()); - - long nonEmptyPages = (pageMem.loadedPages() - freeList.emptyDataPages()); + long nonEmptyPages = (pageMem.loadedPages() - freeListMap.get(regCfg.getName()).emptyDataPages()); // The maximum number of pages that can be allocated (memorySize / systemPageSize) // should be greater or equal to pages required for inserting a new entry plus @@ -1271,7 +1267,7 @@ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg // Note that not the whole page can be used to storing links, // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, // in any way, the number of required pages is less than 1 percent. - boolean oomThreshold = (memorySize / pageMem.systemPageSize()) < + boolean oomThreshold = (regCfg.getMaxSize() / pageMem.systemPageSize()) < ((double)dataRowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); if (oomThreshold) @@ -1300,17 +1296,13 @@ private void ensureFreeSpaceForEviction( ) throws IgniteOutOfMemoryException, IgniteCheckedException { PageMemory pageMem = region.pageMemory(); - long pageSize = pageMem.pageSize(); - // Maximum payload bytes that a single data page can hold for a fragmented row. - long pagePayload = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; + long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; // A row that fits into the steady-state empty-pages pool is satisfied by normal threshold eviction, so the // fast path is a single comparison (no page computation, free-list lookup or page-memory reads on the hot // small-put path). - long maxFastRowBytes = regCfg.getEmptyPagesPoolSize() * pagePayload; - - if (dataRowSize <= maxFastRowBytes) + if (dataRowSize <= regCfg.getEmptyPagesPoolSize() * pagePayload) return; CacheFreeList freeList = freeListMap.get(regCfg.getName()); @@ -1336,22 +1328,18 @@ private void ensureFreeSpaceForEviction( // lazy re-reserve in AbstractFreeList#writeSinglePage. long emptyPages = freeList.emptyDataPages(); - long headroom = totalPages - pageMem.loadedPages(); - - long pagesThreshold = (long)(totalPages * regCfg.getEvictionThreshold()); - // The gate reuses evictionThreshold as a regime boundary, not as "when to start eviction" (evictionRequired() // does that, stopping on emptyPages >= poolSize; no last 10% of page memory is left unusable). Below the // threshold the region has real slack, so a row fitting into the combined spare space is satisfied without // eviction (live, e.g. short-TTL, entries are not evicted just to accumulate empty pages). At/above it headroom // is no longer trustworthy (concurrent writers could commit the same headroom - TOCTOU), so only real empty // pages are counted and eviction is driven below. - boolean evictionRegime = pageMem.loadedPages() >= pagesThreshold; + boolean evictionRegime = pageMem.loadedPages() >= (long)(totalPages * regCfg.getEvictionThreshold()); // Fast path: skip eviction when (a) enough real empty pages exist, or (b) below the regime with enough spare // space to grow into. This is a snapshot and only necessary, not sufficient: under contention two writers can // both pass and consume the same pages - recovered by the lazy re-reserve in AbstractFreeList#writeSinglePage. - if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + headroom >= requiredPages)) + if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + (totalPages - pageMem.loadedPages()) >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 77ab3436c05d6..916496363036d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -129,18 +129,19 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit } /** + * Size-aware pre-reserve for the rebalance batch, done when persistence is off and page eviction is enabled + * (single puts get the same guarantee via the per-put reserve in {@link #addRow}). Reserving "for each row" + * collapses to reserving for the largest one: all reserves run before any insert and only enforce a lower bound + * on the shared empty-pages counter, so a single reserve for the max row is equivalent and is what is done here. + * The reserve evicts non-blockingly even though the batch path holds no entry locks (blocking would be safe here): + * the path is shared with single-row insertion, which runs under an entry lock and must not block. + * * @param rows Rows. * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - // Size-aware pre-reserve for the rebalance batch (single puts get the same guarantee via the per-put - // reserve in addRow). Reserving "for each row" collapses to reserving for the largest one (all reserves run - // before any insert and only enforce a lower bound on the shared empty-pages counter), so a single reserve - // for the max row is equivalent and is what is done here. The reserve evicts non-blockingly even though the - // batch path holds no entry locks (blocking would be safe here): the path is shared with single-row - // insertion, which runs under an entry lock and must not block. int maxRowSize = 0; for (CacheDataRow row : rows) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java index 137c331d5ade9..0e85f17148bf4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java @@ -39,7 +39,6 @@ public class NoOpPageEvictionTracker implements PageEvictionTracker { /** {@inheritDoc} */ @Override public boolean evictDataPage(boolean tryLock) { - // No-op: eviction is disabled, nothing is ever evicted. return false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index 9f0bd9bbbcd42..d97a2b95e1c64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -91,8 +91,8 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker /** * @param pageIdx Page index. * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries - * (e.g. when size-aware eviction runs while the current thread already holds entry locks), avoiding a - * lock-ordering deadlock. + * (e.g. when size-aware eviction runs while the current thread already holds entry locks), avoiding a + * lock-ordering deadlock. * @return true if at least one data row has been evicted * @throws IgniteCheckedException If failed. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java index 8d90fda724303..49d6760608541 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java @@ -58,8 +58,8 @@ public default boolean evictDataPage() throws IgniteCheckedException { * not possible, as some of them can be used by active transactions. * * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries - * instead of blocking on them. Used by size-aware eviction that may run while the current thread already - * holds other entry locks, to avoid a lock-ordering deadlock. + * instead of blocking on them. Used by size-aware eviction that may run while the current thread already + * holds other entry locks, to avoid a lock-ordering deadlock. * @return {@code true} if at least one data row has been evicted. * @throws IgniteCheckedException In case of page memory error. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index dc44d3873dad6..f6d5df5448bbd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -612,7 +612,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { /** * @return {@code true} when the region has effectively no headroom left (allocated pages reached the configured - * max), so a fresh {@code allocateDataPage} could no longer grow it. + * max), so a fresh {@code allocateDataPage} could no longer grow it. */ private boolean regionEffectivelyFull() { return pageMem.loadedPages() >= dataRegion.config().getMaxSize() / pageMem.systemPageSize(); @@ -665,7 +665,6 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) throw e; } catch (IgniteOutOfMemoryException e) { - // OOM (reserve path reports it as a critical failure) must not be mislabelled as free-list corruption. throw e; } catch (Throwable t) { @@ -723,7 +722,6 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) } } catch (IgniteOutOfMemoryException e) { - // OOM (reserve path reports it as a critical failure) must not be mislabelled as free-list corruption. throw e; } catch (RuntimeException e) { @@ -758,6 +756,15 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC /** * Take a page and write row on it. + *

    + * The page is acquired via {@link #takePageWithReserve}: the size-aware reserve (RowStore.addRow/addRows) only + * bounds the shared empty-pages counter and does not pin pages to this thread, so a concurrent writer may consume + * them before this allocation — the lazy re-reserve closes that gap instead of falling straight to a raw + * {@code allocateDataPage}. Reached from the BPlusTree.invoke row-creation closure, this re-reserve is an inline + * demand-eviction that removes entries with no data-tree page locks held (the search releases the read lock before + * the closure; the leaf write lock is taken only afterwards — see BPlusTree.invokeDown). The entry-level tryLock + * only skips contended/self-held entries and never blocks; the residual risk is that the TTL expiration worker can + * still deadlock via cross-tree lock ordering (data->pending here vs pending->data there) — a known limitation. * * @param row Row to write. * @param written Written size. @@ -766,13 +773,6 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC * @throws IgniteCheckedException If failed. */ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { - // Lazy re-reserve: the size-aware reserve (RowStore.addRow/addRows) only bounds the shared empty-pages counter - // and does not pin pages to this thread, so a concurrent writer may consume them before this allocation. The - // re-reserve below is an inline demand-eviction; reached from the BPlusTree.invoke row-creation closure, it - // removes entries with no data-tree page locks held (search releases the read lock before the closure, and the - // leaf write lock is taken only afterwards - see BPlusTree.invokeDown). Entry-level tryLock only skips - // contended/self-held entries and never blocks; the residual cross-tree data->pending vs pending->data - // lock-ordering risk with the TTL expiration worker is documented (known limitation, see design notes). AbstractDataPageIO initIo = null; long pageId = takePageWithReserve(row.size() - written, row, statHolder); From 13387dc3695849316c8b893f6a90800e34a589e4 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 21 Sep 2026 18:00:20 +0500 Subject: [PATCH 13/26] address review comments p2 --- .../processors/cache/GridCacheMapEntry.java | 14 +++- .../IgniteCacheDatabaseSharedManager.java | 10 +-- .../freelist/AbstractFreeList.java | 34 ++++++---- ...eEvictionConcurrentWritesAbstractTest.java | 64 ++++--------------- 4 files changed, 51 insertions(+), 71 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index dc34161f9b3ba..fe24549fcdb12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3703,8 +3703,14 @@ private CacheEntryImplEx wrapVersionedWithValue() { try { if (F.isEmptyOrNulls(filter)) { - // With tryLock=true the lock is taken non-blockingly and a contended/self-held entry is skipped + // With tryLock=true the lock is taken non-blockingly and a contended entry is skipped // (returns false) to avoid a lock-ordering deadlock; the tracker then picks another page. + // Reentry is forbidden here to prevent self-eviction: if the current thread already holds + // the entry lock (e.g. during a row insert that triggers size-aware eviction), evicting + // the entry from under itself would corrupt in-flight operations. + if (tryLock && lockedByCurrentThread()) + return false; + if (!lockEntry(tryLock)) return false; @@ -3740,6 +3746,10 @@ private CacheEntryImplEx wrapVersionedWithValue() { } else { // For optimistic check. + // Reentry is forbidden when tryLock=true to prevent self-eviction (see the comment above). + if (tryLock && lockedByCurrentThread()) + return false; + while (true) { GridCacheVersion v; @@ -4209,7 +4219,7 @@ private int extrasSize() { */ private boolean lockEntry(boolean tryLock) { if (tryLock) - return !lock.isHeldByCurrentThread() && tryLockEntry(0); + return tryLockEntry(0); lockEntry(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 5139d9f263c72..542cd2e9a746a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1329,11 +1329,11 @@ private void ensureFreeSpaceForEviction( long emptyPages = freeList.emptyDataPages(); // The gate reuses evictionThreshold as a regime boundary, not as "when to start eviction" (evictionRequired() - // does that, stopping on emptyPages >= poolSize; no last 10% of page memory is left unusable). Below the - // threshold the region has real slack, so a row fitting into the combined spare space is satisfied without - // eviction (live, e.g. short-TTL, entries are not evicted just to accumulate empty pages). At/above it headroom - // is no longer trustworthy (concurrent writers could commit the same headroom - TOCTOU), so only real empty - // pages are counted and eviction is driven below. + // does that, stopping on emptyPages >= poolSize; no last (1 - evictionThreshold) of page memory is left + // unusable). Below the threshold the region has real slack, so a row fitting into the combined spare space is + // satisfied without eviction (live, e.g. short-TTL, entries are not evicted just to accumulate empty pages). + // At/above it headroom is no longer trustworthy (concurrent writers could commit the same headroom - TOCTOU), + // so only real empty pages are counted and eviction is driven below. boolean evictionRegime = pageMem.loadedPages() >= (long)(totalPages * regCfg.getEvictionThreshold()); // Fast path: skip eviction when (a) enough real empty pages exist, or (b) below the regime with enough spare diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index f6d5df5448bbd..91f209d559b7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.metric.IoStatisticsHolder; @@ -615,7 +616,15 @@ private long allocateDataPage(int part) throws IgniteCheckedException { * max), so a fresh {@code allocateDataPage} could no longer grow it. */ private boolean regionEffectivelyFull() { - return pageMem.loadedPages() >= dataRegion.config().getMaxSize() / pageMem.systemPageSize(); + long maxPages = dataRegion.config().getMaxSize() / pageMem.systemPageSize(); + + // Each of up to 16 segments loses up to one page to allocation overhead (lastAllocatedIdxPtr + alignment), + // so the theoretical max is never reached in practice. Subtract the worst-case segment loss to get an + // effective limit that reflects real product scenarios. + if (maxPages > 16) + maxPages -= 16; + + return pageMem.loadedPages() >= maxPages; } /** @@ -635,7 +644,8 @@ private boolean regionEffectivelyFull() { private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { long pageId = takePage(size, row, statHolder); - if (pageId == 0L && dbMgr != null) { + if (pageId == 0L && dbMgr != null && !dataRegion.config().isPersistenceEnabled() + && dataRegion.config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { int reReserveAttempts = regionEffectivelyFull() ? RE_RESERVE_ATTEMPTS : 1; for (int i = 0; pageId == 0L && i < reReserveAttempts; i++) { @@ -661,10 +671,7 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) } while (written != COMPLETE); } - catch (IgniteCheckedException | Error e) { - throw e; - } - catch (IgniteOutOfMemoryException e) { + catch (IgniteCheckedException | Error | IgniteOutOfMemoryException e) { throw e; } catch (Throwable t) { @@ -757,14 +764,13 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC /** * Take a page and write row on it. *

    - * The page is acquired via {@link #takePageWithReserve}: the size-aware reserve (RowStore.addRow/addRows) only - * bounds the shared empty-pages counter and does not pin pages to this thread, so a concurrent writer may consume - * them before this allocation — the lazy re-reserve closes that gap instead of falling straight to a raw - * {@code allocateDataPage}. Reached from the BPlusTree.invoke row-creation closure, this re-reserve is an inline - * demand-eviction that removes entries with no data-tree page locks held (the search releases the read lock before - * the closure; the leaf write lock is taken only afterwards — see BPlusTree.invokeDown). The entry-level tryLock - * only skips contended/self-held entries and never blocks; the residual risk is that the TTL expiration worker can - * still deadlock via cross-tree lock ordering (data->pending here vs pending->data there) — a known limitation. + * The page is acquired via {@link #takePageWithReserve}, which handles the size-aware re-reserve and TOCTOU + * recovery (see its javadoc for details). Reached from the BPlusTree.invoke row-creation closure, this re-reserve + * is an inline demand-eviction that runs without any BPlusTree page locks held by the row-creation closure (the + * search releases the read lock before the closure; the leaf write lock is taken only afterwards — see + * BPlusTree.invokeDown). The entry-level tryLock only skips contended/self-held entries and never blocks. + * The TTL expiration worker is also safe: it removes entries from the data tree and pending tree in separate + * BPlusTree operations, each releasing all page locks before returning, so no cross-tree nested locks occur. * * @param row Row to write. * @param written Written size. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index b25114b08dd2b..3c6de658d6f76 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -19,8 +19,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; @@ -28,6 +27,9 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -65,7 +67,7 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo private static final int SMALL_ENTRIES = 48_000; /** Number of writer threads. */ - private static final int THREADS = 2; + private static final int THREADS = 10; /** Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the * last large writes overflow the region and require size-aware eviction to free small entry pages. */ @@ -117,60 +119,22 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { byte[] largeVal = new byte[LARGE_RECORD_SIZE]; - AtomicLong errors = new AtomicLong(); - - AtomicReference firstErr = new AtomicReference<>(); - CountDownLatch startLatch = new CountDownLatch(1); - long deadline = System.currentTimeMillis() + DEADLINE; - - Thread[] threads = new Thread[THREADS]; - - for (int i = 0; i < THREADS; i++) { - final int threadIdx = i; - - threads[i] = new Thread(() -> { - try { - startLatch.await(); - - for (int k = 0; k < LARGE_ROWS_PER_THREAD; k++) - cache.put(SMALL_ENTRIES + threadIdx * LARGE_ROWS_PER_THREAD + k, largeVal); - } - catch (Throwable e) { - errors.incrementAndGet(); + AtomicInteger threadIdx = new AtomicInteger(); - firstErr.compareAndSet(null, e); + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { + U.awaitQuiet(startLatch); - log.error("Unexpected error in writer thread", e); - } - }, "paged-writer-" + i); + int idx = threadIdx.getAndIncrement(); - threads[i].start(); - } + for (int k = 0; k < LARGE_ROWS_PER_THREAD; k++) + cache.put(SMALL_ENTRIES + idx * LARGE_ROWS_PER_THREAD + k, largeVal); + }, + THREADS, "paged-writer"); startLatch.countDown(); - long start = System.currentTimeMillis(); - - for (Thread t : threads) - t.join(Math.max(1, deadline - System.currentTimeMillis())); - - // The core assertion of this deadlock test: every writer must have completed (no thread is stuck waiting on - // an entry lock held by size-aware eviction running under another entry lock). - for (Thread t : threads) { - if (t.isAlive()) { - log.error("Writer thread " + t.getName() + " is still alive after " + - (System.currentTimeMillis() - start) + "ms, state=" + t.getState()); - - for (StackTraceElement frame : t.getStackTrace()) - log.error(" at " + frame); - } - } - - for (Thread t : threads) - assertFalse("Writer thread " + t.getName() + " did not finish (possible deadlock)", t.isAlive()); - - assertEquals("Writer threads reported errors, reason: " + firstErr.get(), 0, errors.get()); + fut.get(DEADLINE); } } From b0fda5af93cfb2f6190a2cb144522bb386f3c0da Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 22 Sep 2026 13:54:17 +0500 Subject: [PATCH 14/26] refactor tests --- .../paged/PageEvictionAbstractTest.java | 34 ++++ ...eEvictionConcurrentWritesAbstractTest.java | 85 ++------- .../paged/PageEvictionGuardOomTest.java | 87 ++------- .../PageEvictionSizeAwareAbstractTest.java | 180 +++++++----------- ...eEvictionWithExpiryPolicyAbstractTest.java | 146 +++++--------- ...m2LruPageEvictionConcurrentWritesTest.java | 2 - .../Random2LruPageEvictionSizeAwareTest.java | 2 - ...m2LruPageEvictionWithExpiryPolicyTest.java | 2 - ...omLruPageEvictionConcurrentWritesTest.java | 2 - .../RandomLruPageEvictionSizeAwareTest.java | 2 - ...omLruPageEvictionWithExpiryPolicyTest.java | 2 - 11 files changed, 182 insertions(+), 362 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java index 18622516a2030..d48c954876a5a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.internal.processors.cache.eviction.paged; +import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; @@ -26,8 +27,13 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; +import javax.cache.expiry.Duration; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static javax.cache.expiry.CreatedExpiryPolicy.factoryOf; /** * @@ -130,4 +136,32 @@ protected CacheConfiguration cacheConfig( return cacheConfiguration; } + + /** + * @param ignite Ignite node. + * @param cacheName Cache name. + * @return Cache with a small partition count. + */ + protected static IgniteCache createCache(IgniteEx ignite, String cacheName) { + return createCache(ignite, cacheName, 0, false); + } + + /** + * @param ignite Ignite node. + * @param cacheName Cache name. + * @param ttl TTL in milliseconds ({@code 0} for no expiry). + * @return Cache with a small partition count and, if {@code ttl > 0}, eager TTL expiry. + */ + protected static IgniteCache createCache(IgniteEx ignite, String cacheName, long ttl, boolean transactional) { + CacheConfiguration ccfg = new CacheConfiguration(cacheName) + .setAffinity(new RendezvousAffinityFunction(false, 32)); + + if (transactional) + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + if (ttl > 0) + ccfg.setExpiryPolicyFactory(factoryOf(new Duration(MILLISECONDS, ttl))).setEagerTtl(true); + + return ignite.createCache(ccfg); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index 3c6de658d6f76..053e5cc8024e9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -21,20 +21,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; - /** * Concurrent deadlock test for size-aware page eviction. *

    @@ -45,46 +39,10 @@ * small entries rather than overrunning the free list. The test asserts that no deadlock occurs (all threads finish * within a global deadline). */ -public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommonAbstractTest { - /** Off-heap region size. */ - private static final int SIZE = 256 * 1024 * 1024; - - /** Partition count (kept low so that index-tree structures do not exhaust the region). */ - private static final int PARTITIONS = 32; - - /** Large record size (larger than the empty-pages pool so that each write is size-aware). */ - private static final int LARGE_RECORD_SIZE = 2 * 1024 * 1024; - - /** Small record size used to pre-fill the region with evictable data. */ - private static final int SMALL_RECORD_SIZE = 4096; - - /** Empty pages pool size. */ - private static final int POOL_SIZE = 100; - - /** Number of small pre-fill entries, leaving a buffer that is exceeded by the total of the large writes, so that - * the last of them can only be stored by freeing pages via size-aware eviction. The large records are small - * enough that concurrent size-aware eviction reliably frees the required pages (no spurious guard OOM). */ - private static final int SMALL_ENTRIES = 48_000; - - /** Number of writer threads. */ - private static final int THREADS = 10; - - /** Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the - * last large writes overflow the region and require size-aware eviction to free small entry pages. */ - private static final int LARGE_ROWS_PER_THREAD = 20; - - /** Global deadline for the whole test (protects against a deadlock/busy-spin hang). */ - private static final long DEADLINE = TimeUnit.MINUTES.toMillis(3); - +public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvictionAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - return super.getConfiguration(gridName) - .setDataStorageConfiguration(new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setInitialSize(SIZE) - .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE)) - .setPageSize(DFLT_PAGE_SIZE)); + return super.getConfiguration(gridName).setDataStorageConfiguration(new DataStorageConfiguration()); } /** {@inheritDoc} */ @@ -92,15 +50,6 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends GridCommo stopAllGrids(); } - /** - * @param ignite Ignite node. - * @return Cache with a small partition count (reduces structural page overhead). - */ - private IgniteCache createCache(IgniteEx ignite) { - return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS))); - } - /** * Concurrent large inserts into a region pre-filled with small entries must complete within the deadline without * deadlock, and without corrupting the free list (eviction frees small entries rather than overrunning the region). @@ -109,32 +58,40 @@ private IgniteCache createCache(IgniteEx ignite) { */ @Test public void testConcurrentLargeWritesNoDeadlock() throws Exception { + // Number of small pre-fill entries, leaving a buffer that is exceeded by the total of the large writes, so that + // the last of them can only be stored by freeing pages via size-aware eviction. The large records are small + // enough that concurrent size-aware eviction reliably frees the required pages (no spurious guard OOM). + int smallEntries = 48_000; + + // Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the + // last large writes overflow the region and require size-aware eviction to free small entry pages. + int largeRowsPerThread = 20; + IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); // Pre-fill the region with many small entries so that eviction always has evictable pages to free. - for (int i = 0; i < SMALL_ENTRIES; i++) - cache.put(i, new byte[SMALL_RECORD_SIZE]); + for (int i = 0; i < smallEntries; i++) + cache.put(i, new byte[4096]); - byte[] largeVal = new byte[LARGE_RECORD_SIZE]; + byte[] largeVal = new byte[2 * 1024 * 1024]; CountDownLatch startLatch = new CountDownLatch(1); AtomicInteger threadIdx = new AtomicInteger(); IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { - U.awaitQuiet(startLatch); + U.awaitQuiet(startLatch); - int idx = threadIdx.getAndIncrement(); + int idx = threadIdx.getAndIncrement(); - for (int k = 0; k < LARGE_ROWS_PER_THREAD; k++) - cache.put(SMALL_ENTRIES + idx * LARGE_ROWS_PER_THREAD + k, largeVal); - }, - THREADS, "paged-writer"); + for (int k = 0; k < largeRowsPerThread; k++) + cache.put(smallEntries + idx * largeRowsPerThread + k, largeVal); + }, 10, "paged-writer"); startLatch.countDown(); - fut.get(DEADLINE); + fut.get(TimeUnit.MINUTES.toMillis(3)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 4c706d7347f2b..019f07bfca8b3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -21,47 +21,29 @@ import java.util.Collection; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; +import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionSizeAwareAbstractTest.isOutOfMemory; - /** * Negative test for the size-aware eviction progress guard. *

    * When every resident entry is locked by another thread/transaction, page eviction cannot free any page: the guarded * {@code tryLockEntry(0)} in {@code evictInternal} fails for every candidate, so {@code ensureFreeSpaceForEviction} - * makes no progress and must fail with an - * {@code IgniteOutOfMemoryException} within bounded time instead of busy-spinning forever (deadlock). - *

    - * The test is self-guarded by {@code @Test(timeout = ...)}: a deadlock or unbounded busy-spin would fail the - * deadline. + * makes no progress and must fail with an {@code IgniteOutOfMemoryException} within bounded time instead of deadlock. */ -public class PageEvictionGuardOomTest extends GridCommonAbstractTest { +public class PageEvictionGuardOomTest extends PageEvictionAbstractTest { /** Off-heap region size. */ private static final int SIZE = 12 * 1024 * 1024; - /** Partition count (kept low so that index-tree structures do not exhaust the region). */ - private static final int PARTITIONS = 32; - - /** Empty pages pool size. */ - private static final int POOL_SIZE = 100; - - /** Small record size chosen to occupy roughly one data page ({@link DFLT_PAGE_SIZE}) each. */ - private static final int FILL_VALUE_SIZE = 3_800; - /** * Number of resident entries (each ~one page) filling the region to ~55% of its capacity. This keeps the region * comfortably below the eviction threshold (so the ordinary threshold-based {@code ensureFreeSpace} path is a @@ -70,9 +52,6 @@ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { */ private static final int FILL_ENTRIES = 1_600; - /** Large record size that does not fit into the remaining free space (requires eviction to be stored). */ - private static final int LARGE_RECORD_SIZE = 8 * 1024 * 1024; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { return super.getConfiguration(gridName) @@ -80,9 +59,7 @@ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE) - .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)) - .setPageSize(DFLT_PAGE_SIZE)); + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU))); } /** {@inheritDoc} */ @@ -90,18 +67,6 @@ public class PageEvictionGuardOomTest extends GridCommonAbstractTest { stopAllGrids(); } - /** - * @param ignite Ignite node. - * @return Cache with a small partition count (reduces structural page overhead). - */ - private IgniteCache createCache(IgniteEx ignite) { - // TRANSACTIONAL is required so that cache.lockAll(...) can hold entry locks (the root cause of the - // "no evictable page" scenario this test exercises). - return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)) - .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); - } - /** * Filling the region with locked entries and then writing a row that needs more free pages than remain must fail * with OOM (bounded time), not hang: eviction cannot free any page because every candidate entry is locked. @@ -112,10 +77,10 @@ private IgniteCache createCache(IgniteEx ignite) { public void testGuardOomWhenAllEntriesLocked() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, 0, true); // Pre-fill the region so that less than one large record of free space remains, without overflowing it. - byte[] fillVal = new byte[FILL_VALUE_SIZE]; + byte[] fillVal = new byte[3_800]; for (int i = 1; i <= FILL_ENTRIES; i++) cache.put(i, fillVal); @@ -129,48 +94,34 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { CountDownLatch release = new CountDownLatch(1); - AtomicReference lockerErr = new AtomicReference<>(); - // Hold entry locks on every resident key from a background thread so that eviction has no evictable page. - Thread locker = new Thread(() -> { - try { - Lock lock = cache.lockAll(keys); + IgniteInternalFuture lockerFut = GridTestUtils.runAsync(() -> { + Lock lock = cache.lockAll(keys); - lock.lock(); + lock.lock(); + try { ready.countDown(); release.await(); - - lock.unlock(); } - catch (Throwable e) { - lockerErr.set(e); - - ready.countDown(); + finally { + lock.unlock(); } }, "size-aware-guard-locker"); - locker.start(); - try { assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); - assertNull("Unexpected error while locking entries: " + lockerErr.get(), lockerErr.get()); - - try { - cache.put(FILL_ENTRIES + 1, new byte[LARGE_RECORD_SIZE]); - - fail("Expected out-of-memory because all resident entries are locked, but put succeeded"); - } - catch (Exception e) { - assertTrue("Expected an out-of-memory (progress guard) failure, but got: " + e, isOutOfMemory(e)); - } + GridTestUtils.assertThrowsWithCause( + () -> cache.put(FILL_ENTRIES + 1, new byte[8 * 1024 * 1024]), + IgniteOutOfMemoryException.class + ); } finally { release.countDown(); - locker.join(TimeUnit.SECONDS.toMillis(10)); + lockerFut.get(10, TimeUnit.SECONDS); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index b3ea9d0e87df8..4a33b342a5c15 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -21,19 +21,14 @@ import java.util.HashMap; import java.util.Map; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; - /** * Tests size-aware page eviction on in-memory (non-persistent) data regions. *

    @@ -42,47 +37,35 @@ * which fundamentally cannot fit into the region fails with OOM instead of hanging in an infinite eviction loop. * The batch path ({@code putAll} of large rows) and the update path (growing a row) are covered as well. */ -public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstractTest { +public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbstractTest { /** Off-heap region size (large enough to hold cache structural pages with the configured partition count). */ private static final int SIZE = 128 * 1024 * 1024; - /** Partition count (kept low so that index-tree structures do not exhaust the region). */ - private static final int PARTITIONS = 32; - - /** Record size: chosen so that a single row requires more pages than are left free when the region is kept at the + /** + * Record size: chosen so that a single row requires more pages than are left free when the region is kept at the * eviction threshold ({@code (1 - threshold) * totalPages}). This guarantees a large put cannot take the fast * path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve - * ({@code ensureFreeSpaceForEviction}), which is the scenario these tests are meant to cover. */ + * ({@code ensureFreeSpaceForEviction}), which is the scenario these tests are meant to cover. + */ private static final int RECORD_SIZE = 32 * 1024 * 1024; - /** Empty pages pool size. */ - private static final int POOL_SIZE = 100; - - /** Entry count to accumulate beyond the region capacity. */ - private static final int ENTRIES = 40; - /** Small record size used to pre-fill the region with evictable data (for putAll tests). */ private static final int SMALL_RECORD_SIZE = 4096; /** * Small pre-fill entries count. Chosen to fill the 128 MiB region close to capacity so that less than one large * record ({@link #RECORD_SIZE}) remains available, forcing {@code ensureFreeSpaceForInsert} to actually evict - * prefilled entries rather than taking its fast path. + * pre-filled entries rather than taking its fast path. */ private static final int SMALL_ENTRIES = 28_000; - /** Large rows written via putAll. */ - private static final int PUT_ALL_LARGE_ROWS = 3; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { return super.getConfiguration(gridName) .setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) - .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE)) - .setPageSize(DFLT_PAGE_SIZE)); + .setMaxSize(SIZE))); } /** {@inheritDoc} */ @@ -90,71 +73,10 @@ public abstract class PageEvictionSizeAwareAbstractTest extends GridCommonAbstra stopAllGrids(); } - /** - * @param ignite Ignite node. - * @return Cache with a small partition count (reduces structural page overhead). - */ - private IgniteCache createCache(IgniteEx ignite) { - return ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS))); - } - - /** - * A large record (larger than the empty-pages pool) must be stored without OOM when there is evictable data, - * by evicting previously stored records to free enough space. - *

    - * This scenario intentionally overlaps with - * {@link PageEvictionPutLargeObjectsAbstractTest#testPutLargeObjects}; the added value of this class is the - * additional coverage below (putAll, update-growth, read-back and the larger-than-region OOM case). - * - * @throws Exception If failed. - */ - @Test - public void testPutLargeObjectsDoesNotOom() throws Exception { - IgniteEx ignite = startGrids(2); - - IgniteCache cache = createCache(ignite); - - Object val = new byte[RECORD_SIZE]; - - // Total data (ENTRIES * RECORD_SIZE) exceeds the region size, so at least some records must be evicted. - for (Integer key : primaryKeys(grid(1).cache(DEFAULT_CACHE_NAME), ENTRIES)) - cache.put(key, val); - - // Eviction must have bounded the number of resident entries. - assertTrue("Expected some entries to be evicted, but cache.size()=" + cache.size(), - cache.size() > 0 && cache.size() < ENTRIES); - } - - /** - * A large record written must be readable right away (the just-written entry is the most recently used and is not - * a candidate for eviction before the write completes). - * - * @throws Exception If failed. - */ - @Test - public void testLargeObjectReadBack() throws Exception { - IgniteEx ignite = startGrid(1); - - IgniteCache cache = createCache(ignite); - - byte[] val = new byte[RECORD_SIZE]; - - Arrays.fill(val, (byte)42); - - cache.put(1, val); - - byte[] read = (byte[])cache.get(1); - - assertNotNull("Large value must be readable after put", read); - - assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); - } - /** * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. - * Uses a transactional cache so that the size-aware OOM propagates directly (in an atomic cache it is wrapped in - * a {@code CachePartialUpdateException} and would not be detectable as the specific OOM). + * A batch {@code putAll} of records whose total size exceeds the region must also fail with OOM (exercises the + * batch store path {@code RowStore.addRows} → {@code ensureFreeSpaceForInsert}). * * @throws Exception If failed. */ @@ -162,23 +84,26 @@ public void testLargeObjectReadBack() throws Exception { public void testRecordLargerThanRegionOom() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)) - .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + + GridTestUtils.assertThrowsWithCause( + () -> cache.put(1, new byte[SIZE * 2]), + IgniteOutOfMemoryException.class + ); - boolean rejected = false; + GridTestUtils.assertThrowsWithCause( + () -> { + Map batch = new HashMap<>(); - try { - cache.put(1, new byte[SIZE * 2]); - } - catch (Exception e) { - assertTrue("Expected IgniteOutOfMemoryException because the row cannot fit into the region, but got: " + e, - isOutOfMemory(e)); + Object val = new byte[RECORD_SIZE]; - rejected = true; - } + for (int i = 0; i < 4; i++) + batch.put(i, val); - assertTrue("Record larger than the region must be rejected (no hang), but put succeeded", rejected); + cache.putAll(batch); + }, + IgniteOutOfMemoryException.class + ); } /** @@ -189,12 +114,14 @@ public void testRecordLargerThanRegionOom() throws Exception { */ @Test public void testPutAllLargeRows() throws Exception { + int putAllLargeRows = 3; + IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); // Pre-fill the region to near capacity with small evictable entries so that less than one large row remains - // available. This forces ensureFreeSpaceForInsert to evict prefilled entries rather than taking its fast path. + // available. This forces ensureFreeSpaceForInsert to evict pre-filled entries rather than taking its fast path. byte[] small = new byte[SMALL_RECORD_SIZE]; for (int i = 0; i < SMALL_ENTRIES; i++) @@ -204,19 +131,19 @@ public void testPutAllLargeRows() throws Exception { Object val = new byte[RECORD_SIZE]; - for (int i = 0; i < PUT_ALL_LARGE_ROWS; i++) + for (int i = 0; i < putAllLargeRows; i++) large.put(i, val); cache.putAll(large); - for (int i = 0; i < PUT_ALL_LARGE_ROWS; i++) + for (int i = 0; i < putAllLargeRows; i++) assertNotNull("Large row " + i + " must be readable after putAll", cache.get(i)); } /** * Updating a record from a small to a large value (larger than the empty-pages pool) must succeed with page * eviction enabled: the update goes through the same size-aware reserve as an insert. The region is pre-filled - * to near capacity so that the grown value cannot fit without evicting prefilled entries. + * to near capacity so that the grown value cannot fit without evicting pre-filled entries. * * @throws Exception If failed. */ @@ -224,7 +151,7 @@ public void testPutAllLargeRows() throws Exception { public void testUpdateRowGrows() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); // Pre-fill the region to near capacity with small evictable entries so that the grown value below cannot // fit without eviction. @@ -245,20 +172,41 @@ public void testUpdateRowGrows() throws Exception { byte[] read = (byte[])cache.get(1); assertNotNull("Updated large value must be readable", read); - assertTrue("Updated value must equal the stored value", Arrays.equals(big, read)); } /** - * @param t Throwable. - * @return {@code True} if {@code t} or any of its causes is an out-of-memory. + * Verifies the {@code !evictionRegime} fast path in {@code ensureFreeSpaceForEviction}: when the region is below + * the eviction threshold, a large row (exceeding the empty-pages pool) that fits in the remaining headroom must be + * written successfully — the size-aware reserve trusts headroom and does not trigger size-aware eviction. + * + * @throws Exception If failed. */ - static boolean isOutOfMemory(Throwable t) { - for (Throwable cur = t; cur != null; cur = cur.getCause()) { - if (cur instanceof IgniteOutOfMemoryException) - return true; - } + @Test + public void testGrowBeyondEvictionThreshold() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + + // Pre-fill to ~80 MiB — below the 0.9 * 128 MiB = 115.2 MiB threshold, leaving ~48 MiB of headroom. + byte[] small = new byte[SMALL_RECORD_SIZE]; + + int preFill = 20_000; - return false; + for (int i = 0; i < preFill; i++) + cache.put(i, small); + + // Write a 32 MiB row that fits in the remaining headroom. The size-aware reserve sees loadedPages below the + // threshold, trusts headroom (evictionRegime = false), and skips size-aware eviction. + byte[] val = new byte[RECORD_SIZE]; + + Arrays.fill(val, (byte)1); + + cache.put(preFill, val); + + byte[] read = (byte[])cache.get(preFill); + + assertNotNull("Large row must be readable after writing below threshold", read); + assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 82c05e4a05e66..8462f55d0d2be 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -17,59 +17,37 @@ package org.apache.ignite.internal.processors.cache.eviction.paged; -import javax.cache.expiry.CreatedExpiryPolicy; -import javax.cache.expiry.Duration; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; - /** * Tests the synergy between ExpiryPolicy (TTL cleanup) and size-aware page eviction on an in-memory data region. * Verifies that concurrent TTL cleanup and eviction do not deadlock, that a large row larger than the * empty-pages pool is still written when eviction is enabled, and that TTL-freed space is accounted for by eviction * (a row that only fits after expired entries are removed is still written without OOM). */ -public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommonAbstractTest { +public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvictionAbstractTest { /** Off-heap region size. */ private static final int SIZE = 128 * 1024 * 1024; - /** Partition count (kept low so that index-tree structures do not exhaust the region). */ - private static final int PARTITIONS = 32; - - /** Large record size (much larger than the empty-pages pool, and larger than the space left free when the region + /** + * Large record size (much larger than the empty-pages pool, and larger than the space left free when the region * is held at the eviction threshold {@code (1 - threshold) * totalPages}) so that a large put cannot take the - * fast path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve. */ + * fast path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve. + */ private static final int RECORD_SIZE = 32 * 1024 * 1024; - /** Empty pages pool size. */ - private static final int POOL_SIZE = 100; - - /** Size of a single non-expiring pre-fill record (well below a data-page payload so each record surely occupies - * exactly one data page). */ - private static final int SMALL_RECORD_SIZE = 1024; - - /** Number of non-expiring pre-fill records. Deliberately conservative: the pre-fill keeps a comfortable margin to - * the region capacity so that the large short-TTL records are guaranteed not to be evicted before they expire. */ - private static final int SMALL_ENTRIES = 6_000; - - /** Fresh large records written after TTL expiry. Together with the pre-fill ({@link #SMALL_ENTRIES} small entries, - * each occupying roughly one data page) plus the index structures, they exceed the region capacity (~32768 pages - * for a 128 MiB region with 4 KiB pages), forcing size-aware eviction that accounts for the TTL-freed space. */ - private static final int FRESH_RECORDS = 12; - - /** Short TTL applied to some entries. */ - private static final long TTL = 8000; + /** + * Short TTL applied to some entries. Kept short enough that entries expire while the eviction loop is still + * active (not after all entries have already been evicted), so the TTL worker genuinely runs concurrently with + * size-aware eviction. + */ + private static final long TTL = 2000; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -77,9 +55,7 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo .setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setInitialSize(SIZE) - .setMaxSize(SIZE) - .setEmptyPagesPoolSize(POOL_SIZE)) - .setPageSize(DFLT_PAGE_SIZE)); + .setMaxSize(SIZE))); } /** {@inheritDoc} */ @@ -87,24 +63,6 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends GridCommo stopAllGrids(); } - /** - * @param ignite Ignite node. - * @param cacheName Cache name. - * @param ttl TTL in milliseconds ({@code 0} for no expiry). - * @return Cache with a small partition count and, if {@code ttl > 0}, eager TTL expiry. - */ - private IgniteCache createCache(IgniteEx ignite, String cacheName, long ttl) { - CacheConfiguration ccfg = new CacheConfiguration(cacheName) - .setAffinity(new RendezvousAffinityFunction(false, PARTITIONS)); - - if (ttl > 0) { - ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(MILLISECONDS, ttl))) - .setEagerTtl(true); - } - - return ignite.createCache(ccfg); - } - /** * Concurrent TTL cleanup and eviction must not deadlock, and a large record (larger than the empty-pages pool) * must still be stored on a region with enabled eviction even in the presence of short-TTL entries. @@ -116,84 +74,68 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { IgniteEx ignite = startGrid(1); // Short-TTL entries keep the TTL worker actively freeing pages while eviction runs. - IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, TTL); + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, TTL, false); Object val = new byte[RECORD_SIZE]; // Writing more data than the region can hold forces eviction; concurrent expiry of short-TTL entries must not - // deadlock with it. The test itself is protected against a hang by the framework test timeout. - for (int i = 0; i < 30; i++) + // deadlock with it. The entry count and TTL are chosen so that early entries expire while the eviction loop is + // still active for later entries, giving the TTL worker a real chance to run concurrently with eviction. + for (int i = 0; i < 60; i++) cache.put(i, val); // The most recently written entry cannot have expired yet (TTL is far larger than this read), so a non-null // read both verifies the cache is responsive after concurrent expiry/eviction (the test's goal) and is not // racy. Reading the first written key would be racy (it may already have expired under the TTL). - assertNotNull("Cache must remain responsive after concurrent expiry and eviction", cache.get(29)); + assertNotNull("Cache must remain responsive after concurrent expiry and eviction", cache.get(59)); } /** - * Space freed by TTL cleanup must be taken into account by size-aware eviction: large records written after some - * entries have expired must be accepted (no OOM) because their pages become available. - *

    - * The region is pre-filled with non-expiring small entries and large short-TTL entries that later expire and free - * their pages. After expiry, fresh large records are written — totalling more than the space freed by TTL, so that - * the pre-fill plus the fresh records exceed the region size. The writes can only succeed because size-aware - * eviction accounts for the TTL-freed pages (as available) and frees further pages for the rest. + * Space freed by TTL cleanup must be available for subsequent writes: after short-TTL entries expire and free + * their pages, putting the same amount of data to a non-expiring cache must succeed without evicting the pre-filled + * small entries — the TTL-freed pages should be reused instead. * * @throws Exception If failed. */ @Test public void testTtlFreedSpaceAccountedForByEviction() throws Exception { + int ttlEntries = 2; + int smallEntries = 6_000; + IgniteEx ignite = startGrid(1); // Non-expiring cache for prefill and the final large put. - IgniteCache plainCache = createCache(ignite, "plain-cache", 0); + IgniteCache plainCache = createCache(ignite, "plain-cache"); // Short-TTL cache for entries that will expire and free pages. - IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL); + IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); - // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries to be - // written without evicting them (so they are guaranteed to be present until they expire). - byte[] small = new byte[SMALL_RECORD_SIZE]; + // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries. + byte[] small = new byte[1024]; - for (int i = 0; i < SMALL_ENTRIES; i++) + for (int i = 0; i < smallEntries; i++) plainCache.put(i, small); - info("Pre-fill done [entries=" + SMALL_ENTRIES + ", plainSize=" + plainCache.size() + - ", loadedPages=" + ignite.dataRegionMetrics(DFLT_DATA_REG_DEFAULT_NAME).getTotalAllocatedPages() + ']'); - - // Add large short-TTL entries that occupy significant space and will expire. They fit in the remaining free - // space, and being the freshest entries they are not evicted while they are being stored. + // Add large short-TTL entries that will expire and free their pages. Object val = new byte[RECORD_SIZE]; - for (int i = 0; i < 2; i++) + for (int i = 0; i < ttlEntries; i++) ttlCache.put(i, val); - // Verify the TTL entries are present before expiry. assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); - // Wait for the TTL worker to expire and free the short-TTL entries. Polled instead of a fixed sleep so that a - // slow CI machine does not proceed before the entries have actually expired. - long expiryDeadline = System.currentTimeMillis() + 30_000; - - while (ttlCache.get(0) != null && System.currentTimeMillis() < expiryDeadline) - U.sleep(200); - - // Verify the TTL entries have expired. - assertNull("TTL entry must be expired", ttlCache.get(0)); - - // Now write large records to the non-expiring cache, totalling more than the space freed by TTL (the prefill - // plus the fresh large records exceed the region size). The put can only succeed because size-aware eviction - // accounts for the pages freed by TTL (as available) and frees further pages for the rest; this verifies the - // synergy between TTL cleanup and size-aware eviction — a row that only fits once expired entries are removed - // is written without OOM. - for (int i = 0; i < FRESH_RECORDS; i++) - plainCache.put(100 + i, val); - - // The most recently written large record is still present: it was accepted without OOM thanks to the space - // freed by TTL and the additional pages freed by size-aware eviction. (Earlier large records may already have - // been evicted to make room for the subsequent ones, so only the freshest is asserted.) - assertNotNull("Fresh large record must be present after TTL-assisted eviction", - plainCache.get(100 + FRESH_RECORDS - 1)); + GridTestUtils.waitForCondition(() -> ttlCache.get(0) == null, 10_000); + + // After expiry, put the same amount of large data to the non-expiring cache. The TTL-freed pages should be + // reused, so to write succeeds without evicting pre-filled small entries. + for (int i = 0; i < ttlEntries; i++) + plainCache.put(smallEntries + i, val); + + for (int i = 0; i < ttlEntries; i++) + assertNotNull("Fresh large record must be present", plainCache.get(smallEntries + i)); + + // Verify that pre-filled small entries were not evicted. + for (int i = 0; i < smallEntries; i++) + assertNotNull("Pre-filled entry " + i + " must not be evicted", plainCache.get(i)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java index 347c724007c3f..22ebaad0f453e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionConcurrentWritesTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java index 8feebf712572a..58f5cab37585f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionSizeAwareTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java index 128549bfa67cb..5563620507507 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/Random2LruPageEvictionWithExpiryPolicyTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_2_LRU}. */ public class Random2LruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java index 8b2fd795e8485..be2dbcea3c91d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionConcurrentWritesTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** Concurrent eviction/insertion test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionConcurrentWritesTest extends PageEvictionConcurrentWritesAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java index eac6af507b879..cb70cf540f825 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionSizeAwareTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** Size-aware page eviction test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionSizeAwareTest extends PageEvictionSizeAwareAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java index 0709c75eafee3..98f5e7bf42a36 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/RandomLruPageEvictionWithExpiryPolicyTest.java @@ -20,8 +20,6 @@ import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; -import static org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionAbstractTest.setEvictionMode; - /** ExpiryPolicy + page eviction synergy test for {@link DataPageEvictionMode#RANDOM_LRU}. */ public class RandomLruPageEvictionWithExpiryPolicyTest extends PageEvictionWithExpiryPolicyAbstractTest { /** {@inheritDoc} */ From fcca5abbae98fae8d017b827a74d6b74dd68db11 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Wed, 23 Sep 2026 00:18:05 +0500 Subject: [PATCH 15/26] reduce comments, cleanup --- .../jmh/cache/JmhPageEvictionBenchmark.java | 81 ++++-------------- .../IgniteCacheDatabaseSharedManager.java | 53 +----------- .../freelist/AbstractFreeList.java | 8 +- .../paged/PageEvictionAbstractTest.java | 2 +- ...eEvictionConcurrentWritesAbstractTest.java | 16 +--- .../paged/PageEvictionGuardOomTest.java | 8 +- ...geEvictionPutLargeObjectsAbstractTest.java | 3 - .../PageEvictionSizeAwareAbstractTest.java | 83 +++++++++++++++---- ...eEvictionWithExpiryPolicyAbstractTest.java | 32 +------ 9 files changed, 95 insertions(+), 191 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java index 29145f07d4ea5..8124aafa96839 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhPageEvictionBenchmark.java @@ -45,19 +45,12 @@ import org.openjdk.jmh.annotations.Warmup; /** - * Measures the impact of size-aware page eviction on an in-memory (non-persistent) data region. + * Measures the impact of size-aware page eviction on an in-memory data region. *

    - * Two benchmark methods: - *

      - *
    • {@link #putSmall()} - puts of small values (well below the empty-pages pool, so the size-aware reserve - * in {@code RowStore.addRow} hits its fast path) within a bounded key range that keeps the region below the - * eviction threshold. This is the hot path whose per-operation cost the patch adds on every put, and is the - * primary A/B metric for detecting a performance regression between the unpatched baseline and this branch.
      • - *
    • {@link #putLarge()} - puts of large values (larger than the empty-pages pool) against a region that has - * been pre-filled to near capacity, so that each large put must actually run the size-aware eviction loop. - * This exercises the new eviction behavior; on an unpatched build such a put fails with an out-of-memory - * error, so this benchmark only runs meaningfully on the patched build.
      • - *
    + * {@link #putSmall()} — small values within a bounded key range that keeps the region below the eviction threshold + * (size-aware reserve fast path). + *

    + * {@link #putLarge()} — large values against a pre-filled region near capacity (size-aware eviction loop). */ @State(Scope.Benchmark) @Fork(1) @@ -70,32 +63,16 @@ public class JmhPageEvictionBenchmark { /** Default cache name. */ private static final String CACHE_NAME = "default"; - /** Empty pages pool size (kept low so that the LARGE scenario reliably exceeds it). */ - private static final int POOL_SIZE = 100; - /** Small value size (bytes): a single data page, far below the empty-pages pool. */ private static final int SMALL_VALUE_SIZE = 1024; /** Large value size (bytes): larger than the empty-pages pool in page terms. */ private static final int LARGE_VALUE_SIZE = 2 * 1024 * 1024; - /** - * Number of pre-fill small entries for the LARGE scenario. Chosen so that the total written data - * (400k x 1 KiB) far exceeds the region capacity: threshold eviction then pins the region at the eviction - * threshold (default ~90% of {@code maxSize}), leaving the free list with only its empty-pages pool. At that - * point a {@link #LARGE_VALUE_SIZE} put cannot take the fast path and must actually run the size-aware eviction - * loop. (A modest pre-fill such as 48k x 1 KiB would leave the region only ~19% full and let every large put - * fit into the headroom via the fast path, so it would never exercise the code under measurement.) - */ + /** Pre-fill entries for the LARGE scenario; exceeds region capacity to pin it at the eviction threshold. */ private static final int PRE_FILL_ENTRIES = 400_000; - /** - * Bounded key range for {@link #putSmall()}. Each {@link #SMALL_VALUE_SIZE} value occupies one data page, so a - * working set of this many resident keys (~32k x 4 KiB ~ 128 MiB) stays comfortably below the eviction - * threshold (~90% of the 256 MiB region). Overwriting within this bounded range (instead of append-style fresh - * keys) keeps the region from filling up and drifting into steady-state threshold eviction during measurement, - * so the run isolates the per-put cost of the size-aware-reserve fast path. - */ + /** Bounded key range for {@link #putSmall()} to keep the region below the eviction threshold. */ private static final int SMALL_KEY_RANGE = 32_000; /** Benchmark scenario: selects the value size and the pre-fill strategy. */ @@ -111,17 +88,10 @@ public class JmhPageEvictionBenchmark { /** Pre-allocated large value (reused to avoid allocation noise in the hot path). */ private final byte[] largeVal = new byte[LARGE_VALUE_SIZE]; - /** Monotonic key source: bounded (mod {@link #SMALL_KEY_RANGE}) for {@link #putSmall()} to keep the region below - * the eviction threshold, and unbounded (append-style) for {@link #putLarge()} to avoid overwriting entries. */ + /** Monotonic key source. */ private final AtomicInteger keyGen = new AtomicInteger(); - /** Page eviction mode used for the data region. */ - @Param("RANDOM_LRU") - private String evictionMode; - - /** Put of a small value (hot path, size-aware reserve takes its fast path). Keys are wrapped within a bounded - * range ({@link #SMALL_KEY_RANGE}) so the resident working set stays below the eviction threshold and the run - * isolates the fast-path cost instead of drifting into steady-state threshold eviction. */ + /** Small value put within a bounded key range (fast path, no eviction). */ @Benchmark public void putSmall() { int key = keyGen.incrementAndGet() % SMALL_KEY_RANGE; @@ -129,15 +99,7 @@ public void putSmall() { cache.put(key, smallVal); } - /** - * Put of a large value against a nearly-full region (runs the size-aware eviction loop). - *

    - * Pinned to a single thread: the size-aware reserve accumulates {@code requiredPages} real empty pages - * in the shared free list before writing, and with multiple concurrent writers those free pages are consumed - * by rivals as fast as they are freed, so no thread ever accumulates enough and the loop exhausts its - * no-progress budget into an out-of-memory. At one thread the free-page count grows monotonically and the - * reserve completes, measuring the honest per-put cost of eviction. - */ + /** Large value put against a nearly-full region (size-aware eviction loop). Single-threaded to avoid OOM. */ @Benchmark @Threads(1) public void putLarge() { @@ -146,17 +108,14 @@ public void putLarge() { cache.put(key, largeVal); } - /** Starts Ignite with an in-memory, eviction-enabled data region and pre-fills it for the LARGE scenario. */ + /** */ @Setup(Level.Trial) public void setup() { - long regionSize = 256 * 1024L * 1024L; - DataStorageConfiguration dsCfg = new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setPersistenceEnabled(false) - .setMaxSize(regionSize) - .setEmptyPagesPoolSize(POOL_SIZE) - .setPageEvictionMode(DataPageEvictionMode.valueOf(evictionMode))); + .setMaxSize(256 * 1024L * 1024L) + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)); IgniteConfiguration cfg = new IgniteConfiguration() .setIgniteInstanceName("test") @@ -167,8 +126,6 @@ public void setup() { cache = ignite.getOrCreateCache(new CacheConfiguration(CACHE_NAME).setBackups(0)); - // Pre-fill the region with small entries for the LARGE scenario until threshold eviction pins it at the - // eviction threshold, so that a large put has no headroom to grow into and must actually evict. if ("LARGE".equalsIgnoreCase(scenario)) { try (IgniteDataStreamer ldr = ignite.dataStreamer(CACHE_NAME)) { ldr.perNodeBufferSize(1024); @@ -177,26 +134,18 @@ public void setup() { ldr.addData(i, smallVal); } - // The pre-fill consumed keys [0, PRE_FILL_ENTRIES). Start large puts after that range so they write - // brand-new keys (true append), leaving the pre-filled small entries in place to be the eviction - // candidates, instead of overwriting them in place. keyGen.set(PRE_FILL_ENTRIES); } } - /** @return Test data. */ - @Override public String toString() { - return "JmhPageEvictionBenchmark[scenario=" + scenario + ", evictionMode=" + evictionMode + ']'; - } - - /** Stops all Ignite instances started by this benchmark. */ + /** */ @TearDown public void tearDown() { Ignition.stopAll(true); } /** - * Runs this benchmark over both {@code SMALL} and {@code LARGE} scenarios (configured by {@code @Param}). + * Runs benchmark. * * @param args Ignored. * @throws Exception If failed. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 542cd2e9a746a..e051883bcce0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -148,22 +148,13 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without either an eviction or a - * new highest empty-pages count before failing with an out-of-memory. Being time-based (measured from the last - * progress) rather than a fixed attempt count means a slow-but-progressing eviction is never torn down, while a - * genuinely stuck eviction (nothing evictable, or contenders that never release their locks) still terminates in - * bounded time instead of busy-spinning forever. The overall cycle is additionally capped by - * {@link #EVICTION_MAX_CYCLE_TIME_MILLIS}. + * new highest empty-pages count before failing with an out-of-memory. */ private static final long EVICTION_NO_PROGRESS_TIMEOUT_MILLIS = 1_000L; /** * Hard upper bound (milliseconds) on the total duration of one size-aware eviction cycle, regardless of - * per-iteration progress. {@link #EVICTION_NO_PROGRESS_TIMEOUT_MILLIS} bounds the period with no progress, but - * eviction that keeps making partial progress (e.g. evicting a contended page's available entries one-by-one - * without ever emptying a page, so the empty-pages count never reaches the target) could otherwise extend the - * loop indefinitely; this absolute deadline guarantees the cycle still fails with an out-of-memory in bounded - * time while the per-progress timeout above allows a genuinely progressing eviction to run to its natural end. - * It also caps the worst-case time the reserve can hold the caller's entry lock. + * per-iteration progress. */ private static final long EVICTION_MAX_CYCLE_TIME_MILLIS = 5 * EVICTION_NO_PROGRESS_TIMEOUT_MILLIS; @@ -1218,12 +1209,6 @@ public WALPointer latestWalPointerReservedForPreloading() { * The size-aware reserve is required because page eviction by itself only keeps a steady-state pool of empty pages * ({@link DataRegionConfiguration#getEmptyPagesPoolSize()}) and does not guarantee enough space for a single row * larger than this pool. - *

    - * Worst case: when called while the entry being written is locked (single-row insertion), the eviction loop can - * hold that lock for up to {@link #EVICTION_MAX_CYCLE_TIME_MILLIS} — only if eviction cannot consolidate enough - * empty pages within that time (e.g. a long-running transaction holding all evictable entries, or only partial - * progress), after which the call fails with {@link IgniteOutOfMemoryException} (reported as a critical failure - * to the configured failure handler). * * @param region Data region to be checked. * @param dataRowSize Size of data row to be inserted. @@ -1296,12 +1281,8 @@ private void ensureFreeSpaceForEviction( ) throws IgniteOutOfMemoryException, IgniteCheckedException { PageMemory pageMem = region.pageMemory(); - // Maximum payload bytes that a single data page can hold for a fragmented row. long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - // A row that fits into the steady-state empty-pages pool is satisfied by normal threshold eviction, so the - // fast path is a single comparison (no page computation, free-list lookup or page-memory reads on the hot - // small-put path). if (dataRowSize <= regCfg.getEmptyPagesPoolSize() * pagePayload) return; @@ -1312,44 +1293,20 @@ private void ensureFreeSpaceForEviction( long totalPages = regCfg.getMaxSize() / pageMem.systemPageSize(); - // Pages the row will actually occupy once written, and which the free list must hand out on demand during - // the fragmented write. long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; - // The row fundamentally cannot fit into the whole region. if (requiredPages > totalPages) throw outOfMemory(regCfg); - // The reserve must guarantee `requiredPages` REAL empty pages, not just apparent headroom. Both are shared and - // non-exclusive (emptyDataPages() is a snapshot; any writer can consume them), but once the region is full - // (loadedPages == totalPages) headroom can no longer grow it (fresh allocateDataPage -> raw OOM), while empty - // pages in the reuse bucket stay reachable via takePage(). So empty pages are the only resource the fragmented - // write can consume on a full region. The TOCTOU between this reserve and the actual write is closed by the - // lazy re-reserve in AbstractFreeList#writeSinglePage. long emptyPages = freeList.emptyDataPages(); - // The gate reuses evictionThreshold as a regime boundary, not as "when to start eviction" (evictionRequired() - // does that, stopping on emptyPages >= poolSize; no last (1 - evictionThreshold) of page memory is left - // unusable). Below the threshold the region has real slack, so a row fitting into the combined spare space is - // satisfied without eviction (live, e.g. short-TTL, entries are not evicted just to accumulate empty pages). - // At/above it headroom is no longer trustworthy (concurrent writers could commit the same headroom - TOCTOU), - // so only real empty pages are counted and eviction is driven below. boolean evictionRegime = pageMem.loadedPages() >= (long)(totalPages * regCfg.getEvictionThreshold()); - // Fast path: skip eviction when (a) enough real empty pages exist, or (b) below the regime with enough spare - // space to grow into. This is a snapshot and only necessary, not sufficient: under contention two writers can - // both pass and consume the same pages - recovered by the lazy re-reserve in AbstractFreeList#writeSinglePage. if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + (totalPages - pageMem.loadedPages()) >= requiredPages)) return; PageEvictionTracker evictionTracker = region.evictionTracker(); - // Evict until the free list holds enough real empty pages. Progress counts as either an actual eviction this - // iteration (evictDataPage returned true) or a new high-water empty-pages count. The count alone is unreliable - // under concurrency: writers consume the shared counter as fast as eviction frees pages, so a thread may never - // see its high-water exceeded while eviction still makes real global progress - tearing down there would be a - // false OOM. Counting the actual eviction prevents that; when eviction genuinely cannot proceed the method - // returns false, so the loop still fails with OOM in bounded time. long bestEmptyPages = emptyPages; long lastProgressNanos = System.nanoTime(); @@ -1364,16 +1321,12 @@ private void ensureFreeSpaceForEviction( " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); } - // tryLock=true: skip contended/self-held entries (the reserve can run while the current thread already - // holds entry locks on the single-row path), avoiding a lock-ordering deadlock. boolean evicted = evictionTracker.evictDataPage(true); region.metrics().updateEvictionRate(); long curEmptyPages = freeList.emptyDataPages(); - // A new high-water count or an actual eviction re-arms the no-progress timeout; a stalled iteration backs - // off rather than busy-spinning. if (evicted || curEmptyPages > bestEmptyPages) { if (curEmptyPages > bestEmptyPages) bestEmptyPages = curEmptyPages; @@ -1388,8 +1341,6 @@ private void ensureFreeSpaceForEviction( backoffNanos = Math.min(backoffNanos << 1, EVICTION_BACKOFF_MAX_NANOS); } - // OOM when there was no eviction/count growth for EVICTION_NO_PROGRESS_TIMEOUT_MILLIS (stuck eviction) or - // the whole cycle exceeds EVICTION_MAX_CYCLE_TIME_MILLIS (only partial progress, target never reached). long nowNanos = System.nanoTime(); long maxNoProgress = TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 91f209d559b7a..89442c4b0117c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -766,11 +766,9 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC *

    * The page is acquired via {@link #takePageWithReserve}, which handles the size-aware re-reserve and TOCTOU * recovery (see its javadoc for details). Reached from the BPlusTree.invoke row-creation closure, this re-reserve - * is an inline demand-eviction that runs without any BPlusTree page locks held by the row-creation closure (the - * search releases the read lock before the closure; the leaf write lock is taken only afterwards — see - * BPlusTree.invokeDown). The entry-level tryLock only skips contended/self-held entries and never blocks. - * The TTL expiration worker is also safe: it removes entries from the data tree and pending tree in separate - * BPlusTree operations, each releasing all page locks before returning, so no cross-tree nested locks occur. + * is an inline demand-eviction that runs while the BPlusTree leaf page read lock is held (the closure is invoked + * inside {@code read} before the write lock is taken). The entry-level tryLock only skips contended/self-held + * entries and never blocks, avoiding a lock-ordering deadlock with the entry lock already held by the caller. * * @param row Row to write. * @param written Written size. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java index d48c954876a5a..f68ae83f22c39 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.internal.processors.cache.eviction.paged; +import javax.cache.expiry.Duration; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; @@ -30,7 +31,6 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; -import javax.cache.expiry.Duration; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static javax.cache.expiry.CreatedExpiryPolicy.factoryOf; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index 053e5cc8024e9..f474983eeb023 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -29,16 +29,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -/** - * Concurrent deadlock test for size-aware page eviction. - *

    - * The region is first filled with a large number of small entries (so there is plenty of evictable page space), then - * several threads concurrently insert large rows (larger than the empty-pages pool). Each large insert goes through - * the size-aware reserve and, for the single-row path, eviction under the new entry lock with the non-blocking - * {@code tryLockEntry}. The average data volume is kept within the region capacity, so eviction frees already-stored - * small entries rather than overrunning the free list. The test asserts that no deadlock occurs (all threads finish - * within a global deadline). - */ +/** Concurrent deadlock test for size-aware page eviction. */ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvictionAbstractTest { /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -59,8 +50,7 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict @Test public void testConcurrentLargeWritesNoDeadlock() throws Exception { // Number of small pre-fill entries, leaving a buffer that is exceeded by the total of the large writes, so that - // the last of them can only be stored by freeing pages via size-aware eviction. The large records are small - // enough that concurrent size-aware eviction reliably frees the required pages (no spurious guard OOM). + // the last of them can only be stored by freeing pages via size-aware eviction. int smallEntries = 48_000; // Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the @@ -88,7 +78,7 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { for (int k = 0; k < largeRowsPerThread; k++) cache.put(smallEntries + idx * largeRowsPerThread + k, largeVal); - }, 10, "paged-writer"); + }, 10, "paged-writer"); startLatch.countDown(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 019f07bfca8b3..10216bd2b0b02 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -33,13 +33,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -/** - * Negative test for the size-aware eviction progress guard. - *

    - * When every resident entry is locked by another thread/transaction, page eviction cannot free any page: the guarded - * {@code tryLockEntry(0)} in {@code evictInternal} fails for every candidate, so {@code ensureFreeSpaceForEviction} - * makes no progress and must fail with an {@code IgniteOutOfMemoryException} within bounded time instead of deadlock. - */ +/** Negative test for the size-aware eviction progress guard. */ public class PageEvictionGuardOomTest extends PageEvictionAbstractTest { /** Off-heap region size. */ private static final int SIZE = 12 * 1024 * 1024; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java index 4ece99be95f6e..c2d3a7a378909 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPutLargeObjectsAbstractTest.java @@ -70,9 +70,6 @@ public void testPutLargeObjects() throws Exception { for (Integer key : primaryKeys(grid(1).cache(DEFAULT_CACHE_NAME), ENTRIES)) cache.put(key, val); - // With size-aware eviction the large records do not fail with OOM: older records are evicted to - // make room for the newer ones. The resident set must therefore be bounded well below the total written - // (50 x 80MB >> 1GB region) but stay non-empty (at least the most recently written entries survive). assertTrue(cache.size() > 0 && cache.size() < ENTRIES); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 4a33b342a5c15..a2af4865e1c0b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -26,17 +26,11 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; +import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -/** - * Tests size-aware page eviction on in-memory (non-persistent) data regions. - *

    - * Verifies that a row larger than the configured {@code emptyPagesPoolSize} (in pages) is still written successfully - * when page eviction is enabled, by evicting old entries to free enough space. Also verifies that a row - * which fundamentally cannot fit into the region fails with OOM instead of hanging in an infinite eviction loop. - * The batch path ({@code putAll} of large rows) and the update path (growing a row) are covered as well. - */ +/** Tests size-aware page eviction on in-memory (non-persistent) data regions. */ public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbstractTest { /** Off-heap region size (large enough to hold cache structural pages with the configured partition count). */ private static final int SIZE = 128 * 1024 * 1024; @@ -75,8 +69,9 @@ public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbst /** * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. - * A batch {@code putAll} of records whose total size exceeds the region must also fail with OOM (exercises the - * batch store path {@code RowStore.addRows} → {@code ensureFreeSpaceForInsert}). + * A batch {@code putAll} of records whose total size equals the region capacity must also fail with OOM because + * structural pages (free list, index tree) leave no room for the data (exercises the batch store path + * {@code RowStore.addRows} → {@code ensureFreeSpaceForInsert}). * * @throws Exception If failed. */ @@ -176,19 +171,22 @@ public void testUpdateRowGrows() throws Exception { } /** - * Verifies the {@code !evictionRegime} fast path in {@code ensureFreeSpaceForEviction}: when the region is below - * the eviction threshold, a large row (exceeding the empty-pages pool) that fits in the remaining headroom must be - * written successfully — the size-aware reserve trusts headroom and does not trigger size-aware eviction. + * Verifies the {@code evictionRegime} gate in {@code ensureFreeSpaceForEviction}: when the region is below the + * eviction threshold, a large row (exceeding the empty-pages pool) that fits in the remaining headroom is written + * successfully without triggering eviction — the size-aware reserve trusts headroom + * ({@code evictionRegime = false}) and skips the eviction loop. If the gate were broken (always {@code true}), + * size-aware eviction would run unnecessarily, evicting pre-filled entries and setting the evictions-started flag. * * @throws Exception If failed. */ @Test - public void testGrowBeyondEvictionThreshold() throws Exception { + public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception { IgniteEx ignite = startGrid(1); IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); // Pre-fill to ~80 MiB — below the 0.9 * 128 MiB = 115.2 MiB threshold, leaving ~48 MiB of headroom. + // After pre-fill, loadedPages is well below threshold, so evictionRegime = false. byte[] small = new byte[SMALL_RECORD_SIZE]; int preFill = 20_000; @@ -196,17 +194,68 @@ public void testGrowBeyondEvictionThreshold() throws Exception { for (int i = 0; i < preFill; i++) cache.put(i, small); - // Write a 32 MiB row that fits in the remaining headroom. The size-aware reserve sees loadedPages below the - // threshold, trusts headroom (evictionRegime = false), and skips size-aware eviction. + DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); + + assertFalse("Eviction must not have started during pre-fill below threshold", metrics.isEvictionsStarted()); + + // Write a 32 MiB row that fits in the remaining headroom (~48 MiB). Total ~112 MiB stays below the 115.2 MiB + // threshold, so evictionRegime remains false throughout and no eviction is triggered. byte[] val = new byte[RECORD_SIZE]; Arrays.fill(val, (byte)1); cache.put(preFill, val); + // Eviction must not have started: the region never crossed the threshold, so the size-aware reserve + // trusted headroom and the normal threshold eviction in insertDataRows never fired. + assertFalse("Eviction must not start when the region stays below the threshold", metrics.isEvictionsStarted()); + + for (int i = 0; i < preFill; i++) + assertNotNull("Pre-filled entry " + i + " must not be evicted below threshold", cache.get(i)); + byte[] read = (byte[])cache.get(preFill); - assertNotNull("Large row must be readable after writing below threshold", read); + assertNotNull("Large row must be readable", read); + assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); + } + + /** + * Verifies that a large row (exceeding the empty-pages pool) is written successfully when the region is already + * at or above the eviction threshold — the size-aware eviction loop ({@code evictionRegime = true}) evicts enough + * pre-filled entries to free the required pages, and the write succeeds. + * + * @throws Exception If failed. + */ + @Test + public void testLargeRowAboveThresholdEvictsAndSucceeds() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + + // Pre-fill to near capacity with small evictable entries. 28 000 x 4 KiB ~ 112 MiB of data; with page + // overhead loadedPages is at or above the 0.9 threshold, so evictionRegime = true. + byte[] small = new byte[SMALL_RECORD_SIZE]; + + for (int i = 0; i < SMALL_ENTRIES; i++) + cache.put(i, small); + + // Write a 32 MiB row that exceeds the empty-pages pool. Since the region is at or above the eviction + // threshold, the size-aware reserve cannot trust headroom and must actually evict pre-filled entries. + byte[] val = new byte[RECORD_SIZE]; + + Arrays.fill(val, (byte)1); + + int largeKey = SMALL_ENTRIES + 1; + + cache.put(largeKey, val); + + DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); + + assertTrue("Eviction must have started after a large put near capacity", metrics.isEvictionsStarted()); + + byte[] read = (byte[])cache.get(largeKey); + + assertNotNull("Large row must be readable after eviction", read); assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 8462f55d0d2be..8ad8dde73c096 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -25,12 +25,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -/** - * Tests the synergy between ExpiryPolicy (TTL cleanup) and size-aware page eviction on an in-memory data region. - * Verifies that concurrent TTL cleanup and eviction do not deadlock, that a large row larger than the - * empty-pages pool is still written when eviction is enabled, and that TTL-freed space is accounted for by eviction - * (a row that only fits after expired entries are removed is still written without OOM). - */ +/** Tests the synergy between ExpiryPolicy (TTL cleanup) and size-aware page eviction on an in-memory data region. */ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvictionAbstractTest { /** Off-heap region size. */ private static final int SIZE = 128 * 1024 * 1024; @@ -42,11 +37,7 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvict */ private static final int RECORD_SIZE = 32 * 1024 * 1024; - /** - * Short TTL applied to some entries. Kept short enough that entries expire while the eviction loop is still - * active (not after all entries have already been evicted), so the TTL worker genuinely runs concurrently with - * size-aware eviction. - */ + /** Short TTL applied to some entries. */ private static final long TTL = 2000; /** {@inheritDoc} */ @@ -73,27 +64,19 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvict public void testLargePutWithExpiryNoDeadlock() throws Exception { IgniteEx ignite = startGrid(1); - // Short-TTL entries keep the TTL worker actively freeing pages while eviction runs. IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, TTL, false); Object val = new byte[RECORD_SIZE]; - // Writing more data than the region can hold forces eviction; concurrent expiry of short-TTL entries must not - // deadlock with it. The entry count and TTL are chosen so that early entries expire while the eviction loop is - // still active for later entries, giving the TTL worker a real chance to run concurrently with eviction. for (int i = 0; i < 60; i++) cache.put(i, val); - // The most recently written entry cannot have expired yet (TTL is far larger than this read), so a non-null - // read both verifies the cache is responsive after concurrent expiry/eviction (the test's goal) and is not - // racy. Reading the first written key would be racy (it may already have expired under the TTL). assertNotNull("Cache must remain responsive after concurrent expiry and eviction", cache.get(59)); } /** - * Space freed by TTL cleanup must be available for subsequent writes: after short-TTL entries expire and free - * their pages, putting the same amount of data to a non-expiring cache must succeed without evicting the pre-filled - * small entries — the TTL-freed pages should be reused instead. + * After short-TTL entries expire and free their pages, putting the same amount of data to a non-expiring cache + * must succeed, and the pre-filled small entries must remain intact (not evicted). * * @throws Exception If failed. */ @@ -104,10 +87,7 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { IgniteEx ignite = startGrid(1); - // Non-expiring cache for prefill and the final large put. IgniteCache plainCache = createCache(ignite, "plain-cache"); - - // Short-TTL cache for entries that will expire and free pages. IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries. @@ -116,7 +96,6 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { for (int i = 0; i < smallEntries; i++) plainCache.put(i, small); - // Add large short-TTL entries that will expire and free their pages. Object val = new byte[RECORD_SIZE]; for (int i = 0; i < ttlEntries; i++) @@ -126,15 +105,12 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { GridTestUtils.waitForCondition(() -> ttlCache.get(0) == null, 10_000); - // After expiry, put the same amount of large data to the non-expiring cache. The TTL-freed pages should be - // reused, so to write succeeds without evicting pre-filled small entries. for (int i = 0; i < ttlEntries; i++) plainCache.put(smallEntries + i, val); for (int i = 0; i < ttlEntries; i++) assertNotNull("Fresh large record must be present", plainCache.get(smallEntries + i)); - // Verify that pre-filled small entries were not evicted. for (int i = 0; i < smallEntries; i++) assertNotNull("Pre-filled entry " + i + " must not be evicted", plainCache.get(i)); } From 4befc1af3b82793bf18d32575ce580d9d555c5cc Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 24 Sep 2026 11:28:17 +0500 Subject: [PATCH 16/26] small fix --- .../paged/PageEvictionWithExpiryPolicyAbstractTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index 8ad8dde73c096..edf1902c3f465 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -103,7 +103,7 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); - GridTestUtils.waitForCondition(() -> ttlCache.get(0) == null, 10_000); + assertTrue(GridTestUtils.waitForCondition(() -> ttlCache.get(0) == null, 10_000)); for (int i = 0; i < ttlEntries; i++) plainCache.put(smallEntries + i, val); From a8145279872d967123336c8a49dff1fdfbf78167 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 25 Sep 2026 14:55:28 +0500 Subject: [PATCH 17/26] fix review comments --- .../pagemem/impl/PageMemoryNoStoreImpl.java | 2 +- .../IgniteCacheDatabaseSharedManager.java | 42 ++-- .../evict/Random2LruPageEvictionTracker.java | 6 +- .../evict/RandomLruPageEvictionTracker.java | 6 +- .../freelist/AbstractFreeList.java | 37 +++- .../paged/PageEvictionAbstractTest.java | 27 +++ ...eEvictionConcurrentWritesAbstractTest.java | 30 ++- .../paged/PageEvictionGuardOomTest.java | 35 +--- .../paged/PageEvictionMetricTest.java | 7 +- .../PageEvictionSizeAwareAbstractTest.java | 193 +++++++++++------- ...eEvictionWithExpiryPolicyAbstractTest.java | 97 +++++---- 11 files changed, 296 insertions(+), 186 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index 08507d4a82ce5..b2a7bae5e296c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -109,7 +109,7 @@ public class PageMemoryNoStoreImpl implements PageMemory { private static final int SEG_BITS = 4; /** Number of bits required to store segment index. */ - private static final int SEG_CNT = (1 << SEG_BITS); + public static final int SEG_CNT = (1 << SEG_BITS); /** Number of bits left to store page index. */ private static final int IDX_BITS = PageIdUtils.PAGE_IDX_SIZE - SEG_BITS; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index e051883bcce0c..b07a552cb76a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -107,6 +107,7 @@ import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_SUB_INTERVALS; import static org.apache.ignite.configuration.DataStorageConfiguration.HALF_MAX_WAL_ARCHIVE_SIZE; import static org.apache.ignite.configuration.DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE; +import static org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.SEG_CNT; import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.METASTORE_DATA_REGION_NAME; import static org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor.VOLATILE_DATA_REGION_NAME; @@ -1260,12 +1261,11 @@ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg } /** - * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the free list holds - * enough real empty pages to accommodate the row, or throws {@link IgniteOutOfMemoryException} if the goal is - * unreachable / no progress can be made. Progress is measured against the number of empty pages in the free list - * (the only resource a subsequent fragmented write can reliably consume once the region is effectively full); the - * region's spare capacity (headroom) is only trusted in the fast path while the region is below the eviction - * threshold. + * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the combined space + * (empty pages in the free list + unallocated headroom) is enough to accommodate the row, or throws + * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. Both empty pages and + * headroom are treated as concurrent resources: the fast path returns as soon as their sum covers the row, and + * the eviction loop only evicts the deficit (pages that neither headroom nor the free list can provide). * * @param region Data region. * @param regCfg Data region configuration. @@ -1283,9 +1283,6 @@ private void ensureFreeSpaceForEviction( long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - if (dataRowSize <= regCfg.getEmptyPagesPoolSize() * pagePayload) - return; - CacheFreeList freeList = freeListMap.get(regCfg.getName()); if (freeList == null) @@ -1300,9 +1297,9 @@ private void ensureFreeSpaceForEviction( long emptyPages = freeList.emptyDataPages(); - boolean evictionRegime = pageMem.loadedPages() >= (long)(totalPages * regCfg.getEvictionThreshold()); - - if (emptyPages >= requiredPages || (!evictionRegime && emptyPages + (totalPages - pageMem.loadedPages()) >= requiredPages)) + // Effective headroom accounts for per-segment allocation overhead (up to one page per segment), + // so the fast path does not trust headroom that cannot actually be allocated. + if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= requiredPages) return; PageEvictionTracker evictionTracker = region.evictionTracker(); @@ -1315,7 +1312,7 @@ private void ensureFreeSpaceForEviction( long backoffNanos = EVICTION_BACKOFF_START_NANOS; - while (freeList.emptyDataPages() < requiredPages) { + while (freeList.emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < requiredPages) { if (region.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); @@ -1350,6 +1347,20 @@ private void ensureFreeSpaceForEviction( } } + /** + * Computes the effective headroom (unallocated pages that can actually be allocated), accounting for per-segment + * allocation overhead (up to one page per segment). + * + * @param totalPages Total pages in the region (maxSize / systemPageSize). + * @param loadedPages Currently loaded pages. + * @return Effective headroom, never negative. + */ + private static long effectiveHeadroom(long totalPages, long loadedPages) { + long effectiveTotal = totalPages > SEG_CNT ? totalPages - SEG_CNT : 0; + + return Math.max(0, effectiveTotal - loadedPages); + } + /** * @param regCfg Data region configuration. * @return New {@link IgniteOutOfMemoryException} (also reported as a critical failure) for the given region. @@ -1380,6 +1391,11 @@ public void ensureFreeSpace(DataRegion memPlc) throws IgniteCheckedException { if (memPlc == null) return; + long totalPages = memPlc.config().getMaxSize() / memPlc.pageMemory().systemPageSize(); + + if (effectiveHeadroom(totalPages, memPlc.pageMemory().loadedPages()) > memPlc.config().getEmptyPagesPoolSize()) + return; + while (memPlc.evictionTracker().evictionRequired()) { if (memPlc.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index ac361fcb7b10d..5511cbad2ddf6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -161,7 +161,11 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); - assert firstTs >= 0 : "[firstTs=" + firstTs + ", secondTs=" + second(trackingData) + "]"; + // Under concurrent writes a fragment chain may be partially built (a concurrent writer + // has not yet linked the head page), so after two hops the timestamp can still be + // negative. Skip this sample and try another page. + if (firstTs < 0) + continue; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index ff94ac714b3e9..08530ab56f94e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -141,7 +141,11 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); - assert compactTs >= 0 : "[compactTs=" + compactTs + "]"; + // Under concurrent writes a fragment chain may be partially built (a concurrent writer + // has not yet linked the head page), so after two hops the timestamp can still be + // negative. Skip this sample and try another page. + if (compactTs < 0) + continue; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 89442c4b0117c..d0239c189df17 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -55,6 +55,7 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.SEG_CNT; /** */ @@ -601,14 +602,31 @@ private int bucket(int freeSpace, boolean allowReuse) { } /** - * @param part Partition. + * @param row Row to write. + * @param written Written size. + * @param statHolder Statistics holder to track IO operations. * @return Page ID. * @throws IgniteCheckedException If failed. */ - private long allocateDataPage(int part) throws IgniteCheckedException { - assert part <= PageIdAllocator.MAX_PARTITION_ID; + private long allocateDataPage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { + assert row.partition() <= PageIdAllocator.MAX_PARTITION_ID; + + long pageId; + + try { + pageId = pageMem.allocatePage(grpId, row.partition(), FLAG_DATA); + } + catch (IgniteOutOfMemoryException oom) { + // Headroom was consumed after the pre-reserve trusted it. Evict to create empty pages and retry. + dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); + + pageId = takePage(row.size() - written, row, statHolder); + + if (pageId == 0L) + throw oom; + } - return pageMem.allocatePage(grpId, part, FLAG_DATA); + return pageId; } /** @@ -621,8 +639,8 @@ private boolean regionEffectivelyFull() { // Each of up to 16 segments loses up to one page to allocation overhead (lastAllocatedIdxPtr + alignment), // so the theoretical max is never reached in practice. Subtract the worst-case segment loss to get an // effective limit that reflects real product scenarios. - if (maxPages > 16) - maxPages -= 16; + if (maxPages > SEG_CNT) + maxPages -= SEG_CNT; return pageMem.loadedPages() >= maxPages; } @@ -690,8 +708,7 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. */ - @Override public void insertDataRows(Collection rows, - IoStatisticsHolder statHolder) throws IgniteCheckedException { + @Override public void insertDataRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { try { GridCursor cur = new GridCursorIteratorWrapper<>(rows.iterator()); @@ -718,7 +735,7 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { - pageId = allocateDataPage(row.partition()); + pageId = allocateDataPage(row, written, statHolder); initIo = row.ioVersions().latest(); } @@ -782,7 +799,7 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { - pageId = allocateDataPage(row.partition()); + pageId = allocateDataPage(row, written, statHolder); initIo = row.ioVersions().latest(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java index f68ae83f22c39..3541aa97b63cd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java @@ -18,6 +18,7 @@ import javax.cache.expiry.Duration; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; @@ -29,6 +30,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; @@ -164,4 +166,29 @@ protected static IgniteCache createCache(IgniteEx ignite, Strin return ignite.createCache(ccfg); } + + /** @return Default data region. */ + protected static DataRegion defaultRegion(IgniteEx ignite) throws IgniteCheckedException { + return ignite.context().cache().context().database().dataRegion(null); + } + + /** @return The maximum default data region size in bytes. */ + protected static long regionMaxSize(IgniteEx ignite) throws IgniteCheckedException { + return defaultRegion(ignite).config().getMaxSize(); + } + + /** @return The page size in bytes for the default data region. */ + protected static int pageSize(IgniteEx ignite) throws IgniteCheckedException { + return defaultRegion(ignite).pageMemory().pageSize(); + } + + /** @return The total number of pages in the default data region. */ + protected static long totalPages(IgniteEx ignite) throws IgniteCheckedException { + return regionMaxSize(ignite) / defaultRegion(ignite).pageMemory().systemPageSize(); + } + + /** @return {@code true} if page eviction has started in the default data region. */ + protected static boolean isEvictionsStarted(IgniteEx ignite) throws IgniteCheckedException { + return defaultRegion(ignite).metrics().isEvictionsStarted(); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index f474983eeb023..fbc064ff176bc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -21,8 +21,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.typedef.internal.U; @@ -31,11 +29,6 @@ /** Concurrent deadlock test for size-aware page eviction. */ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvictionAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - return super.getConfiguration(gridName).setDataStorageConfiguration(new DataStorageConfiguration()); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); @@ -49,28 +42,29 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict */ @Test public void testConcurrentLargeWritesNoDeadlock() throws Exception { - // Number of small pre-fill entries, leaving a buffer that is exceeded by the total of the large writes, so that - // the last of them can only be stored by freeing pages via size-aware eviction. - int smallEntries = 48_000; - - // Large rows inserted per thread. Their total (threads x rows) exceeds the buffer left by the pre-fill, so the - // last large writes overflow the region and require size-aware eviction to free small entry pages. + int smallEntries = 1_000; int largeRowsPerThread = 20; IgniteEx ignite = startGrid(1); IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill the region with many small entries so that eviction always has evictable pages to free. + long regionMax = regionMaxSize(ignite); + + // Small value: sized so that 'smallEntries' of them fill ~80% of the region, staying under the 90% eviction threshold. + byte[] smallVal = new byte[(int)(regionMax * 0.8 / smallEntries)]; + for (int i = 0; i < smallEntries; i++) - cache.put(i, new byte[4096]); + cache.put(i, smallVal); - byte[] largeVal = new byte[2 * 1024 * 1024]; + assertFalse("Eviction must not have started during pre-fill", isEvictionsStarted(ignite)); CountDownLatch startLatch = new CountDownLatch(1); - AtomicInteger threadIdx = new AtomicInteger(); + // Large value: 5% of the region — large enough to trigger eviction during concurrent writes. + byte[] largeVal = new byte[(int)(regionMax / 20)]; + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { U.awaitQuiet(startLatch); @@ -83,5 +77,7 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { startLatch.countDown(); fut.get(TimeUnit.MINUTES.toMillis(3)); + + assertTrue("Eviction must have started during concurrent large writes", isEvictionsStarted(ignite)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 10216bd2b0b02..4bb8d20842685 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -24,8 +24,6 @@ import java.util.concurrent.locks.Lock; import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.DataPageEvictionMode; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; @@ -35,25 +33,9 @@ /** Negative test for the size-aware eviction progress guard. */ public class PageEvictionGuardOomTest extends PageEvictionAbstractTest { - /** Off-heap region size. */ - private static final int SIZE = 12 * 1024 * 1024; - - /** - * Number of resident entries (each ~one page) filling the region to ~55% of its capacity. This keeps the region - * comfortably below the eviction threshold (so the ordinary threshold-based {@code ensureFreeSpace} path is a - * no-op) while leaving less free space than a single large record needs, so the size-aware eviction guard is - * exercised. - */ - private static final int FILL_ENTRIES = 1_600; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - return super.getConfiguration(gridName) - .setDataStorageConfiguration(new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setInitialSize(SIZE) - .setMaxSize(SIZE) - .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU))); + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); } /** {@inheritDoc} */ @@ -73,15 +55,17 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, 0, true); - // Pre-fill the region so that less than one large record of free space remains, without overflowing it. - byte[] fillVal = new byte[3_800]; + // Fill to ~85% of total pages, below the 0.9 eviction threshold. Each entry is ~one page. + int fillEntries = (int)(totalPages(ignite) * 0.85); + + byte[] fillVal = new byte[pageSize(ignite) - 200]; - for (int i = 1; i <= FILL_ENTRIES; i++) + for (int i = 1; i <= fillEntries; i++) cache.put(i, fillVal); - Collection keys = new ArrayList<>(FILL_ENTRIES); + Collection keys = new ArrayList<>(fillEntries); - for (int i = 1; i <= FILL_ENTRIES; i++) + for (int i = 1; i <= fillEntries; i++) keys.add(i); CountDownLatch ready = new CountDownLatch(1); @@ -107,8 +91,9 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { try { assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); + // Large row that needs more free pages than remain — eviction cannot free any page (all locked) → OOM. GridTestUtils.assertThrowsWithCause( - () -> cache.put(FILL_ENTRIES + 1, new byte[8 * 1024 * 1024]), + () -> cache.put(fillEntries + 1, new byte[4 * 1024 * 1024]), IgniteOutOfMemoryException.class ); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java index 6efd5b6bffbd0..1e637f81985da 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java @@ -50,7 +50,7 @@ public void testPageEvictionMetric() throws Exception { } /** - * Regression: ordinary small records that keep the region below the eviction threshold must not trigger page + * Regression test: ordinary small records that keep the region below the eviction threshold must not trigger page * eviction at all (eviction is not started, eviction rate stays zero). * * @throws Exception If failed. @@ -59,7 +59,7 @@ public void testPageEvictionMetric() throws Exception { public void testNoEvictionBelowThreshold() throws Exception { IgniteEx ignite = startGrid(0); - DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); + DataRegionMetricsImpl metrics = defaultRegion(ignite).metrics(); metrics.enableMetrics(); @@ -85,8 +85,7 @@ public void testNoEvictionBelowThreshold() throws Exception { private void checkPageEvictionMetric(CacheAtomicityMode atomicityMode) throws Exception { IgniteEx ignite = startGrid(0); - DataRegionMetricsImpl metrics = - ignite.context().cache().context().database().dataRegion(null).metrics(); + DataRegionMetricsImpl metrics = defaultRegion(ignite).metrics(); metrics.enableMetrics(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index a2af4865e1c0b..18116339c908f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -21,47 +21,14 @@ import java.util.HashMap; import java.util.Map; import org.apache.ignite.IgniteCache; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; -import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; /** Tests size-aware page eviction on in-memory (non-persistent) data regions. */ public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbstractTest { - /** Off-heap region size (large enough to hold cache structural pages with the configured partition count). */ - private static final int SIZE = 128 * 1024 * 1024; - - /** - * Record size: chosen so that a single row requires more pages than are left free when the region is kept at the - * eviction threshold ({@code (1 - threshold) * totalPages}). This guarantees a large put cannot take the fast - * path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve - * ({@code ensureFreeSpaceForEviction}), which is the scenario these tests are meant to cover. - */ - private static final int RECORD_SIZE = 32 * 1024 * 1024; - - /** Small record size used to pre-fill the region with evictable data (for putAll tests). */ - private static final int SMALL_RECORD_SIZE = 4096; - - /** - * Small pre-fill entries count. Chosen to fill the 128 MiB region close to capacity so that less than one large - * record ({@link #RECORD_SIZE}) remains available, forcing {@code ensureFreeSpaceForInsert} to actually evict - * pre-filled entries rather than taking its fast path. - */ - private static final int SMALL_ENTRIES = 28_000; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - return super.getConfiguration(gridName) - .setDataStorageConfiguration(new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setInitialSize(SIZE) - .setMaxSize(SIZE))); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); @@ -81,8 +48,10 @@ public void testRecordLargerThanRegionOom() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + long regionSize = regionMaxSize(ignite); + GridTestUtils.assertThrowsWithCause( - () -> cache.put(1, new byte[SIZE * 2]), + () -> cache.put(1, new byte[(int)(regionSize * 2)]), IgniteOutOfMemoryException.class ); @@ -90,7 +59,7 @@ public void testRecordLargerThanRegionOom() throws Exception { () -> { Map batch = new HashMap<>(); - Object val = new byte[RECORD_SIZE]; + Object val = new byte[(int)(regionSize / 4)]; for (int i = 0; i < 4; i++) batch.put(i, val); @@ -109,28 +78,34 @@ public void testRecordLargerThanRegionOom() throws Exception { */ @Test public void testPutAllLargeRows() throws Exception { - int putAllLargeRows = 3; - IgniteEx ignite = startGrid(1); IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill the region to near capacity with small evictable entries so that less than one large row remains - // available. This forces ensureFreeSpaceForInsert to evict pre-filled entries rather than taking its fast path. - byte[] small = new byte[SMALL_RECORD_SIZE]; + // Pre-fill to ~85% of capacity so that less than one large row remains available. + int smallEntries = (int)(totalPages(ignite) * 0.85); + + byte[] small = new byte[pageSize(ignite) - 200]; - for (int i = 0; i < SMALL_ENTRIES; i++) - cache.put(SMALL_ENTRIES + i, small); + for (int i = 0; i < smallEntries; i++) + cache.put(smallEntries + i, small); + + assertTrue("Pre-fill must load >80% of pages", loadedPages(ignite) > totalPages(ignite) * 0.80); + assertFalse("Eviction must not start during pre-fill", isEvictionsStarted(ignite)); + + int putAllLargeRows = 3; Map large = new HashMap<>(); - Object val = new byte[RECORD_SIZE]; + Object val = new byte[(int)(regionMaxSize(ignite) / 4)]; for (int i = 0; i < putAllLargeRows; i++) large.put(i, val); cache.putAll(large); + assertTrue("Eviction must start after putAll of large rows near capacity", isEvictionsStarted(ignite)); + for (int i = 0; i < putAllLargeRows; i++) assertNotNull("Large row " + i + " must be readable after putAll", cache.get(i)); } @@ -148,22 +123,24 @@ public void testUpdateRowGrows() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill the region to near capacity with small evictable entries so that the grown value below cannot - // fit without eviction. - byte[] small = new byte[SMALL_RECORD_SIZE]; + int smallEntries = (int)(totalPages(ignite) * 0.85); + + byte[] small = new byte[pageSize(ignite) - 200]; - for (int i = 0; i < SMALL_ENTRIES; i++) - cache.put(SMALL_ENTRIES + i, small); + for (int i = 0; i < smallEntries; i++) + cache.put(smallEntries + i, small); // Insert key 1 with a small value, then update it to a large value that requires size-aware eviction. cache.put(1, new byte[1024]); - byte[] big = new byte[RECORD_SIZE]; + byte[] big = new byte[(int)(regionMaxSize(ignite) / 4)]; Arrays.fill(big, (byte)7); cache.put(1, big); + assertTrue("Eviction must have started after growing a row near capacity", isEvictionsStarted(ignite)); + byte[] read = (byte[])cache.get(1); assertNotNull("Updated large value must be readable", read); @@ -171,11 +148,11 @@ public void testUpdateRowGrows() throws Exception { } /** - * Verifies the {@code evictionRegime} gate in {@code ensureFreeSpaceForEviction}: when the region is below the + * Verifies the fast path in {@code ensureFreeSpaceForEviction}: when the region is below the * eviction threshold, a large row (exceeding the empty-pages pool) that fits in the remaining headroom is written * successfully without triggering eviction — the size-aware reserve trusts headroom - * ({@code evictionRegime = false}) and skips the eviction loop. If the gate were broken (always {@code true}), - * size-aware eviction would run unnecessarily, evicting pre-filled entries and setting the evictions-started flag. + * and skips the eviction loop. If the fast path were broken, size-aware eviction would run unnecessarily, + * evicting pre-filled entries and setting the evictions-started flag. * * @throws Exception If failed. */ @@ -185,30 +162,29 @@ public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill to ~80 MiB — below the 0.9 * 128 MiB = 115.2 MiB threshold, leaving ~48 MiB of headroom. - // After pre-fill, loadedPages is well below threshold, so evictionRegime = false. - byte[] small = new byte[SMALL_RECORD_SIZE]; + // Pre-fill to ~40% of total pages — well below the eviction threshold, leaving ~50% headroom. + int preFill = (int)(totalPages(ignite) * 0.4); - int preFill = 20_000; + byte[] small = new byte[pageSize(ignite) - 200]; for (int i = 0; i < preFill; i++) cache.put(i, small); - DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); + long loadedPages = loadedPages(ignite); + long totalPages = totalPages(ignite); - assertFalse("Eviction must not have started during pre-fill below threshold", metrics.isEvictionsStarted()); + assertTrue("Pre-fill must load ~40% of pages", loadedPages > totalPages * 0.30 && loadedPages < totalPages * 0.50); + assertFalse("Eviction must not have started during pre-fill below threshold", isEvictionsStarted(ignite)); - // Write a 32 MiB row that fits in the remaining headroom (~48 MiB). Total ~112 MiB stays below the 115.2 MiB - // threshold, so evictionRegime remains false throughout and no eviction is triggered. - byte[] val = new byte[RECORD_SIZE]; + // Write a large row that fits in the remaining headroom. After pre-fill at ~40%, ~60% remains. + // Use ~30% of region so total ~70% stays below the 90% threshold. + byte[] val = new byte[(int)(regionMaxSize(ignite) * 0.3)]; Arrays.fill(val, (byte)1); cache.put(preFill, val); - // Eviction must not have started: the region never crossed the threshold, so the size-aware reserve - // trusted headroom and the normal threshold eviction in insertDataRows never fired. - assertFalse("Eviction must not start when the region stays below the threshold", metrics.isEvictionsStarted()); + assertFalse("Eviction must not start when the region stays below the threshold", isEvictionsStarted(ignite)); for (int i = 0; i < preFill; i++) assertNotNull("Pre-filled entry " + i + " must not be evicted below threshold", cache.get(i)); @@ -221,8 +197,8 @@ public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception /** * Verifies that a large row (exceeding the empty-pages pool) is written successfully when the region is already - * at or above the eviction threshold — the size-aware eviction loop ({@code evictionRegime = true}) evicts enough - * pre-filled entries to free the required pages, and the write succeeds. + * at or above the eviction threshold — the size-aware eviction loop evicts enough + * pre-filled entries to free the required pages, and to write succeeds. * * @throws Exception If failed. */ @@ -232,30 +208,91 @@ public void testLargeRowAboveThresholdEvictsAndSucceeds() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill to near capacity with small evictable entries. 28 000 x 4 KiB ~ 112 MiB of data; with page - // overhead loadedPages is at or above the 0.9 threshold, so evictionRegime = true. - byte[] small = new byte[SMALL_RECORD_SIZE]; + // Pre-fill to ~85% of capacity — above the 0.9 eviction threshold (with page overhead). + int smallEntries = (int)(totalPages(ignite) * 0.85); + + byte[] small = new byte[pageSize(ignite) - 200]; - for (int i = 0; i < SMALL_ENTRIES; i++) + for (int i = 0; i < smallEntries; i++) cache.put(i, small); - // Write a 32 MiB row that exceeds the empty-pages pool. Since the region is at or above the eviction - // threshold, the size-aware reserve cannot trust headroom and must actually evict pre-filled entries. - byte[] val = new byte[RECORD_SIZE]; + assertTrue("Pre-fill must load >80% of pages", loadedPages(ignite) > totalPages(ignite) * 0.80); + + // Write a large row that exceeds the combined headroom + empty-pages pool, so the size-aware + // reserve must actually evict pre-filled entries to make room. + byte[] val = new byte[(int)(regionMaxSize(ignite) / 4)]; Arrays.fill(val, (byte)1); - int largeKey = SMALL_ENTRIES + 1; + int largeKey = smallEntries + 1; cache.put(largeKey, val); - DataRegionMetricsImpl metrics = ignite.context().cache().context().database().dataRegion(null).metrics(); - - assertTrue("Eviction must have started after a large put near capacity", metrics.isEvictionsStarted()); + assertTrue("Eviction must have started after a large put near capacity", isEvictionsStarted(ignite)); byte[] read = (byte[])cache.get(largeKey); assertNotNull("Large row must be readable after eviction", read); assertTrue("Value read back must equal the stored value", Arrays.equals(val, read)); } + + /** + * Verifies that the data region can be filled beyond the eviction threshold (default 0.9) — at least to + * 95% of total pages — without triggering eviction. Before the fix, the headroom gate + * caused the size-aware reserve to ignore headroom above the threshold, so the region effectively stopped + * growing at ~90% and evicted pre-filled entries unnecessarily. + * + * @throws Exception If failed. + */ + @Test + public void testRegionFillsBeyondEvictionThreshold() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + + long total = totalPages(ignite); + + byte[] small = new byte[pageSize(ignite) - 200]; + + // Put entries until the region is at least 95% full. + for (int i = 0; i < total; i++) { + cache.put(i, small); + + if (loadedPages(ignite) >= total * 0.95) + break; + } + + assertTrue("Region must fill beyond the 0.9 eviction threshold", loadedPages(ignite) >= total * 0.95); + assertFalse("Eviction must not start when headroom is still available", isEvictionsStarted(ignite)); + + stopGrid(1); + ignite = startGrid(1); + + assertEquals(0, loadedPages(ignite)); + + cache = createCache(ignite, DEFAULT_CACHE_NAME); + + int batchSize = 100; + + // Put entries in batches until the region is at least 95% full. + for (int base = 0; base < total; base += batchSize) { + Map batch = new HashMap<>(); + + for (int i = 0; i < batchSize && base + i < total; i++) + batch.put(base + i, small); + + cache.putAll(batch); + + if (loadedPages(ignite) >= total * 0.95) + break; + } + + assertTrue("Region must fill beyond the 0.9 eviction threshold via putAll", loadedPages(ignite) >= total * 0.95); + assertFalse("Eviction must not start when headroom is still available", isEvictionsStarted(ignite)); + } + + /** @return Loaded pages. */ + private long loadedPages(IgniteEx ignite) throws IgniteCheckedException { + return defaultRegion(ignite).pageMemory().loadedPages(); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index edf1902c3f465..ba602815af9a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -17,46 +17,33 @@ package org.apache.ignite.internal.processors.cache.eviction.paged; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + /** Tests the synergy between ExpiryPolicy (TTL cleanup) and size-aware page eviction on an in-memory data region. */ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvictionAbstractTest { - /** Off-heap region size. */ - private static final int SIZE = 128 * 1024 * 1024; - - /** - * Large record size (much larger than the empty-pages pool, and larger than the space left free when the region - * is held at the eviction threshold {@code (1 - threshold) * totalPages}) so that a large put cannot take the - * fast path of {@code ensureFreeSpaceForInsert} and must actually run the size-aware eviction reserve. - */ - private static final int RECORD_SIZE = 32 * 1024 * 1024; - /** Short TTL applied to some entries. */ private static final long TTL = 2000; - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - return super.getConfiguration(gridName) - .setDataStorageConfiguration(new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setInitialSize(SIZE) - .setMaxSize(SIZE))); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); } /** - * Concurrent TTL cleanup and eviction must not deadlock, and a large record (larger than the empty-pages pool) - * must still be stored on a region with enabled eviction even in the presence of short-TTL entries. + * Concurrent page eviction and TTL cleanup on the same data region must not deadlock. + * Multiple threads insert large records into a plain cache (triggering eviction) while small + * entries in a separate TTL cache (sharing the same data region) expire and are cleaned up + * by the eager-TTL background worker. * * @throws Exception If failed. */ @@ -64,14 +51,52 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvict public void testLargePutWithExpiryNoDeadlock() throws Exception { IgniteEx ignite = startGrid(1); - IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, TTL, false); + IgniteCache plainCache = createCache(ignite, "plain-cache"); + IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); - Object val = new byte[RECORD_SIZE]; + int smallEntries = 1_000; + + for (int i = 0; i < smallEntries; i++) + plainCache.put(i, new byte[16 * 1024]); - for (int i = 0; i < 60; i++) - cache.put(i, val); + assertFalse("Eviction must not have started during pre-fill", isEvictionsStarted(ignite)); + + int ttlEntries = 10; + byte[] ttlVal = new byte[100]; + + for (int i = 0; i < ttlEntries; i++) + ttlCache.put(i, ttlVal); - assertNotNull("Cache must remain responsive after concurrent expiry and eviction", cache.get(59)); + int largeRowsPerThread = 40; + byte[] largeVal = new byte[1024 * 1024]; + + CountDownLatch startLatch = new CountDownLatch(1); + AtomicInteger threadIdx = new AtomicInteger(); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { + U.awaitQuiet(startLatch); + + int idx = threadIdx.getAndIncrement(); + + for (int k = 0; k < largeRowsPerThread; k++) + plainCache.put(smallEntries + idx * largeRowsPerThread + k, largeVal); + }, 10, "paged-writer"); + + startLatch.countDown(); + + assertTrue("TTL entries must expire after TTL duration", + waitForCondition(() -> { + for (int i = 0; i < ttlEntries; i++) { + if (ttlCache.get(i) != null) + return false; + } + + return true; + }, TTL + 10_000)); + + fut.get(TimeUnit.MINUTES.toMillis(3)); + + assertTrue("Eviction must have started during concurrent large writes", isEvictionsStarted(ignite)); } /** @@ -82,28 +107,28 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { */ @Test public void testTtlFreedSpaceAccountedForByEviction() throws Exception { - int ttlEntries = 2; - int smallEntries = 6_000; - IgniteEx ignite = startGrid(1); IgniteCache plainCache = createCache(ignite, "plain-cache"); IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); - // Pre-fill the region with small non-expiring entries, but leave enough room for the large TTL entries. - byte[] small = new byte[1024]; + int ttlEntries = 2; + + int smallEntries = (int)(totalPages(ignite) * 0.3); + + byte[] small = new byte[pageSize(ignite) - 200]; for (int i = 0; i < smallEntries; i++) plainCache.put(i, small); - Object val = new byte[RECORD_SIZE]; + Object val = new byte[(int)(regionMaxSize(ignite) / 5)]; for (int i = 0; i < ttlEntries; i++) ttlCache.put(i, val); assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); - assertTrue(GridTestUtils.waitForCondition(() -> ttlCache.get(0) == null, 10_000)); + assertTrue(waitForCondition(() -> ttlCache.size() == 0, 10_000)); for (int i = 0; i < ttlEntries; i++) plainCache.put(smallEntries + i, val); From 4c8714131a17e53a66852ea79b1a0814629769ba Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 29 Sep 2026 01:17:13 +0500 Subject: [PATCH 18/26] fix testPageEviction in multinode tests --- .../paged/PageEvictionMultinodeAbstractTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java index 2bf21fbd32fb9..b7138964e9572 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java @@ -30,6 +30,9 @@ * */ public abstract class PageEvictionMultinodeAbstractTest extends PageEvictionAbstractTest { + /** Number of entries, sized to fill the data region past the headroom gate in {@code ensureFreeSpace}. */ + private static final int MULTINODE_ENTRIES = 16_000; + /** Cache modes. */ private static final CacheMode[] CACHE_MODES = {CacheMode.PARTITIONED, CacheMode.REPLICATED}; @@ -86,7 +89,7 @@ public void testPageEviction() throws Exception { protected void createCacheAndTestEviction(CacheConfiguration cfg) throws Exception { IgniteCache cache = clientGrid().getOrCreateCache(cfg); - for (int i = 1; i <= ENTRIES; i++) { + for (int i = 1; i <= MULTINODE_ENTRIES; i++) { ThreadLocalRandom r = ThreadLocalRandom.current(); if (r.nextInt() % 5 == 0) @@ -101,7 +104,7 @@ else if (r.nextInt() % 11 == 0) else if (r.nextInt() % 13 == 0) cache.put(r.nextInt(i), new TestObject(r.nextInt(PAGE_SIZE / 2))); // Update. - if (i % (ENTRIES / 10) == 0) + if (i % (MULTINODE_ENTRIES / 10) == 0) System.out.println(">>> Entries put: " + i); } @@ -110,7 +113,7 @@ else if (r.nextInt() % 13 == 0) System.out.println(">>> Resulting size: " + resultingSize); // Eviction started, no OutOfMemory occurred, success. - assertTrue(resultingSize < ENTRIES * 71 / 77); + assertTrue(resultingSize < MULTINODE_ENTRIES * 71 / 77); clientGrid().destroyCache(cfg.getName()); } From 074185d5c937c6447bdbd9f0e7c9158963601456 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 29 Sep 2026 12:38:32 +0500 Subject: [PATCH 19/26] minor --- ...eEvictionConcurrentWritesAbstractTest.java | 33 ++++++++++++++++++- .../PageEvictionSizeAwareAbstractTest.java | 11 ++++--- ...eEvictionWithExpiryPolicyAbstractTest.java | 31 ++++++++++------- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index fbc064ff176bc..d41668e07fbd6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.eviction.paged; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -37,6 +38,8 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict /** * Concurrent large inserts into a region pre-filled with small entries must complete within the deadline without * deadlock, and without corrupting the free list (eviction frees small entries rather than overrunning the region). + * After the concurrent phase, the cache must remain functional: a new put+get must succeed, and at least some of + * the large entries written by concurrent threads must be readable. * * @throws Exception If failed. */ @@ -44,6 +47,7 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict public void testConcurrentLargeWritesNoDeadlock() throws Exception { int smallEntries = 1_000; int largeRowsPerThread = 20; + int threads = 10; IgniteEx ignite = startGrid(1); @@ -72,12 +76,39 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { for (int k = 0; k < largeRowsPerThread; k++) cache.put(smallEntries + idx * largeRowsPerThread + k, largeVal); - }, 10, "paged-writer"); + }, threads, "paged-writer"); startLatch.countDown(); fut.get(TimeUnit.MINUTES.toMillis(3)); assertTrue("Eviction must have started during concurrent large writes", isEvictionsStarted(ignite)); + + // Verify the free list is not corrupted: a new put+get must succeed. + byte[] probeVal = new byte[pageSize(ignite) - 200]; + + Arrays.fill(probeVal, (byte)1); + + int probeKey = smallEntries + threads * largeRowsPerThread + 1; + + cache.put(probeKey, probeVal); + + byte[] read = (byte[])cache.get(probeKey); + + assertNotNull(read); + assertTrue(Arrays.equals(probeVal, read)); + + int readableLarge = 0; + + for (int t = 0; t < threads; t++) { + for (int k = 0; k < largeRowsPerThread; k++) { + int key = smallEntries + t * largeRowsPerThread + k; + + if (cache.get(key) != null) + readableLarge++; + } + } + + assertTrue(readableLarge > 0); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 18116339c908f..5976de57108fe 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -36,9 +36,9 @@ public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbst /** * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. - * A batch {@code putAll} of records whose total size equals the region capacity must also fail with OOM because - * structural pages (free list, index tree) leave no room for the data (exercises the batch store path - * {@code RowStore.addRows} → {@code ensureFreeSpaceForInsert}). + * A {@code putAll} of records whose total size equals the region capacity must also fail with OOM because + * structural pages (free list, index tree) leave no room for the data. {@code putAll} writes entries individually + * via {@code RowStore.addRow} → {@code FreeList.insertDataRow} (single-row path). * * @throws Exception If failed. */ @@ -71,8 +71,9 @@ public void testRecordLargerThanRegionOom() throws Exception { } /** - * A batch putAll of several large records (each larger than the empty-pages pool) must be stored successfully when - * page eviction is enabled. Exercises the size-aware reserve in the batch store path ({@code RowStore.addRows}). + * A {@code putAll} of several large records (each larger than the empty-pages pool) must be stored successfully + * when page eviction is enabled. Exercises the size-aware reserve in the single-row path + * ({@code RowStore.addRow} → {@code FreeList.insertDataRow}). * * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index ba602815af9a3..b701cb74dc9ef 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.eviction.paged; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -41,9 +42,9 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvict /** * Concurrent page eviction and TTL cleanup on the same data region must not deadlock. - * Multiple threads insert large records into a plain cache (triggering eviction) while small + * Multiple threads insert large records into a plain cache (triggering eviction) while page-sized * entries in a separate TTL cache (sharing the same data region) expire and are cleaned up - * by the eager-TTL background worker. + * by the eager-TTL background worker, freeing real pages that the eviction machinery may race with. * * @throws Exception If failed. */ @@ -61,8 +62,9 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { assertFalse("Eviction must not have started during pre-fill", isEvictionsStarted(ignite)); - int ttlEntries = 10; - byte[] ttlVal = new byte[100]; + // Page-sized TTL entries so their cleanup frees real pages concurrently with eviction. + int ttlEntries = 100; + byte[] ttlVal = new byte[pageSize(ignite) - 200]; for (int i = 0; i < ttlEntries; i++) ttlCache.put(i, ttlVal); @@ -101,7 +103,7 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { /** * After short-TTL entries expire and free their pages, putting the same amount of data to a non-expiring cache - * must succeed, and the pre-filled small entries must remain intact (not evicted). + * must succeed, and the new data must be readable. * * @throws Exception If failed. */ @@ -112,29 +114,34 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { IgniteCache plainCache = createCache(ignite, "plain-cache"); IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); - int ttlEntries = 2; - - int smallEntries = (int)(totalPages(ignite) * 0.3); + int smallEntries = (int)(totalPages(ignite) * 0.80); byte[] small = new byte[pageSize(ignite) - 200]; for (int i = 0; i < smallEntries; i++) plainCache.put(i, small); - Object val = new byte[(int)(regionMaxSize(ignite) / 5)]; + assertFalse("Eviction must not start during pre-fill", isEvictionsStarted(ignite)); + + int ttlEntries = 2; + byte[] val = new byte[(int)(regionMaxSize(ignite) * 0.05)]; for (int i = 0; i < ttlEntries; i++) ttlCache.put(i, val); assertNotNull("TTL entry must be present before expiry", ttlCache.get(0)); - assertTrue(waitForCondition(() -> ttlCache.size() == 0, 10_000)); + assertTrue("TTL entries must expire", waitForCondition(() -> ttlCache.size() == 0, 10_000)); for (int i = 0; i < ttlEntries; i++) plainCache.put(smallEntries + i, val); - for (int i = 0; i < ttlEntries; i++) - assertNotNull("Fresh large record must be present", plainCache.get(smallEntries + i)); + for (int i = 0; i < ttlEntries; i++) { + byte[] read = (byte[])plainCache.get(smallEntries + i); + + assertNotNull("Fresh large record must be present", read); + assertTrue("Fresh large record must match", Arrays.equals(val, read)); + } for (int i = 0; i < smallEntries; i++) assertNotNull("Pre-filled entry " + i + " must not be evicted", plainCache.get(i)); From 93045785ae67827488ff6c0c1090881af2053381 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 2 Oct 2026 12:54:50 +0500 Subject: [PATCH 20/26] fix review comments --- .../DataRegionConfiguration.java | 12 ++- .../IgniteCacheDatabaseSharedManager.java | 21 ++-- .../evict/Random2LruPageEvictionTracker.java | 6 +- .../evict/RandomLruPageEvictionTracker.java | 6 +- .../freelist/AbstractFreeList.java | 99 +++++-------------- 5 files changed, 48 insertions(+), 96 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java index 9666295a5203b..e8ddb992aa34e 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java @@ -346,8 +346,10 @@ public DataRegionConfiguration setEvictionThreshold(double evictionThreshold) { * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of * (key, value) pair is slightly larger than page size / 2. * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool, - * it is no longer required to increase this parameter up to the size of the largest cache entry; - * it may be kept at its default as the steady-state reserve of empty pages. + * it is no longer required to increase this parameter up to the size of the largest cache entry. + * It also acts as the concurrency-safety margin for size-aware eviction: the reserve keeps this many empty pages + * on top of each row's own requirement, so the insert path is safe for up to {@code emptyPagesPoolSize} concurrent + * writers. * * @return Minimum number of empty pages in reuse list. */ @@ -360,8 +362,10 @@ public int getEmptyPagesPoolSize() { * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of * (key, value) pair is slightly larger than page size / 2. * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool, - * it is no longer required to increase this parameter up to the size of the largest cache entry; - * it may be kept at its default as the steady-state reserve of empty pages. + * it is no longer required to increase this parameter up to the size of the largest cache entry. + * It also acts as the concurrency-safety margin for size-aware eviction: the reserve keeps this many empty pages + * on top of each row's own requirement, so the insert path is safe for up to {@code emptyPagesPoolSize} concurrent + * writers. * * @param emptyPagesPoolSize Empty pages pool size. * @return {@code this} for chaining. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index b07a552cb76a7..140266ade3d42 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1206,10 +1206,6 @@ public WALPointer latestWalPointerReservedForPreloading() { * row does not fit into the currently available page space, data pages are evicted until either enough space is * freed or it becomes clear that the goal is unreachable (in which case an * {@link IgniteOutOfMemoryException} is thrown). - *

    - * The size-aware reserve is required because page eviction by itself only keeps a steady-state pool of empty pages - * ({@link DataRegionConfiguration#getEmptyPagesPoolSize()}) and does not guarantee enough space for a single row - * larger than this pool. * * @param region Data region to be checked. * @param dataRowSize Size of data row to be inserted. @@ -1262,10 +1258,13 @@ private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg /** * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the combined space - * (empty pages in the free list + unallocated headroom) is enough to accommodate the row, or throws - * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. Both empty pages and - * headroom are treated as concurrent resources: the fast path returns as soon as their sum covers the row, and - * the eviction loop only evicts the deficit (pages that neither headroom nor the free list can provide). + * (empty pages in the free list + unallocated headroom) is enough to accommodate the row plus the + * {@link DataRegionConfiguration#getEmptyPagesPoolSize() empty-pages pool}, or throws + * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. The pool margin is kept + * on top of the row's own requirement so that, after this thread consumes its pages, at least {@code poolSize} + * pages remain for concurrent writers; this makes the path safe for up to {@code poolSize} concurrent threads. + * Both empty pages and headroom are treated as concurrent resources: the fast path returns as soon as + * their sum covers the target, and the eviction loop only evicts the deficit. * * @param region Data region. * @param regCfg Data region configuration. @@ -1295,11 +1294,13 @@ private void ensureFreeSpaceForEviction( if (requiredPages > totalPages) throw outOfMemory(regCfg); + long targetPages = Math.min(requiredPages + regCfg.getEmptyPagesPoolSize(), totalPages); + long emptyPages = freeList.emptyDataPages(); // Effective headroom accounts for per-segment allocation overhead (up to one page per segment), // so the fast path does not trust headroom that cannot actually be allocated. - if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= requiredPages) + if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= targetPages) return; PageEvictionTracker evictionTracker = region.evictionTracker(); @@ -1312,7 +1313,7 @@ private void ensureFreeSpaceForEviction( long backoffNanos = EVICTION_BACKOFF_START_NANOS; - while (freeList.emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < requiredPages) { + while (freeList.emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < targetPages) { if (region.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index 5511cbad2ddf6..ac361fcb7b10d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -161,11 +161,7 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); - // Under concurrent writes a fragment chain may be partially built (a concurrent writer - // has not yet linked the head page), so after two hops the timestamp can still be - // negative. Skip this sample and try another page. - if (firstTs < 0) - continue; + assert firstTs >= 0 : "[firstTs=" + firstTs + ", secondTs=" + second(trackingData) + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index 08530ab56f94e..ff94ac714b3e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -141,11 +141,7 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); - // Under concurrent writes a fragment chain may be partially built (a concurrent writer - // has not yet linked the head page), so after two hops the timestamp can still be - // negative. Skip this sample and try another page. - if (compactTs < 0) - continue; + assert compactTs >= 0 : "[compactTs=" + compactTs + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index d0239c189df17..512337e04b620 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -55,7 +55,6 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; -import static org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.SEG_CNT; /** */ @@ -78,13 +77,6 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private static final int MIN_PAGE_FREE_SPACE = 8; - /** - * Bounded number of lazy size-aware re-reserve attempts on a fragmented write before falling back to - * allocating a brand-new page. Each attempt runs size-aware eviction (which itself fails with a clean OOM when - * eviction cannot progress), so this bounds the retry even under heavy contention. - */ - private static final int RE_RESERVE_ATTEMPTS = 4; - /** * Step between buckets in free list, measured in powers of two. * For example, for page size 4096 and 256 buckets, shift is 4 and step is 16 bytes. @@ -109,10 +101,10 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; - /** Data region this free list belongs to (used for lazy size-aware re-reserve on fragmented writes). */ + /** Data region this free list belongs to (used for proactive size-aware reserve on fragmented writes). */ private final DataRegion dataRegion; - /** Database shared manager (used for lazy size-aware re-reserve on fragmented writes). */ + /** Database shared manager (used for proactive size-aware reserve on fragmented writes). */ private final IgniteCacheDatabaseSharedManager dbMgr; /** Page list cache limit. */ @@ -480,8 +472,8 @@ public AbstractFreeList( this.evictionTracker = dataRegion.evictionTracker(); this.dataRegion = dataRegion; - // dbMgr is needed only for the on-demand re-reserve (eviction-enabled in-memory region); null in unit tests - // without a cache processor (where eviction is disabled and the re-reserve never fires). + // dbMgr is needed only for the proactive reserve (eviction-enabled in-memory region); null in unit tests + // without a cache processor (where eviction is disabled and the reserve never fires). dbMgr = ctx.cache() == null ? null : ctx.cache().context().database(); this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); @@ -602,75 +594,40 @@ private int bucket(int freeSpace, boolean allowReuse) { } /** - * @param row Row to write. - * @param written Written size. - * @param statHolder Statistics holder to track IO operations. + * @param part Partition. * @return Page ID. * @throws IgniteCheckedException If failed. */ - private long allocateDataPage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { - assert row.partition() <= PageIdAllocator.MAX_PARTITION_ID; - - long pageId; + private long allocateDataPage(int part) throws IgniteCheckedException { + assert part <= PageIdAllocator.MAX_PARTITION_ID; - try { - pageId = pageMem.allocatePage(grpId, row.partition(), FLAG_DATA); - } - catch (IgniteOutOfMemoryException oom) { - // Headroom was consumed after the pre-reserve trusted it. Evict to create empty pages and retry. - dbMgr.ensureFreeSpaceForInsert(dataRegion, row.size() - written); - - pageId = takePage(row.size() - written, row, statHolder); - - if (pageId == 0L) - throw oom; - } - - return pageId; + return pageMem.allocatePage(grpId, part, FLAG_DATA); } /** - * @return {@code true} when the region has effectively no headroom left (allocated pages reached the configured - * max), so a fresh {@code allocateDataPage} could no longer grow it. - */ - private boolean regionEffectivelyFull() { - long maxPages = dataRegion.config().getMaxSize() / pageMem.systemPageSize(); - - // Each of up to 16 segments loses up to one page to allocation overhead (lastAllocatedIdxPtr + alignment), - // so the theoretical max is never reached in practice. Subtract the worst-case segment loss to get an - // effective limit that reflects real product scenarios. - if (maxPages > SEG_CNT) - maxPages -= SEG_CNT; - - return pageMem.loadedPages() >= maxPages; - } - - /** - * Take a page, and if the free list cannot hand one out, re-run the size-aware reserve and retry. The reserve only - * bounds the shared empty-pages counter and does not pin pages to this thread, so a concurrent writer may consume - * them before this allocation; retrying closes that TOCTOU instead of falling straight to a raw - * {@code allocateDataPage}. How hard to retry depends on whether the region can still grow: on an effectively-full - * region re-reserving is bounded (each attempt itself fails with a clean OOM when eviction cannot progress), while - * with headroom a single re-reserve suffices and the subsequent {@code allocateDataPage} grows the region. + * Proactively reserves enough free space for the row and then takes a page. The reserve targets + * {@code requiredPages + emptyPagesPoolSize} (see {@link IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert}), + * so after this thread consumes its pages at least {@code emptyPagesPoolSize} pages remain for concurrent writers. * * @param size Free space required on the page. * @param row Row to write. * @param statHolder Statistics holder to track IO operations. - * @return Page identifier or 0 if no page could be obtained after re-reserving. + * @return Page identifier or 0 if no page could be obtained after reserving. * @throws IgniteCheckedException If failed. */ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { - long pageId = takePage(size, row, statHolder); + if (dbMgr == null || dataRegion.config().isPersistenceEnabled() || + dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) + return takePage(size, row, statHolder); - if (pageId == 0L && dbMgr != null && !dataRegion.config().isPersistenceEnabled() - && dataRegion.config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - int reReserveAttempts = regionEffectivelyFull() ? RE_RESERVE_ATTEMPTS : 1; + dbMgr.ensureFreeSpaceForInsert(dataRegion, size); - for (int i = 0; pageId == 0L && i < reReserveAttempts; i++) { - dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + long pageId = takePage(size, row, statHolder); - pageId = takePage(size, row, statHolder); - } + if (pageId == 0L) { + dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + + pageId = takePage(size, row, statHolder); } return pageId; @@ -735,7 +692,7 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { - pageId = allocateDataPage(row, written, statHolder); + pageId = allocateDataPage(row.partition()); initIo = row.ioVersions().latest(); } @@ -780,12 +737,6 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC /** * Take a page and write row on it. - *

    - * The page is acquired via {@link #takePageWithReserve}, which handles the size-aware re-reserve and TOCTOU - * recovery (see its javadoc for details). Reached from the BPlusTree.invoke row-creation closure, this re-reserve - * is an inline demand-eviction that runs while the BPlusTree leaf page read lock is held (the closure is invoked - * inside {@code read} before the write lock is taken). The entry-level tryLock only skips contended/self-held - * entries and never blocks, avoiding a lock-ordering deadlock with the entry lock already held by the caller. * * @param row Row to write. * @param written Written size. @@ -799,7 +750,7 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t long pageId = takePageWithReserve(row.size() - written, row, statHolder); if (pageId == 0L) { - pageId = allocateDataPage(row, written, statHolder); + pageId = allocateDataPage(row.partition()); initIo = row.ioVersions().latest(); } @@ -920,6 +871,8 @@ private long initReusedPage(T row, long reusedPageId, IoStatisticsHolder statHol assert nextLink != FAIL_L; // Can't fail here. + evictionTracker.forgetPage(pageId); + while (nextLink != 0L) { memMetrics.decrementLargeEntriesPages(); @@ -929,6 +882,8 @@ private long initReusedPage(T row, long reusedPageId, IoStatisticsHolder statHol nextLink = write(pageId, rmvRow, bag, itemId, FAIL_L, statHolder); assert nextLink != FAIL_L; // Can't fail here. + + evictionTracker.forgetPage(pageId); } reuseList.addForRecycle(bag); From 98c759e3c3ade36ebfbe54b0a0f7f374701cb260 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 5 Oct 2026 17:42:59 +0500 Subject: [PATCH 21/26] try to fix assertion error --- .../evict/Random2LruPageEvictionTracker.java | 20 +++++++++++++++++-- .../evict/RandomLruPageEvictionTracker.java | 20 +++++++++++++++++-- .../freelist/AbstractFreeList.java | 2 -- .../PageEvictionSizeAwareAbstractTest.java | 4 +--- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index ac361fcb7b10d..df60e83be309f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -140,7 +140,11 @@ public Random2LruPageEvictionTracker( while (dataPagesCnt < SAMPLE_SIZE) { int trackingIdx = rnd.nextInt(trackingSize); - long trackingData = GridUnsafe.getLongVolatile(null, trackingArrPtr + trackingIdx * 8L); + int origTrackingIdx = trackingIdx; + + long origTrackingData = GridUnsafe.getLongVolatile(null, trackingArrPtr + origTrackingIdx * 8L); + + long trackingData = origTrackingData; int firstTs = first(trackingData); @@ -161,7 +165,19 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); - assert firstTs >= 0 : "[firstTs=" + firstTs + ", secondTs=" + second(trackingData) + "]"; + if (firstTs < 0) { + GridUnsafe.compareAndSwapLong(null, trackingArrPtr + origTrackingIdx * 8L, origTrackingData, 0L); + + sampleSpinCnt++; + + if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { + LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT); + + return false; + } + + continue; + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index ff94ac714b3e9..6d8c1378a0ee2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -126,7 +126,11 @@ public RandomLruPageEvictionTracker( while (dataPagesCnt < SAMPLE_SIZE) { int sampleTrackingIdx = rnd.nextInt(trackingSize); - int compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); + int origSampleIdx = sampleTrackingIdx; + + int origSampleData = GridUnsafe.getIntVolatile(null, trackingArrPtr + origSampleIdx * 4L); + + int compactTs = origSampleData; if (compactTs < 0) { // For page containing fragmented row data timestamps stored in the row's head page are used. @@ -141,7 +145,19 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); - assert compactTs >= 0 : "[compactTs=" + compactTs + "]"; + if (compactTs < 0) { + GridUnsafe.compareAndSwapInt(null, trackingArrPtr + origSampleIdx * 4L, origSampleData, 0); + + sampleSpinCnt++; + + if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { + LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT); + + return false; + } + + continue; + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 512337e04b620..a4ccd658ab947 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -871,8 +871,6 @@ private long initReusedPage(T row, long reusedPageId, IoStatisticsHolder statHol assert nextLink != FAIL_L; // Can't fail here. - evictionTracker.forgetPage(pageId); - while (nextLink != 0L) { memMetrics.decrementLargeEntriesPages(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 5976de57108fe..86865b56e6137 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -239,9 +239,7 @@ public void testLargeRowAboveThresholdEvictsAndSucceeds() throws Exception { /** * Verifies that the data region can be filled beyond the eviction threshold (default 0.9) — at least to - * 95% of total pages — without triggering eviction. Before the fix, the headroom gate - * caused the size-aware reserve to ignore headroom above the threshold, so the region effectively stopped - * growing at ~90% and evicted pre-filled entries unnecessarily. + * 95% of total pages — without triggering eviction. * * @throws Exception If failed. */ From f17ca4f95c6f06e43d45d5e47430163b4c6db275 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 12 Oct 2026 20:16:31 +0800 Subject: [PATCH 22/26] fix possible deadlock for putAll/rebalance + repro for tracker assertions --- .../pagemem/impl/PageMemoryNoStoreImpl.java | 17 ++ .../evict/PageAbstractEvictionTracker.java | 5 +- .../freelist/AbstractFreeList.java | 66 ++++-- .../internal/util/OffheapReadWriteLock.java | 17 ++ ...eEvictionConcurrentWritesAbstractTest.java | 143 ++++++++++-- ...eEvictionTrackerFragmentChainSelfTest.java | 212 ++++++++++++++++++ .../IgniteCacheEvictionSelfTestSuite.java | 3 + 7 files changed, 422 insertions(+), 41 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index d9bd2b8bacaf0..7558ab8cb97cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -525,6 +525,23 @@ private long fromSegmentIndex(int segIdx, long pageIdx) { return 0L; } + /** + * Attempts to acquire the read lock regardless of the page tag, without ever blocking. + * + * @param cacheId Cache group ID. + * @param pageId Page ID. + * @param page Page absolute pointer. + * @return Page address or {@code 0L} if the page is currently write-locked. + */ + public long tryReadLock(int cacheId, long pageId, long page) { + assert started; + + if (rwLock.tryReadLock(page + LOCK_OFFSET, -1)) + return page + PAGE_OVERHEAD; + + return 0L; + } + /** {@inheritDoc} */ @Override public void readUnlock(int cacheId, long pageId, long page) { assert started; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index d97a2b95e1c64..700151c3aff2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -104,7 +104,10 @@ final boolean evictDataPage(int pageIdx, boolean tryLock) throws IgniteCheckedEx List rowsToEvict; try { - long pageAddr = pageMem.readLockForce(0, fakePageId, page); + long pageAddr = pageMem.tryReadLock(0, fakePageId, page); + + if (pageAddr == 0L) + return false; // Candidate page is write-locked: skip it. try { if (PageIO.getType(pageAddr) != PageIO.T_DATA) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index a6a2773640beb..a34d0c319b706 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; +import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; @@ -50,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; import org.apache.ignite.internal.util.GridCursorIteratorWrapper; +import org.apache.ignite.internal.util.OffheapReadWriteLock; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.Nullable; @@ -350,7 +352,7 @@ private final class WriteRowsHandler extends PageHandler, Integer> if (written == COMPLETE) { // If the data row was completely written without remainder, proceed to the next. - if ((written = writeWholePages(row, statHolder)) == COMPLETE) + if ((written = writeWholePages(row, statHolder, false)) == COMPLETE) continue; if (io.getFreeSpace(pageAddr) < row.size() - written) @@ -605,32 +607,36 @@ private long allocateDataPage(int part) throws IgniteCheckedException { } /** - * Proactively reserves enough free space for the row and then takes a page. The reserve targets - * {@code requiredPages + emptyPagesPoolSize} (see {@link IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert}), - * so after this thread consumes its pages at least {@code emptyPagesPoolSize} pages remain for concurrent writers. + * Takes a page, optionally running the size-aware reserve first (see {@link #ensureFreeSpaceForInsert(int)}). + * The reserve targets {@code requiredPages + emptyPagesPoolSize}, so after this thread consumes its pages + * at least {@code emptyPagesPoolSize} pages remain for concurrent writers. * * @param size Free space required on the page. * @param row Row to write. * @param statHolder Statistics holder to track IO operations. + * @param reserve {@code true} to run the size-aware reserve before taking the page. Must be {@code false} + * when the current thread already holds a page write lock: the reserve may evict, and eviction takes a + * blocking read lock on the candidate data page, while {@link OffheapReadWriteLock} is not reentrant + * (see {@link PageAbstractEvictionTracker}). * @return Page identifier or 0 if no page could be obtained after reserving. * @throws IgniteCheckedException If failed. */ - private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { - if (dbMgr == null || dataRegion.config().isPersistenceEnabled() || - dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) - return takePage(size, row, statHolder); + private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { + if (reserve) { + ensureFreeSpaceForInsert(size); - dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + long pageId = takePage(size, row, statHolder); - long pageId = takePage(size, row, statHolder); + if (pageId == 0L) { + ensureFreeSpaceForInsert(size); - if (pageId == 0L) { - dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + pageId = takePage(size, row, statHolder); + } - pageId = takePage(size, row, statHolder); + return pageId; } - return pageId; + return takePage(size, row, statHolder); } /** {@inheritDoc} */ @@ -642,7 +648,7 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) if (written != 0) memMetrics.incrementLargeEntriesPages(); - written = writeSinglePage(row, written, statHolder); + written = writeSinglePage(row, written, statHolder, true); } while (written != COMPLETE); } @@ -682,14 +688,14 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) } if (written == COMPLETE) { - written = writeWholePages(row, statHolder); + written = writeWholePages(row, statHolder, true); continue; } AbstractDataPageIO initIo = null; - long pageId = takePageWithReserve(row.size() - written, row, statHolder); + long pageId = takePageWithReserve(row.size() - written, row, statHolder, true); if (pageId == 0L) { pageId = allocateDataPage(row.partition()); @@ -716,18 +722,20 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder) * * @param row Row to process. * @param statHolder Statistics holder to track IO operations. + * @param reserve {@code true} to run the size-aware reserve before each page allocation; must be {@code false} + * when the current thread holds a page write lock. * @return Number of bytes written, {@link #COMPLETE} if the row was fully written, {@code 0} if data row was * ignored because it is less than the max payload of an empty data page. * @throws IgniteCheckedException If failed. */ - private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { + private int writeWholePages(T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { assert row.link() == 0 : row.link(); int written = 0; int rowSize = row.size(); while (rowSize - written >= MIN_SIZE_FOR_DATA_PAGE) { - written = writeSinglePage(row, written, statHolder); + written = writeSinglePage(row, written, statHolder, reserve); memMetrics.incrementLargeEntriesPages(); } @@ -741,13 +749,15 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC * @param row Row to write. * @param written Written size. * @param statHolder Statistics holder to track IO operations. + * @param reserve {@code true} to run the size-aware reserve before taking the page; must be {@code false} + * when the current thread holds a page write lock. * @return Number of bytes written, {@link #COMPLETE} if the row was fully written. * @throws IgniteCheckedException If failed. */ - private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { + private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { AbstractDataPageIO initIo = null; - long pageId = takePageWithReserve(row.size() - written, row, statHolder); + long pageId = takePageWithReserve(row.size() - written, row, statHolder, reserve); if (pageId == 0L) { pageId = allocateDataPage(row.partition()); @@ -968,6 +978,20 @@ public int emptyDataPages() { } } + /** + * Runs the size-aware reserve for an in-memory region with enabled page eviction; no-op otherwise. + * + * @param size Size of the row (or its remaining part) to be inserted. + * @throws IgniteCheckedException If failed to evict data pages. + */ + private void ensureFreeSpaceForInsert(int size) throws IgniteCheckedException { + if (dbMgr == null || dataRegion.config().isPersistenceEnabled() || + dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) + return; + + dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + } + /** {@inheritDoc} */ @Override public String toString() { return "FreeList [name=" + name() + ']'; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java index 9c91281ada7c1..86773c5997b51 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java @@ -209,6 +209,23 @@ public boolean tryWriteLock(long lock, int tag) { GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 0)); } + /** + * Attempts to acquire the read lock without ever blocking. + * + * @param lock Lock address. + * @param tag Validation tag. + * @return {@code True} if the read lock was acquired, {@code false} if the lock is held by a writer (or the tag + * validation failed) — the caller may retry later or skip the page. + */ + public boolean tryReadLock(long lock, int tag) { + long state = GridUnsafe.getLongVolatile(null, lock); + + assert state != 0; + + return checkTag(state, tag) && canReadLock(state) && + GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, 1, 0, 0)); + } + /** * @param lock Lock address. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index d41668e07fbd6..90eb9b7099c8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -22,13 +22,15 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; -/** Concurrent deadlock test for size-aware page eviction. */ +/** Concurrent deadlock and data-integrity tests for size-aware page eviction. */ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvictionAbstractTest { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { @@ -38,8 +40,6 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict /** * Concurrent large inserts into a region pre-filled with small entries must complete within the deadline without * deadlock, and without corrupting the free list (eviction frees small entries rather than overrunning the region). - * After the concurrent phase, the cache must remain functional: a new put+get must succeed, and at least some of - * the large entries written by concurrent threads must be readable. * * @throws Exception If failed. */ @@ -56,10 +56,10 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { long regionMax = regionMaxSize(ignite); // Small value: sized so that 'smallEntries' of them fill ~80% of the region, staying under the 90% eviction threshold. - byte[] smallVal = new byte[(int)(regionMax * 0.8 / smallEntries)]; + int smallValSize = (int)(regionMax * 0.8 / smallEntries); for (int i = 0; i < smallEntries; i++) - cache.put(i, smallVal); + cache.put(i, valueFor(i, smallValSize)); assertFalse("Eviction must not have started during pre-fill", isEvictionsStarted(ignite)); @@ -67,15 +67,18 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { AtomicInteger threadIdx = new AtomicInteger(); // Large value: 5% of the region — large enough to trigger eviction during concurrent writes. - byte[] largeVal = new byte[(int)(regionMax / 20)]; + int largeValSize = (int)(regionMax / 20); IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { U.awaitQuiet(startLatch); int idx = threadIdx.getAndIncrement(); - for (int k = 0; k < largeRowsPerThread; k++) - cache.put(smallEntries + idx * largeRowsPerThread + k, largeVal); + for (int k = 0; k < largeRowsPerThread; k++) { + int key = smallEntries + idx * largeRowsPerThread + k; + + cache.put(key, valueFor(key, largeValSize)); + } }, threads, "paged-writer"); startLatch.countDown(); @@ -85,12 +88,10 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { assertTrue("Eviction must have started during concurrent large writes", isEvictionsStarted(ignite)); // Verify the free list is not corrupted: a new put+get must succeed. - byte[] probeVal = new byte[pageSize(ignite) - 200]; - - Arrays.fill(probeVal, (byte)1); - int probeKey = smallEntries + threads * largeRowsPerThread + 1; + byte[] probeVal = valueFor(probeKey, pageSize(ignite) - 200); + cache.put(probeKey, probeVal); byte[] read = (byte[])cache.get(probeKey); @@ -98,17 +99,121 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { assertNotNull(read); assertTrue(Arrays.equals(probeVal, read)); - int readableLarge = 0; + verifyContent(cache, 0, smallEntries, smallValSize); - for (int t = 0; t < threads; t++) { - for (int k = 0; k < largeRowsPerThread; k++) { - int key = smallEntries + t * largeRowsPerThread + k; + int survivedLarge = verifyContent(cache, smallEntries, threads * largeRowsPerThread, largeValSize); + + assertTrue("At least one large entry must survive", survivedLarge > 0); + } + + /** + * Batch insert path test. + * + * @throws Exception If failed. + */ + @Test + public void testRebalanceBatchInsertsWithEviction() throws Exception { + IgniteEx ignite1 = startGrid(1); + + int largeValSize = (int)(regionMaxSize(ignite1) / 20); + int largeRows = 25; + + IgniteCache cache = ignite1.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)).setBackups(1)); + + for (int k = 0; k < largeRows; k++) + cache.put(k, valueFor(k, largeValSize)); + + byte[] smallVal = new byte[pageSize(ignite1) - 200]; + + int smallKeyBase = largeRows + 1; + + for (int i = 0; i < totalPages(ignite1); i++) { + cache.put(smallKeyBase + i, smallVal); + + if (isEvictionsStarted(ignite1)) + break; + } + + assertTrue("Eviction must start during overflow fill", isEvictionsStarted(ignite1)); + + IgniteEx ignite2 = startGrid(2); + + // Writers keep adding large rows on the coordinator while the joining node stores rebalance batches. + CountDownLatch startLatch = new CountDownLatch(1); + AtomicInteger threadIdx = new AtomicInteger(); + + int writers = 4; + int rowsPerWriter = 5; + + int writerKeyBase = smallKeyBase + (int)totalPages(ignite1); - if (cache.get(key) != null) - readableLarge++; + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { + U.awaitQuiet(startLatch); + + int idx = threadIdx.getAndIncrement(); + + for (int k = 0; k < rowsPerWriter; k++) { + int key = writerKeyBase + idx * rowsPerWriter + k; + + cache.put(key, valueFor(key, largeValSize)); } + }, writers, "paged-writer"); + + startLatch.countDown(); + + awaitPartitionMapExchange(); + + cache.rebalance().get(TimeUnit.MINUTES.toMillis(2)); + + fut.get(TimeUnit.MINUTES.toMillis(2)); + + assertTrue("Eviction must have started on the rebalanced node", isEvictionsStarted(ignite2)); + + int survived = verifyContent(cache, 0, largeRows, largeValSize) + + verifyContent(cache, writerKeyBase, writers * rowsPerWriter, largeValSize); + + assertTrue("At least one large entry must survive", survived > 0); + } + + /** + * @param key Key the value is stored under. + * @param size Value size in bytes. + * @return Deterministic per-key value content. + */ + private static byte[] valueFor(int key, int size) { + byte[] val = new byte[size]; + + for (int i = 0; i < size; i++) + val[i] = (byte)(key * 31 + i); + + return val; + } + + /** + * Reads {@code cnt} entries starting from {@code base} and asserts that every non-null value is byte-identical + * to the deterministic content written for its key. + * + * @param cache Cache to read from. + * @param base First key. + * @param cnt Number of keys to check. + * @param valSize Size of the values written for these keys. + * @return Number of entries that survived eviction. + */ + private static int verifyContent(IgniteCache cache, int base, int cnt, int valSize) { + int survived = 0; + + for (int k = base; k < base + cnt; k++) { + byte[] read = (byte[])cache.get(k); + + if (read == null) + continue; // Evicted. + + assertTrue("Content mismatch for key " + k, Arrays.equals(valueFor(k, valSize), read)); + + survived++; } - assertTrue(readableLarge > 0); + return survived; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java new file mode 100644 index 0000000000000..3e2c10d6f6a95 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.evict; + +import java.util.Arrays; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.PageUtils; +import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; +import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.testframework.junits.GridTestKernalContext; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; + +/** + * Verifies the evictor tolerates a fragment chain mixed from two rows: a stale link of a removed row crossing + * a chain of a new row that recycled a forgotten slot resolves to a link after two hops. The state is reachable + * by legal protocol operations (no data corruption); the tolerant handling untracks the stale slot and keeps + * sampling. + *

    + * Page data bytes are asserted untouched across the evictor pass; end-to-end row content integrity is covered by + * {@code PageEvictionConcurrentWritesAbstractTest}. + */ +public class PageEvictionTrackerFragmentChainSelfTest extends GridCommonAbstractTest { + /** Page size. */ + private static final int PAGE_SIZE = 1024; + + /** Data region max size — small on purpose: keeps the tracking array tiny so the stale slot gets sampled. */ + private static final long REGION_MAX_SIZE = 16 * 1024; + + /** Page index of the stale entry fragment of the removed row. */ + private static final int IDX_F = 1; + + /** Page index of the fragment of the removed row already forgotten and recycled by the new row. */ + private static final int IDX_T = 2; + + /** Page index of the tail fragment of the new row. */ + private static final int IDX_TAIL2 = 3; + + /** Page index of the head fragment of the new row. */ + private static final int IDX_HEAD2 = 4; + + /** + * Checks the mixed chain handling for {@link RandomLruPageEvictionTracker}. + * + * @throws Exception If failed. + */ + @Test + public void testStaleFragmentLinkAfterRecycleRandomLru() throws Exception { + checkStaleFragmentLinkAfterRecycle(false); + } + + /** + * Checks the mixed chain handling for {@link Random2LruPageEvictionTracker}. + * + * @throws Exception If failed. + */ + @Test + public void testStaleFragmentLinkAfterRecycleRandom2Lru() throws Exception { + checkStaleFragmentLinkAfterRecycle(true); + } + + /** + * @param random2Lru {@code true} to test {@link Random2LruPageEvictionTracker}, {@code false} for + * {@link RandomLruPageEvictionTracker}. + * @throws Exception If failed. + */ + private void checkStaleFragmentLinkAfterRecycle(boolean random2Lru) throws Exception { + DataRegionConfiguration plcCfg = new DataRegionConfiguration().setInitialSize(REGION_MAX_SIZE).setMaxSize(REGION_MAX_SIZE); + + PageMemoryNoStoreImpl pageMem = new PageMemoryNoStoreImpl(new NullLogger(), new UnsafeMemoryProvider(log()), null, PAGE_SIZE, + plcCfg, new DataRegionMetricsImpl(plcCfg, new GridTestKernalContext(log())), false); + + DataRegionConfiguration regionCfg = new DataRegionConfiguration() + .setName("fragment-chain-diag") + .setInitialSize(REGION_MAX_SIZE) + .setMaxSize(REGION_MAX_SIZE) + .setEmptyPagesPoolSize(10) + .setEvictionThreshold(0.9); + + PageAbstractEvictionTracker tracker = random2Lru ? + new Random2LruPageEvictionTracker(pageMem, regionCfg, sharedCtx()) : + new RandomLruPageEvictionTracker(pageMem, regionCfg, sharedCtx()); + + try { + pageMem.start(); + tracker.start(); + + for (int i = 0; i <= IDX_HEAD2; i++) + pageMem.allocatePage(0, 0, FLAG_DATA); + + // Fill the page bodies with a per-page pattern (keeping the first short non-data, so the evictor + // rejects the pages by its type check without parsing them): the evictor pass below must not + // modify page data bytes. + byte[][] pageData = new byte[IDX_HEAD2 + 1][]; + + for (int idx = 0; idx <= IDX_HEAD2; idx++) { + long absPtr = pageMem.acquirePage(0, pageId(idx)); + + try { + byte[] exp = new byte[PAGE_SIZE]; + + for (int i = 0; i < PAGE_SIZE; i++) + exp[i] = (byte)(idx * 31 + i); + + exp[0] = (byte)0xFF; + exp[1] = (byte)0xFF; + + PageUtils.putBytes(absPtr + PageMemoryNoStoreImpl.PAGE_OVERHEAD, 0, exp); + + pageData[idx] = exp; + } + finally { + pageMem.releasePage(0, pageId(idx), absPtr); + } + } + + tracker.trackFragmentPage(pageId(IDX_F), pageId(IDX_T), false); + + assertEquals("F must link to T", slot(tracker, IDX_T), tracker.getFragmentLink(slot(tracker, IDX_F))); + + tracker.forgetPage(pageId(IDX_T)); + tracker.trackFragmentPage(pageId(IDX_T), pageId(IDX_TAIL2), false); + + assertEquals("T must link to the new row tail", slot(tracker, IDX_TAIL2), tracker.getFragmentLink(slot(tracker, IDX_T))); + + tracker.trackFragmentPage(pageId(IDX_HEAD2), pageId(IDX_TAIL2), true); + + assertTrue("The new row head must be tracked", tracker.checkTouch(pageId(IDX_HEAD2))); + assertFalse("No row can be evicted from unwritten pages", tracker.evictDataPage(true)); + assertEquals("The sampled stale slot must be untracked", 0, tracker.getFragmentLink(slot(tracker, IDX_F))); + + assertEquals("The live row fragment link must stay intact", + slot(tracker, IDX_TAIL2), tracker.getFragmentLink(slot(tracker, IDX_T))); + + assertTrue("The live row head must stay tracked", tracker.checkTouch(pageId(IDX_HEAD2))); + + assertFalse(tracker.evictDataPage(true)); + + // The evictor pass must not touch page data bytes. + for (int idx = 0; idx <= IDX_HEAD2; idx++) { + long absPtr = pageMem.acquirePage(0, pageId(idx)); + + try { + byte[] actual = PageUtils.getBytes(absPtr + PageMemoryNoStoreImpl.PAGE_OVERHEAD, 0, PAGE_SIZE); + + assertTrue("Page " + idx + " bytes must stay untouched by the evictor", Arrays.equals(pageData[idx], actual)); + } + finally { + pageMem.releasePage(0, pageId(idx), absPtr); + } + } + } + finally { + tracker.stop(); + pageMem.stop(true); + } + } + + /** + * @param tracker Tracker. + * @param pageIdx Page index. + * @return Tracking array slot of the page. + */ + private static int slot(PageAbstractEvictionTracker tracker, int pageIdx) { + return tracker.trackingIdx(pageIdx); + } + + /** + * @param pageIdx Page index. + * @return Fabricated data page ID. + */ + private static long pageId(int pageIdx) { + return PageIdUtils.pageId(0, FLAG_DATA, pageIdx); + } + + /** @return Minimal shared context for the tracker constructor. */ + private GridCacheSharedContext sharedCtx() { + GridTestKernalContext kernalCtx = new GridTestKernalContext(log()); + + kernalCtx.config().setDataStorageConfiguration(new DataStorageConfiguration().setPageSize(PAGE_SIZE)); + + GridCacheSharedContext sharedCtx = Mockito.mock(GridCacheSharedContext.class); + + Mockito.when(sharedCtx.kernalContext()).thenReturn(kernalCtx); + Mockito.when(sharedCtx.logger((Class)Mockito.any())).thenReturn(new NullLogger()); + + return sharedCtx; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index 0b240cf0f23dd..258eeb597efa9 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -61,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTrackerFragmentChainSelfTest; import org.apache.ignite.testframework.GridTestUtils; /** @@ -122,6 +123,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, PageEvictionPagesRecyclingAndReusingTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionTrackerFragmentChainSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DhtAndNearEvictionTest.class, ignoredTests); return suite; From d40146f092a0bf772d2a1f998f29e6a6a26737b7 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 13 Oct 2026 21:56:05 +0500 Subject: [PATCH 23/26] move --- .../processors/cache/GridCacheMapEntry.java | 2 +- .../IgniteCacheDatabaseSharedManager.java | 200 ------------ .../cache/persistence/RowStore.java | 4 +- .../freelist/AbstractFreeList.java | 290 ++++++++++++++---- .../cache/persistence/freelist/FreeList.java | 10 + 5 files changed, 245 insertions(+), 261 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index f99a3289561a2..536b72a27648a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3624,7 +3624,7 @@ protected void removeValue() throws IgniteCheckedException { private void ensureFreeSpace() throws IgniteCheckedException { // Deadlock alert: evicting a data page removes (and locks) all entries on the page one by one, so this must // only run while NOT holding this entry's lock (all call sites run before lockEntry()). The size-aware path - // (RowStore.addRow -> ensureFreeSpaceForInsert) runs under the lock and instead uses the non-blocking + // (RowStore.addRow -> FreeList.ensureFreeSpace) runs under the lock and instead uses the non-blocking // tryLockEntry(0) inside evictInternal. assert !lock.isHeldByCurrentThread(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 787538ade9eff..0a822048cfd0b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -26,8 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.LockSupport; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -41,15 +39,12 @@ import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.WarmUpConfiguration; -import org.apache.ignite.failure.FailureContext; -import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; -import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider; import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; import org.apache.ignite.internal.pagemem.PageIdAllocator; @@ -76,7 +71,6 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageReadWriteManager; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.warmup.WarmUpStrategy; @@ -141,24 +135,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** Maximum initial size on 32-bit JVM */ private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024; - /** Initial backoff (nanoseconds) between consecutive no-progress eviction attempts. */ - private static final long EVICTION_BACKOFF_START_NANOS = 50_000L; - - /** Upper bound (nanoseconds) for the backoff between no-progress eviction attempts. */ - private static final long EVICTION_BACKOFF_MAX_NANOS = 1_000_000L; - - /** - * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without either an eviction or a - * new highest empty-pages count before failing with an out-of-memory. - */ - private static final long EVICTION_NO_PROGRESS_TIMEOUT_MILLIS = 1_000L; - - /** - * Hard upper bound (milliseconds) on the total duration of one size-aware eviction cycle, regardless of - * per-iteration progress. - */ - private static final long EVICTION_MAX_CYCLE_TIME_MILLIS = 5 * EVICTION_NO_PROGRESS_TIMEOUT_MILLIS; - /** {@code True} to reuse memory on deactive. */ protected final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); @@ -1181,161 +1157,6 @@ public WALPointer latestWalPointerReservedForPreloading() { return null; } - /** - * Checks that the given {@code region} has enough space for putting a new entry of {@code dataRowSize} bytes. - *

    - * For a non-persistent region with page eviction disabled, verifies that the region reserves enough pages to - * support a free list {@link AbstractFreeList}. For example, removing a row from underlying store may require - * allocating a new data page in order to move a tracked page from one bucket to another one which does not have - * a free space for a new stripe. See {@link AbstractFreeList#removeDataRowByLink}. Therefore, inserting a new - * entry should be prevented in case of some threshold is exceeded. - *

    - * For a non-persistent region with page eviction enabled, additionally performs size-aware eviction: when the - * row does not fit into the currently available page space, data pages are evicted until either enough space is - * freed or it becomes clear that the goal is unreachable (in which case an - * {@link IgniteOutOfMemoryException} is thrown). - * - * @param region Data region to be checked. - * @param dataRowSize Size of data row to be inserted. - * @throws IgniteOutOfMemoryException In case the given data region does not have enough free space - * for putting a new entry, even after eviction. - * @throws IgniteCheckedException If failed to evict data pages. - */ - public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { - if (region == null) - return; - - DataRegionConfiguration regCfg = region.config(); - - if (regCfg.isPersistenceEnabled()) - return; - - if (regCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED) - checkOomThreshold(region, regCfg, dataRowSize); - else - ensureFreeSpaceForEviction(region, regCfg, dataRowSize); - } - - /** - * Checks that a non-persistent region with disabled page eviction has enough pages for a new row, taking into - * account the pages required to support the free list. - * - * @param region Data region. - * @param regCfg Data region configuration. - * @param dataRowSize Size of data row to be inserted. - * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. - */ - private void checkOomThreshold(DataRegion region, DataRegionConfiguration regCfg, int dataRowSize) throws IgniteOutOfMemoryException { - PageMemory pageMem = region.pageMemory(); - - long nonEmptyPages = (pageMem.loadedPages() - freeListMap.get(regCfg.getName()).emptyDataPages()); - - // The maximum number of pages that can be allocated (memorySize / systemPageSize) - // should be greater or equal to pages required for inserting a new entry plus - // the current number of non-empty pages plus the number of pages that may be required in order to move - // all pages to a reuse bucket, that is equal to nonEmptyPages * 8 / pageSize, where 8 is the size of a link. - // Note that not the whole page can be used to storing links, - // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, - // in any way, the number of required pages is less than 1 percent. - boolean oomThreshold = (regCfg.getMaxSize() / pageMem.systemPageSize()) < - ((double)dataRowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); - - if (oomThreshold) - throw outOfMemory(regCfg); - } - - /** - * Size-aware reserve for an eviction-enabled non-persistent region. Runs eviction until the combined space - * (empty pages in the free list + unallocated headroom) is enough to accommodate the row plus the - * {@link DataRegionConfiguration#getEmptyPagesPoolSize() empty-pages pool}, or throws - * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. The pool margin is kept - * on top of the row's own requirement so that, after this thread consumes its pages, at least {@code poolSize} - * pages remain for concurrent writers; this makes the path safe for up to {@code poolSize} concurrent threads. - * Both empty pages and headroom are treated as concurrent resources: the fast path returns as soon as - * their sum covers the target, and the eviction loop only evicts the deficit. - * - * @param region Data region. - * @param regCfg Data region configuration. - * @param dataRowSize Size of data row to be inserted. - * @throws IgniteOutOfMemoryException If the target cannot be reached (row too large for the region or eviction - * makes no progress). - * @throws IgniteCheckedException If failed to evict data pages. - */ - private void ensureFreeSpaceForEviction( - DataRegion region, - DataRegionConfiguration regCfg, - int dataRowSize - ) throws IgniteOutOfMemoryException, IgniteCheckedException { - PageMemory pageMem = region.pageMemory(); - - long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - - CacheFreeList freeList = freeListMap.get(regCfg.getName()); - - if (freeList == null) - return; - - long totalPages = regCfg.getMaxSize() / pageMem.systemPageSize(); - - long requiredPages = (dataRowSize + pagePayload - 1) / pagePayload; - - if (requiredPages > totalPages) - throw outOfMemory(regCfg); - - long targetPages = Math.min(requiredPages + regCfg.getEmptyPagesPoolSize(), totalPages); - - long emptyPages = freeList.emptyDataPages(); - - // Effective headroom accounts for per-segment allocation overhead (up to one page per segment), - // so the fast path does not trust headroom that cannot actually be allocated. - if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= targetPages) - return; - - PageEvictionTracker evictionTracker = region.evictionTracker(); - - long bestEmptyPages = emptyPages; - - long lastProgressNanos = System.nanoTime(); - - long cycleDeadlineNanos = lastProgressNanos + TimeUnit.MILLISECONDS.toNanos(EVICTION_MAX_CYCLE_TIME_MILLIS); - - long backoffNanos = EVICTION_BACKOFF_START_NANOS; - - while (freeList.emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < targetPages) { - if (region.metrics().onPageEvictionsStarted()) { - U.warn(log, "Page-based evictions started." + - " Consider increasing 'maxSize' on Data Region configuration: " + regCfg.getName()); - } - - boolean evicted = evictionTracker.evictDataPage(true); - - region.metrics().updateEvictionRate(); - - long curEmptyPages = freeList.emptyDataPages(); - - if (evicted || curEmptyPages > bestEmptyPages) { - if (curEmptyPages > bestEmptyPages) - bestEmptyPages = curEmptyPages; - - lastProgressNanos = System.nanoTime(); - - backoffNanos = EVICTION_BACKOFF_START_NANOS; - } - else { - LockSupport.parkNanos(backoffNanos); - - backoffNanos = Math.min(backoffNanos << 1, EVICTION_BACKOFF_MAX_NANOS); - } - - long nowNanos = System.nanoTime(); - - long maxNoProgress = TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS); - - if (nowNanos - lastProgressNanos > maxNoProgress || nowNanos > cycleDeadlineNanos) - throw outOfMemory(regCfg); - } - } - /** * Computes the effective headroom (unallocated pages that can actually be allocated), accounting for per-segment * allocation overhead (up to one page per segment). @@ -1350,27 +1171,6 @@ private static long effectiveHeadroom(long totalPages, long loadedPages) { return Math.max(0, effectiveTotal - loadedPages); } - /** - * @param regCfg Data region configuration. - * @return New {@link IgniteOutOfMemoryException} (also reported as a critical failure) for the given region. - */ - private IgniteOutOfMemoryException outOfMemory(DataRegionConfiguration regCfg) { - IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + - "name=" + regCfg.getName() + - ", initSize=" + U.readableSize(regCfg.getInitialSize(), false) + - ", maxSize=" + U.readableSize(regCfg.getMaxSize(), false) + - ", persistenceEnabled=" + regCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + - " ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + - " ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + - " ^-- Enable eviction or expiration policies" - ); - - if (cctx.kernalContext() != null) - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); - - return oom; - } - /** * See {@code GridCacheMapEntry#ensureFreeSpace()} * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 916496363036d..6b13cea999ce2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -106,7 +106,7 @@ public void removeRow(long link, IoStatisticsHolder statHolder) throws IgniteChe */ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled) { - ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), row.size()); + freeList.ensureFreeSpace(row.size()); freeList.insertDataRow(row, statHolder); } @@ -152,7 +152,7 @@ public void addRows(Collection rows, IoStatisticsHolder } if (maxRowSize > 0) - ctx.database().ensureFreeSpaceForInsert(grp.dataRegion(), maxRowSize); + freeList.ensureFreeSpace(maxRowSize); } assert ctx.database().checkpointLockIsHeldByThread(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index a34d0c319b706..aaf2681b22a19 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -19,11 +19,16 @@ import java.util.Arrays; import java.util.Collection; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.concurrent.locks.LockSupport; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.metric.IoStatisticsHolder; @@ -38,7 +43,6 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; @@ -57,6 +61,7 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.SEG_CNT; /** */ @@ -79,6 +84,24 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private static final int MIN_PAGE_FREE_SPACE = 8; + /** Initial backoff (nanoseconds) between consecutive no-progress eviction attempts. */ + private static final long EVICTION_BACKOFF_START_NANOS = 50_000L; + + /** Upper bound (nanoseconds) for the backoff between no-progress eviction attempts. */ + private static final long EVICTION_BACKOFF_MAX_NANOS = 1_000_000L; + + /** + * Maximum time (milliseconds) the size-aware eviction guard is willing to wait without either an eviction or a + * new highest empty-pages count before failing with an out-of-memory. + */ + private static final long EVICTION_NO_PROGRESS_TIMEOUT_MILLIS = 1_000L; + + /** + * Hard upper bound (milliseconds) on the total duration of one size-aware eviction cycle, regardless of + * per-iteration progress. + */ + private static final long EVICTION_MAX_CYCLE_TIME_MILLIS = 5 * EVICTION_NO_PROGRESS_TIMEOUT_MILLIS; + /** * Step between buckets in free list, measured in powers of two. * For example, for page size 4096 and 256 buckets, shift is 4 and step is 16 bytes. @@ -103,11 +126,11 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; - /** Data region this free list belongs to (used for proactive size-aware reserve on fragmented writes). */ + /** Data region this free list belongs to. */ private final DataRegion dataRegion; - /** Database shared manager (used for proactive size-aware reserve on fragmented writes). */ - private final IgniteCacheDatabaseSharedManager dbMgr; + /** Kernal context. */ + private final GridKernalContext ctx; /** Page list cache limit. */ private final AtomicLong pageListCacheLimit; @@ -472,11 +495,9 @@ public AbstractFreeList( rmvRow = new RemoveRowHandler(cacheGrpId == 0); - this.evictionTracker = dataRegion.evictionTracker(); + evictionTracker = dataRegion.evictionTracker(); this.dataRegion = dataRegion; - // dbMgr is needed only for the proactive reserve (eviction-enabled in-memory region); null in unit tests - // without a cache processor (where eviction is disabled and the reserve never fires). - dbMgr = ctx.cache() == null ? null : ctx.cache().context().database(); + this.ctx = ctx; this.reuseList = reuseList == null ? this : reuseList; int pageSize = pageMem.pageSize(); @@ -497,7 +518,7 @@ public AbstractFreeList( this.shift = shift; - this.memMetrics = dataRegion.metrics(); + memMetrics = dataRegion.metrics(); this.pageListCacheLimit = pageListCacheLimit; @@ -595,50 +616,6 @@ private int bucket(int freeSpace, boolean allowReuse) { return freeSpace > MIN_PAGE_FREE_SPACE ? bucket(freeSpace, false) : -1; } - /** - * @param part Partition. - * @return Page ID. - * @throws IgniteCheckedException If failed. - */ - private long allocateDataPage(int part) throws IgniteCheckedException { - assert part <= PageIdAllocator.MAX_PARTITION_ID; - - return pageMem.allocatePage(grpId, part, FLAG_DATA); - } - - /** - * Takes a page, optionally running the size-aware reserve first (see {@link #ensureFreeSpaceForInsert(int)}). - * The reserve targets {@code requiredPages + emptyPagesPoolSize}, so after this thread consumes its pages - * at least {@code emptyPagesPoolSize} pages remain for concurrent writers. - * - * @param size Free space required on the page. - * @param row Row to write. - * @param statHolder Statistics holder to track IO operations. - * @param reserve {@code true} to run the size-aware reserve before taking the page. Must be {@code false} - * when the current thread already holds a page write lock: the reserve may evict, and eviction takes a - * blocking read lock on the candidate data page, while {@link OffheapReadWriteLock} is not reentrant - * (see {@link PageAbstractEvictionTracker}). - * @return Page identifier or 0 if no page could be obtained after reserving. - * @throws IgniteCheckedException If failed. - */ - private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { - if (reserve) { - ensureFreeSpaceForInsert(size); - - long pageId = takePage(size, row, statHolder); - - if (pageId == 0L) { - ensureFreeSpaceForInsert(size); - - pageId = takePage(size, row, statHolder); - } - - return pageId; - } - - return takePage(size, row, statHolder); - } - /** {@inheritDoc} */ @Override public void insertDataRow(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { int written = 0; @@ -978,6 +955,66 @@ public int emptyDataPages() { } } + /** {@inheritDoc} */ + @Override public void ensureFreeSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { + if (dataRegion.config().isPersistenceEnabled()) + return; + + if (dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) + checkOomThreshold(rowSize); + else + reserveSpaceForInsert(rowSize); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "FreeList [name=" + name() + ']'; + } + + /** + * @param part Partition. + * @return Page ID. + * @throws IgniteCheckedException If failed. + */ + private long allocateDataPage(int part) throws IgniteCheckedException { + assert part <= PageIdAllocator.MAX_PARTITION_ID; + + return pageMem.allocatePage(grpId, part, FLAG_DATA); + } + + /** + * Takes a page, optionally running the size-aware reserve first (see {@link #ensureFreeSpaceForInsert(int)}). + * The reserve targets {@code requiredPages + emptyPagesPoolSize}, so after this thread consumes its pages + * at least {@code emptyPagesPoolSize} pages remain for concurrent writers. + * + * @param size Free space required on the page. + * @param row Row to write. + * @param statHolder Statistics holder to track IO operations. + * @param reserve {@code true} to run the size-aware reserve before taking the page. Must be {@code false} + * when the current thread already holds a page write lock: the reserve may evict, and eviction takes a + * blocking read lock on the candidate data page, while {@link OffheapReadWriteLock} is not reentrant + * (see {@link PageAbstractEvictionTracker}). + * @return Page identifier or 0 if no page could be obtained after reserving. + * @throws IgniteCheckedException If failed. + */ + private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { + if (reserve) { + ensureFreeSpaceForInsert(size); + + long pageId = takePage(size, row, statHolder); + + if (pageId == 0L) { + ensureFreeSpaceForInsert(size); + + pageId = takePage(size, row, statHolder); + } + + return pageId; + } + + return takePage(size, row, statHolder); + } + /** * Runs the size-aware reserve for an in-memory region with enabled page eviction; no-op otherwise. * @@ -985,15 +1022,152 @@ public int emptyDataPages() { * @throws IgniteCheckedException If failed to evict data pages. */ private void ensureFreeSpaceForInsert(int size) throws IgniteCheckedException { - if (dbMgr == null || dataRegion.config().isPersistenceEnabled() || + if (dataRegion.config().isPersistenceEnabled() || dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) return; - dbMgr.ensureFreeSpaceForInsert(dataRegion, size); + reserveSpaceForInsert(size); } - /** {@inheritDoc} */ - @Override public String toString() { - return "FreeList [name=" + name() + ']'; + /** + * Size-aware reserve for an in-memory region with enabled page eviction. Runs eviction until the combined space + * (empty pages in this free list + unallocated headroom) is enough to accommodate the row plus the + * {@link DataRegionConfiguration#getEmptyPagesPoolSize() empty-pages pool}, or throws + * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. The pool margin is kept + * on top of the row's own requirement so that, after this thread consumes its pages, at least {@code poolSize} + * pages remain for concurrent writers; this makes the path safe for up to {@code poolSize} concurrent threads. + * Both empty pages and headroom are treated as concurrent resources: the fast path returns as soon as + * their sum covers the target, and the eviction loop only evicts the deficit. + * + * @param rowSize Size of the row (or its remaining part) to be inserted. + * @throws IgniteOutOfMemoryException If the target cannot be reached (row too large for the region or eviction + * makes no progress). + * @throws IgniteCheckedException If failed to evict data pages. + */ + private void reserveSpaceForInsert(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { + if (evictionTracker == null) + return; + + DataRegionConfiguration cfg = dataRegion.config(); + + long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; + + long totalPages = cfg.getMaxSize() / pageMem.systemPageSize(); + + long requiredPages = (rowSize + pagePayload - 1) / pagePayload; + + if (requiredPages > totalPages) + throw outOfMemory(cfg); + + long targetPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); + + long emptyPages = emptyDataPages(); + + // Effective headroom accounts for per-segment allocation overhead (up to one page per segment), + // so the fast path does not trust headroom that cannot actually be allocated. + if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= targetPages) + return; + + long bestEmptyPages = emptyPages; + + long lastProgressNanos = System.nanoTime(); + + long cycleDeadlineNanos = lastProgressNanos + TimeUnit.MILLISECONDS.toNanos(EVICTION_MAX_CYCLE_TIME_MILLIS); + + long backoffNanos = EVICTION_BACKOFF_START_NANOS; + + while (emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < targetPages) { + if (memMetrics.onPageEvictionsStarted()) { + U.warn(log, "Page-based evictions started." + + " Consider increasing 'maxSize' on Data Region configuration: " + cfg.getName()); + } + + boolean evicted = evictionTracker.evictDataPage(true); + + memMetrics.updateEvictionRate(); + + long curEmptyPages = emptyDataPages(); + + if (evicted || curEmptyPages > bestEmptyPages) { + if (curEmptyPages > bestEmptyPages) + bestEmptyPages = curEmptyPages; + + lastProgressNanos = System.nanoTime(); + + backoffNanos = EVICTION_BACKOFF_START_NANOS; + } + else { + LockSupport.parkNanos(backoffNanos); + + backoffNanos = Math.min(backoffNanos << 1, EVICTION_BACKOFF_MAX_NANOS); + } + + long nowNanos = System.nanoTime(); + + long maxNoProgress = TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS); + + if (nowNanos - lastProgressNanos > maxNoProgress || nowNanos > cycleDeadlineNanos) + throw outOfMemory(cfg); + } + } + + /** + * Checks that an in-memory region with disabled page eviction has enough pages for a new row, taking into + * account the pages required to support the free list. + * + * @param rowSize Size of the row to be inserted. + * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. + */ + private void checkOomThreshold(int rowSize) { + long nonEmptyPages = pageMem.loadedPages() - emptyDataPages(); + + // The maximum number of pages that can be allocated (maxSize / systemPageSize) + // should be greater or equal to pages required for inserting a new entry plus + // the number of currently non-empty data pages. Also we account for pages that hold pages-list metadata: + // 8 bytes per each row on average, considering that maybe not all list pages are full, multiply by 1.5. + // Plus, add one page for each bucket in the worst case. + // Note that not the whole page can be used to storing links, + // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, + // in any way, the number of required pages is less than 1 percent. + boolean oomThreshold = (dataRegion.config().getMaxSize() / pageMem.systemPageSize()) < + ((double)rowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); + + if (oomThreshold) + throw outOfMemory(dataRegion.config()); + } + + /** + * Computes the effective headroom (unallocated pages that can actually be allocated), accounting for per-segment + * allocation overhead (up to one page per segment). Mirrors the same-named helper in + * IgniteCacheDatabaseSharedManager. + * + * @param totalPages Total pages in the region (maxSize / systemPageSize). + * @param loadedPages Currently loaded pages. + * @return Effective headroom, never negative. + */ + private static long effectiveHeadroom(long totalPages, long loadedPages) { + long effectiveTotal = totalPages > SEG_CNT ? totalPages - SEG_CNT : 0; + + return Math.max(0, effectiveTotal - loadedPages); + } + + /** + * @param cfg Data region configuration. + * @return New {@link IgniteOutOfMemoryException} (also reported as a critical failure) for the given region. + */ + private IgniteOutOfMemoryException outOfMemory(DataRegionConfiguration cfg) { + IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + + "name=" + cfg.getName() + + ", initSize=" + U.readableSize(cfg.getInitialSize(), false) + + ", maxSize=" + U.readableSize(cfg.getMaxSize(), false) + + ", persistenceEnabled=" + cfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + + " ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + + " ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + + " ^-- Enable eviction or expiration policies" + ); + + ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); + + return oom; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java index 7d492fd46f8a3..2ea0e4a55d2b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java @@ -20,6 +20,7 @@ import java.util.Collection; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.metric.IoStatisticsHolder; import org.apache.ignite.internal.processors.cache.persistence.Storable; @@ -56,6 +57,15 @@ public interface FreeList { */ public void removeDataRowByLink(long link, IoStatisticsHolder statHolder) throws IgniteCheckedException; + /** + * Ensures the data region backing this free list can accommodate a row of the given size. + * + * @param rowSize Size of the row to be inserted. + * @throws IgniteOutOfMemoryException If the row cannot be accommodated even after eviction. + * @throws IgniteCheckedException If failed to evict data pages. + */ + public void ensureFreeSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException; + /** * @param log Logger. */ From ba0af22b58a2451980321e7e072187df63ac6f69 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 13 Oct 2026 23:54:55 +0500 Subject: [PATCH 24/26] wip --- .../pagemem/impl/PageMemoryNoStoreImpl.java | 17 -- .../IgniteCacheDatabaseSharedManager.java | 1 - .../evict/PageAbstractEvictionTracker.java | 5 +- .../evict/Random2LruPageEvictionTracker.java | 14 +- .../evict/RandomLruPageEvictionTracker.java | 14 +- .../freelist/AbstractFreeList.java | 35 ++- .../internal/util/OffheapReadWriteLock.java | 17 -- ...eEvictionTrackerFragmentChainSelfTest.java | 212 ------------------ .../IgniteCacheEvictionSelfTestSuite.java | 3 - 9 files changed, 30 insertions(+), 288 deletions(-) delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index 7558ab8cb97cf..d9bd2b8bacaf0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -525,23 +525,6 @@ private long fromSegmentIndex(int segIdx, long pageIdx) { return 0L; } - /** - * Attempts to acquire the read lock regardless of the page tag, without ever blocking. - * - * @param cacheId Cache group ID. - * @param pageId Page ID. - * @param page Page absolute pointer. - * @return Page address or {@code 0L} if the page is currently write-locked. - */ - public long tryReadLock(int cacheId, long pageId, long page) { - assert started; - - if (rwLock.tryReadLock(page + LOCK_OFFSET, -1)) - return page + PAGE_OVERHEAD; - - return 0L; - } - /** {@inheritDoc} */ @Override public void readUnlock(int cacheId, long pageId, long page) { assert started; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 0a822048cfd0b..9ac6cee62c65c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -65,7 +65,6 @@ import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; -import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index 700151c3aff2b..d97a2b95e1c64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -104,10 +104,7 @@ final boolean evictDataPage(int pageIdx, boolean tryLock) throws IgniteCheckedEx List rowsToEvict; try { - long pageAddr = pageMem.tryReadLock(0, fakePageId, page); - - if (pageAddr == 0L) - return false; // Candidate page is write-locked: skip it. + long pageAddr = pageMem.readLockForce(0, fakePageId, page); try { if (PageIO.getType(pageAddr) != PageIO.T_DATA) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index df60e83be309f..57df87bb7eddb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -140,11 +140,7 @@ public Random2LruPageEvictionTracker( while (dataPagesCnt < SAMPLE_SIZE) { int trackingIdx = rnd.nextInt(trackingSize); - int origTrackingIdx = trackingIdx; - - long origTrackingData = GridUnsafe.getLongVolatile(null, trackingArrPtr + origTrackingIdx * 8L); - - long trackingData = origTrackingData; + long trackingData = GridUnsafe.getLongVolatile(null, trackingArrPtr + trackingIdx * 8L); int firstTs = first(trackingData); @@ -165,9 +161,11 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); + // The walk may start from a stale pointer: fragment-chain pages are continuously recycled + // and their tracking slots re-linked by newer rows, which recreates identical link values + // across slot generations (ABA). Every recycling generation deepens the chain indirection, + // so two hops can legitimately land on a link slot of a newer chain. Skip the slot and keep sampling. if (firstTs < 0) { - GridUnsafe.compareAndSwapLong(null, trackingArrPtr + origTrackingIdx * 8L, origTrackingData, 0L); - sampleSpinCnt++; if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { @@ -178,6 +176,8 @@ public Random2LruPageEvictionTracker( continue; } + + assert firstTs >= 0 : "[firstTs=" + firstTs + ", secondTs=" + second(trackingData) + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index 6d8c1378a0ee2..f23abb233263b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -126,11 +126,7 @@ public RandomLruPageEvictionTracker( while (dataPagesCnt < SAMPLE_SIZE) { int sampleTrackingIdx = rnd.nextInt(trackingSize); - int origSampleIdx = sampleTrackingIdx; - - int origSampleData = GridUnsafe.getIntVolatile(null, trackingArrPtr + origSampleIdx * 4L); - - int compactTs = origSampleData; + int compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); if (compactTs < 0) { // For page containing fragmented row data timestamps stored in the row's head page are used. @@ -145,9 +141,11 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); + // The walk may start from a stale pointer: fragment-chain pages are continuously recycled + // and their tracking slots re-linked by newer rows, which recreates identical link values + // across slot generations (ABA). Every recycling generation deepens the chain indirection, + // so two hops can legitimately land on a link slot of a newer chain. Skip the slot and keep sampling. if (compactTs < 0) { - GridUnsafe.compareAndSwapInt(null, trackingArrPtr + origSampleIdx * 4L, origSampleData, 0); - sampleSpinCnt++; if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) { @@ -158,6 +156,8 @@ public RandomLruPageEvictionTracker( continue; } + + assert compactTs >= 0 : "[compactTs=" + compactTs + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index aaf2681b22a19..0d5853436619e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -659,6 +659,11 @@ private int bucket(int freeSpace, boolean allowReuse) { // If eviction is required - free up memory before locking the next page. while (evictionTracker.evictionRequired()) { + if (memMetrics.onPageEvictionsStarted()) { + U.warn(log, "Page-based evictions started." + + " Consider increasing 'maxSize' on Data Region configuration: " + dataRegion.config().getName()); + } + evictionTracker.evictDataPage(); memMetrics.updateEvictionRate(); @@ -839,8 +844,6 @@ private long takePage(int size, T row, IoStatisticsHolder statHolder) throws Ign nextLink = write(pageId, rmvRow, bag, itemId, FAIL_L, statHolder); assert nextLink != FAIL_L; // Can't fail here. - - evictionTracker.forgetPage(pageId); } reuseList.addForRecycle(bag); @@ -983,9 +986,14 @@ private long allocateDataPage(int part) throws IgniteCheckedException { } /** - * Takes a page, optionally running the size-aware reserve first (see {@link #ensureFreeSpaceForInsert(int)}). + * Takes a page, optionally running the size-aware reserve first. * The reserve targets {@code requiredPages + emptyPagesPoolSize}, so after this thread consumes its pages * at least {@code emptyPagesPoolSize} pages remain for concurrent writers. + *

    + * The reserve is a check-then-act on shared counters: concurrent writers may drain the free list and the + * allocation headroom between the gate check and {@link #takePage}. If {@code takePage} finds no page, the + * reserve is re-evaluated on fresh state and given a second chance to evict before the caller falls back to + * allocating a brand-new page, which fails with a raw out-of-memory once the region is physically exhausted. * * @param size Free space required on the page. * @param row Row to write. @@ -998,13 +1006,14 @@ private long allocateDataPage(int part) throws IgniteCheckedException { * @throws IgniteCheckedException If failed. */ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { - if (reserve) { - ensureFreeSpaceForInsert(size); + if (reserve && !dataRegion.config().isPersistenceEnabled() && + !(dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED)) { + reserveSpaceForInsert(size); long pageId = takePage(size, row, statHolder); if (pageId == 0L) { - ensureFreeSpaceForInsert(size); + reserveSpaceForInsert(size); pageId = takePage(size, row, statHolder); } @@ -1015,20 +1024,6 @@ private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, return takePage(size, row, statHolder); } - /** - * Runs the size-aware reserve for an in-memory region with enabled page eviction; no-op otherwise. - * - * @param size Size of the row (or its remaining part) to be inserted. - * @throws IgniteCheckedException If failed to evict data pages. - */ - private void ensureFreeSpaceForInsert(int size) throws IgniteCheckedException { - if (dataRegion.config().isPersistenceEnabled() || - dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) - return; - - reserveSpaceForInsert(size); - } - /** * Size-aware reserve for an in-memory region with enabled page eviction. Runs eviction until the combined space * (empty pages in this free list + unallocated headroom) is enough to accommodate the row plus the diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java index 86773c5997b51..9c91281ada7c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java @@ -209,23 +209,6 @@ public boolean tryWriteLock(long lock, int tag) { GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 0)); } - /** - * Attempts to acquire the read lock without ever blocking. - * - * @param lock Lock address. - * @param tag Validation tag. - * @return {@code True} if the read lock was acquired, {@code false} if the lock is held by a writer (or the tag - * validation failed) — the caller may retry later or skip the page. - */ - public boolean tryReadLock(long lock, int tag) { - long state = GridUnsafe.getLongVolatile(null, lock); - - assert state != 0; - - return checkTag(state, tag) && canReadLock(state) && - GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, 1, 0, 0)); - } - /** * @param lock Lock address. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java deleted file mode 100644 index 3e2c10d6f6a95..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTrackerFragmentChainSelfTest.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.persistence.evict; - -import java.util.Arrays; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; -import org.apache.ignite.internal.pagemem.PageIdUtils; -import org.apache.ignite.internal.pagemem.PageUtils; -import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; -import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; -import org.apache.ignite.logger.NullLogger; -import org.apache.ignite.testframework.junits.GridTestKernalContext; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; -import org.mockito.Mockito; - -import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; - -/** - * Verifies the evictor tolerates a fragment chain mixed from two rows: a stale link of a removed row crossing - * a chain of a new row that recycled a forgotten slot resolves to a link after two hops. The state is reachable - * by legal protocol operations (no data corruption); the tolerant handling untracks the stale slot and keeps - * sampling. - *

    - * Page data bytes are asserted untouched across the evictor pass; end-to-end row content integrity is covered by - * {@code PageEvictionConcurrentWritesAbstractTest}. - */ -public class PageEvictionTrackerFragmentChainSelfTest extends GridCommonAbstractTest { - /** Page size. */ - private static final int PAGE_SIZE = 1024; - - /** Data region max size — small on purpose: keeps the tracking array tiny so the stale slot gets sampled. */ - private static final long REGION_MAX_SIZE = 16 * 1024; - - /** Page index of the stale entry fragment of the removed row. */ - private static final int IDX_F = 1; - - /** Page index of the fragment of the removed row already forgotten and recycled by the new row. */ - private static final int IDX_T = 2; - - /** Page index of the tail fragment of the new row. */ - private static final int IDX_TAIL2 = 3; - - /** Page index of the head fragment of the new row. */ - private static final int IDX_HEAD2 = 4; - - /** - * Checks the mixed chain handling for {@link RandomLruPageEvictionTracker}. - * - * @throws Exception If failed. - */ - @Test - public void testStaleFragmentLinkAfterRecycleRandomLru() throws Exception { - checkStaleFragmentLinkAfterRecycle(false); - } - - /** - * Checks the mixed chain handling for {@link Random2LruPageEvictionTracker}. - * - * @throws Exception If failed. - */ - @Test - public void testStaleFragmentLinkAfterRecycleRandom2Lru() throws Exception { - checkStaleFragmentLinkAfterRecycle(true); - } - - /** - * @param random2Lru {@code true} to test {@link Random2LruPageEvictionTracker}, {@code false} for - * {@link RandomLruPageEvictionTracker}. - * @throws Exception If failed. - */ - private void checkStaleFragmentLinkAfterRecycle(boolean random2Lru) throws Exception { - DataRegionConfiguration plcCfg = new DataRegionConfiguration().setInitialSize(REGION_MAX_SIZE).setMaxSize(REGION_MAX_SIZE); - - PageMemoryNoStoreImpl pageMem = new PageMemoryNoStoreImpl(new NullLogger(), new UnsafeMemoryProvider(log()), null, PAGE_SIZE, - plcCfg, new DataRegionMetricsImpl(plcCfg, new GridTestKernalContext(log())), false); - - DataRegionConfiguration regionCfg = new DataRegionConfiguration() - .setName("fragment-chain-diag") - .setInitialSize(REGION_MAX_SIZE) - .setMaxSize(REGION_MAX_SIZE) - .setEmptyPagesPoolSize(10) - .setEvictionThreshold(0.9); - - PageAbstractEvictionTracker tracker = random2Lru ? - new Random2LruPageEvictionTracker(pageMem, regionCfg, sharedCtx()) : - new RandomLruPageEvictionTracker(pageMem, regionCfg, sharedCtx()); - - try { - pageMem.start(); - tracker.start(); - - for (int i = 0; i <= IDX_HEAD2; i++) - pageMem.allocatePage(0, 0, FLAG_DATA); - - // Fill the page bodies with a per-page pattern (keeping the first short non-data, so the evictor - // rejects the pages by its type check without parsing them): the evictor pass below must not - // modify page data bytes. - byte[][] pageData = new byte[IDX_HEAD2 + 1][]; - - for (int idx = 0; idx <= IDX_HEAD2; idx++) { - long absPtr = pageMem.acquirePage(0, pageId(idx)); - - try { - byte[] exp = new byte[PAGE_SIZE]; - - for (int i = 0; i < PAGE_SIZE; i++) - exp[i] = (byte)(idx * 31 + i); - - exp[0] = (byte)0xFF; - exp[1] = (byte)0xFF; - - PageUtils.putBytes(absPtr + PageMemoryNoStoreImpl.PAGE_OVERHEAD, 0, exp); - - pageData[idx] = exp; - } - finally { - pageMem.releasePage(0, pageId(idx), absPtr); - } - } - - tracker.trackFragmentPage(pageId(IDX_F), pageId(IDX_T), false); - - assertEquals("F must link to T", slot(tracker, IDX_T), tracker.getFragmentLink(slot(tracker, IDX_F))); - - tracker.forgetPage(pageId(IDX_T)); - tracker.trackFragmentPage(pageId(IDX_T), pageId(IDX_TAIL2), false); - - assertEquals("T must link to the new row tail", slot(tracker, IDX_TAIL2), tracker.getFragmentLink(slot(tracker, IDX_T))); - - tracker.trackFragmentPage(pageId(IDX_HEAD2), pageId(IDX_TAIL2), true); - - assertTrue("The new row head must be tracked", tracker.checkTouch(pageId(IDX_HEAD2))); - assertFalse("No row can be evicted from unwritten pages", tracker.evictDataPage(true)); - assertEquals("The sampled stale slot must be untracked", 0, tracker.getFragmentLink(slot(tracker, IDX_F))); - - assertEquals("The live row fragment link must stay intact", - slot(tracker, IDX_TAIL2), tracker.getFragmentLink(slot(tracker, IDX_T))); - - assertTrue("The live row head must stay tracked", tracker.checkTouch(pageId(IDX_HEAD2))); - - assertFalse(tracker.evictDataPage(true)); - - // The evictor pass must not touch page data bytes. - for (int idx = 0; idx <= IDX_HEAD2; idx++) { - long absPtr = pageMem.acquirePage(0, pageId(idx)); - - try { - byte[] actual = PageUtils.getBytes(absPtr + PageMemoryNoStoreImpl.PAGE_OVERHEAD, 0, PAGE_SIZE); - - assertTrue("Page " + idx + " bytes must stay untouched by the evictor", Arrays.equals(pageData[idx], actual)); - } - finally { - pageMem.releasePage(0, pageId(idx), absPtr); - } - } - } - finally { - tracker.stop(); - pageMem.stop(true); - } - } - - /** - * @param tracker Tracker. - * @param pageIdx Page index. - * @return Tracking array slot of the page. - */ - private static int slot(PageAbstractEvictionTracker tracker, int pageIdx) { - return tracker.trackingIdx(pageIdx); - } - - /** - * @param pageIdx Page index. - * @return Fabricated data page ID. - */ - private static long pageId(int pageIdx) { - return PageIdUtils.pageId(0, FLAG_DATA, pageIdx); - } - - /** @return Minimal shared context for the tracker constructor. */ - private GridCacheSharedContext sharedCtx() { - GridTestKernalContext kernalCtx = new GridTestKernalContext(log()); - - kernalCtx.config().setDataStorageConfiguration(new DataStorageConfiguration().setPageSize(PAGE_SIZE)); - - GridCacheSharedContext sharedCtx = Mockito.mock(GridCacheSharedContext.class); - - Mockito.when(sharedCtx.kernalContext()).thenReturn(kernalCtx); - Mockito.when(sharedCtx.logger((Class)Mockito.any())).thenReturn(new NullLogger()); - - return sharedCtx; - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index 258eeb597efa9..0b240cf0f23dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -61,7 +61,6 @@ import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; -import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTrackerFragmentChainSelfTest; import org.apache.ignite.testframework.GridTestUtils; /** @@ -123,8 +122,6 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, PageEvictionPagesRecyclingAndReusingTest.class, ignoredTests); - GridTestUtils.addTestIfNeeded(suite, PageEvictionTrackerFragmentChainSelfTest.class, ignoredTests); - GridTestUtils.addTestIfNeeded(suite, DhtAndNearEvictionTest.class, ignoredTests); return suite; From 5a13e8d0c93632915b3ba3e6d470b5e9895daa25 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 15 Oct 2026 01:00:38 +0500 Subject: [PATCH 25/26] atomic reserve pages --- .../cache/persistence/RowStore.java | 22 - .../evict/Random2LruPageEvictionTracker.java | 9 +- .../evict/RandomLruPageEvictionTracker.java | 9 +- .../freelist/AbstractFreeList.java | 390 +++++++++++------- .../FreeListReservedPagesSelfTest.java | 181 ++++++++ .../IgniteCacheEvictionSelfTestSuite.java | 3 + 6 files changed, 442 insertions(+), 172 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 6b13cea999ce2..15cdb52067745 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -20,7 +20,6 @@ import java.util.Collection; import java.util.function.Supplier; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.internal.metric.IoStatisticsHolder; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; @@ -129,32 +128,11 @@ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws Ignit } /** - * Size-aware pre-reserve for the rebalance batch, done when persistence is off and page eviction is enabled - * (single puts get the same guarantee via the per-put reserve in {@link #addRow}). Reserving "for each row" - * collapses to reserving for the largest one: all reserves run before any insert and only enforce a lower bound - * on the shared empty-pages counter, so a single reserve for the max row is equivalent and is what is done here. - * The reserve evicts non-blockingly even though the batch path holds no entry locks (blocking would be safe here): - * the path is shared with single-row insertion, which runs under an entry lock and must not block. - * * @param rows Rows. * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. */ public void addRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { - if (!persistenceEnabled && grp.dataRegion().config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - int maxRowSize = 0; - - for (CacheDataRow row : rows) { - int rowSize = row.size(); - - if (rowSize > maxRowSize) - maxRowSize = rowSize; - } - - if (maxRowSize > 0) - freeList.ensureFreeSpace(maxRowSize); - } - assert ctx.database().checkpointLockIsHeldByThread(); freeList.insertDataRows(rows, statHolder); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index 57df87bb7eddb..88c0d046fb371 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -161,10 +161,9 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); - // The walk may start from a stale pointer: fragment-chain pages are continuously recycled - // and their tracking slots re-linked by newer rows, which recreates identical link values - // across slot generations (ABA). Every recycling generation deepens the chain indirection, - // so two hops can legitimately land on a link slot of a newer chain. Skip the slot and keep sampling. + // The link read above may be stale: a concurrent row deletion nulls the head slot first, and the freed + // head page can be re-used as an intermediate fragment of a new row, so its slot holds a link again + // and the two-hop walk can end on a link. This is a legal state: skip and resample. if (firstTs < 0) { sampleSpinCnt++; @@ -176,8 +175,6 @@ public Random2LruPageEvictionTracker( continue; } - - assert firstTs >= 0 : "[firstTs=" + firstTs + ", secondTs=" + second(trackingData) + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index f23abb233263b..e775d557c06dd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -141,10 +141,9 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); - // The walk may start from a stale pointer: fragment-chain pages are continuously recycled - // and their tracking slots re-linked by newer rows, which recreates identical link values - // across slot generations (ABA). Every recycling generation deepens the chain indirection, - // so two hops can legitimately land on a link slot of a newer chain. Skip the slot and keep sampling. + // The link read above may be stale: a concurrent row deletion nulls the head slot first, and the freed + // head page can be re-used as an intermediate fragment of a new row, so its slot holds a link again + // and the two-hop walk can end on a link. This is a legal state: skip and resample. if (compactTs < 0) { sampleSpinCnt++; @@ -156,8 +155,6 @@ public RandomLruPageEvictionTracker( continue; } - - assert compactTs >= 0 : "[compactTs=" + compactTs + "]"; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 0d5853436619e..70f018bb6c586 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -45,7 +45,6 @@ import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.internal.processors.cache.persistence.Storable; import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager; -import org.apache.ignite.internal.processors.cache.persistence.evict.PageAbstractEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; @@ -55,7 +54,6 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; import org.apache.ignite.internal.util.GridCursorIteratorWrapper; -import org.apache.ignite.internal.util.OffheapReadWriteLock; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.Nullable; @@ -126,12 +124,18 @@ public abstract class AbstractFreeList extends PagesList imp /** */ private final PageEvictionTracker evictionTracker; + /** Whether the page reservation is engaged for this free list: only for in-memory regions with enabled page eviction. */ + private final boolean reserveEnabled; + /** Data region this free list belongs to. */ private final DataRegion dataRegion; /** Kernal context. */ private final GridKernalContext ctx; + /** Pages reserved by in-flight inserts (see {@link #acquireSpace(int)}). */ + private final AtomicLong reservedPages = new AtomicLong(); + /** Page list cache limit. */ private final AtomicLong pageListCacheLimit; @@ -353,8 +357,37 @@ protected void putPage(int freeSpace, long pageId, long page, long pageAddr, IoS } } + /** + * Per-call write context of {@link #insertDataRows}: the rows cursor plus the reservation accounting of the row + * that is currently being written. The context is shared with {@link WriteRowsHandler}, which may consume the + * following rows from the cursor to fill up the page being written (acquiring their reservations itself), and + * with the outer loop of {@link #insertDataRows}, which continues a row left unfinished by the handler. + */ + private static final class RowsWriteContext { + /** Rows cursor. */ + private GridCursor cur; + + /** Reservation made by {@link #acquireSpace(int)} for the current row. */ + private long reserved; + + /** Number of the current row's pages that have already consumed the reservation. */ + private int consumed; + + /** + * Set by {@link #reserveAndWriteWholePages} when it could not claim the space without the guarded eviction + * while the caller holds a page write lock: the row is left to the outer loop of {@link #insertDataRows}. + */ + private boolean deferred; + + /** + * Set when a row was deferred by the page fill-up handler: the cursor stays positioned on it, so the outer + * loop of {@link #insertDataRows} must process it without advancing the cursor. + */ + private boolean pending; + } + /** */ - private final class WriteRowsHandler extends PageHandler, Integer> { + private final class WriteRowsHandler extends PageHandler, Integer> { /** {@inheritDoc} */ @Override public Integer run( int cacheId, @@ -363,19 +396,30 @@ private final class WriteRowsHandler extends PageHandler, Integer> long pageAddr, PageIO iox, Boolean walPlc, - GridCursor cur, + RowsWriteContext ctx, int written, IoStatisticsHolder statHolder ) throws IgniteCheckedException { AbstractDataPageIO io = (AbstractDataPageIO)iox; // Fill the page up to the end. - while (written != COMPLETE || (!evictionTracker.evictionRequired() && cur.next())) { - T row = cur.get(); + while (written != COMPLETE || (!evictionTracker.evictionRequired() && ctx.cur.next())) { + T row = ctx.cur.get(); if (written == COMPLETE) { + int rowWritten = reserveAndWriteWholePages(ctx, row, statHolder, true); + + if (ctx.deferred) { + ctx.deferred = false; + ctx.pending = true; + + break; + } + + written = rowWritten; + // If the data row was completely written without remainder, proceed to the next. - if ((written = writeWholePages(row, statHolder, false)) == COMPLETE) + if (written == COMPLETE) continue; if (io.getFreeSpace(pageAddr) < row.size() - written) @@ -496,6 +540,10 @@ public AbstractFreeList( rmvRow = new RemoveRowHandler(cacheGrpId == 0); evictionTracker = dataRegion.evictionTracker(); + + reserveEnabled = !dataRegion.config().isPersistenceEnabled() && + dataRegion.config().getPageEvictionMode() != DataPageEvictionMode.DISABLED; + this.dataRegion = dataRegion; this.ctx = ctx; this.reuseList = reuseList == null ? this : reuseList; @@ -620,12 +668,18 @@ private int bucket(int freeSpace, boolean allowReuse) { @Override public void insertDataRow(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { int written = 0; + long reserved = acquireSpace(row.size()); + + int consumed = 0; + try { do { if (written != 0) memMetrics.incrementLargeEntriesPages(); - written = writeSinglePage(row, written, statHolder, true); + written = writeSinglePage(row, written, statHolder); + + consumed++; } while (written != COMPLETE); } @@ -635,6 +689,9 @@ private int bucket(int freeSpace, boolean allowReuse) { catch (Throwable t) { throw new CorruptedFreeListException("Failed to insert data row", t, grpId); } + finally { + releaseSpace(reserved - consumed); + } } /** @@ -650,44 +707,58 @@ private int bucket(int freeSpace, boolean allowReuse) { */ @Override public void insertDataRows(Collection rows, IoStatisticsHolder statHolder) throws IgniteCheckedException { try { - GridCursor cur = new GridCursorIteratorWrapper<>(rows.iterator()); + RowsWriteContext ctx = new RowsWriteContext<>(); + + ctx.cur = new GridCursorIteratorWrapper<>(rows.iterator()); int written = COMPLETE; - while (written != COMPLETE || cur.next()) { - T row = cur.get(); + try { + while (written != COMPLETE || ctx.pending || ctx.cur.next()) { + ctx.pending = false; + + T row = ctx.cur.get(); - // If eviction is required - free up memory before locking the next page. - while (evictionTracker.evictionRequired()) { - if (memMetrics.onPageEvictionsStarted()) { - U.warn(log, "Page-based evictions started." + - " Consider increasing 'maxSize' on Data Region configuration: " + dataRegion.config().getName()); + // If eviction is required - free up memory before locking the next page. + while (evictionTracker.evictionRequired()) { + if (memMetrics.onPageEvictionsStarted()) { + U.warn(log, "Page-based evictions started." + + " Consider increasing 'maxSize' on Data Region configuration: " + dataRegion.config().getName()); + } + + evictionTracker.evictDataPage(); + + memMetrics.updateEvictionRate(); } - evictionTracker.evictDataPage(); + if (written == COMPLETE) { + written = reserveAndWriteWholePages(ctx, row, statHolder, false); - memMetrics.updateEvictionRate(); - } + continue; + } - if (written == COMPLETE) { - written = writeWholePages(row, statHolder, true); + AbstractDataPageIO initIo = null; - continue; - } + long pageId = takePage(row.size() - written, row, statHolder); - AbstractDataPageIO initIo = null; + if (pageId == 0L) { + pageId = allocateDataPage(row.partition()); - long pageId = takePageWithReserve(row.size() - written, row, statHolder, true); + initIo = row.ioVersions().latest(); + } - if (pageId == 0L) { - pageId = allocateDataPage(row.partition()); + if (reserveEnabled) + reservedPages.decrementAndGet(); - initIo = row.ioVersions().latest(); - } + ctx.consumed++; - written = write(pageId, writeRowsHnd, initIo, cur, written, FAIL_I, statHolder); + written = write(pageId, writeRowsHnd, initIo, ctx, written, FAIL_I, statHolder); - assert written != FAIL_I; // We can't fail here. + assert written != FAIL_I; // We can't fail here. + } + } + finally { + releaseSpace(ctx.reserved - ctx.consumed); } } catch (IgniteOutOfMemoryException e) { @@ -698,26 +769,71 @@ private int bucket(int freeSpace, boolean allowReuse) { } } + /** + * Reserves the space for the next row and writes its fragments, which occupy the whole memory page. Every + * obtained page consumes the reservation made here. Releases the leftover reservation of the previous row (the + * personal empty-pages margin it was holding) before making a new one. + * + * @param ctx Write context holding the reservation accounting of the current row. + * @param row Row to process. + * @param statHolder Statistics holder to track IO operations. + * @param underPageLock {@code true} when the caller holds a page write lock: the guarded eviction is never run in + * this case — if the claim cannot be made without it, the row is deferred to the outer loop of + * {@link #insertDataRows} ({@code ctx.deferred} is set and {@code 0} is returned, the return value is to be + * ignored by the caller). + * @return Number of bytes written, {@link #COMPLETE} if the row was fully written by whole-page fragments. + * @throws IgniteCheckedException If failed. + */ + private int reserveAndWriteWholePages( + RowsWriteContext ctx, + T row, + IoStatisticsHolder statHolder, + boolean underPageLock + ) throws IgniteCheckedException { + releaseSpace(ctx.reserved - ctx.consumed); + + ctx.reserved = 0; + ctx.consumed = 0; + + ctx.reserved = underPageLock ? tryAcquireSpace(row.size()) : acquireSpace(row.size()); + + if (ctx.reserved < 0) { + ctx.deferred = true; + + return 0; + } + + int written = writeWholePages(row, statHolder); + + // Each whole-page fragment consumes exactly MIN_SIZE_FOR_DATA_PAGE bytes, and writeSinglePage has already + // decremented reservedPages for each of them: account the fragments in the local counter. If writeWholePages + // fails midway, the fragments it has already written are over-consumed here - bounded by the row size and + // acceptable for a failing insert. + ctx.consumed = written != COMPLETE + ? written / MIN_SIZE_FOR_DATA_PAGE + : (row.size() + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; + + return written; + } + /** * Write fragments of the row, which occupy the whole memory page. A data row is ignored if it is less than the max * payload of an empty data page. * * @param row Row to process. * @param statHolder Statistics holder to track IO operations. - * @param reserve {@code true} to run the size-aware reserve before each page allocation; must be {@code false} - * when the current thread holds a page write lock. * @return Number of bytes written, {@link #COMPLETE} if the row was fully written, {@code 0} if data row was * ignored because it is less than the max payload of an empty data page. * @throws IgniteCheckedException If failed. */ - private int writeWholePages(T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { + private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { assert row.link() == 0 : row.link(); int written = 0; int rowSize = row.size(); while (rowSize - written >= MIN_SIZE_FOR_DATA_PAGE) { - written = writeSinglePage(row, written, statHolder, reserve); + written = writeSinglePage(row, written, statHolder); memMetrics.incrementLargeEntriesPages(); } @@ -726,20 +842,18 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder, boolean reserv } /** - * Take a page and write row on it. + * Take a page and write row on it. Every obtained page consumes the reservation made by {@link #acquireSpace(int)}. * * @param row Row to write. * @param written Written size. * @param statHolder Statistics holder to track IO operations. - * @param reserve {@code true} to run the size-aware reserve before taking the page; must be {@code false} - * when the current thread holds a page write lock. * @return Number of bytes written, {@link #COMPLETE} if the row was fully written. * @throws IgniteCheckedException If failed. */ - private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { + private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) throws IgniteCheckedException { AbstractDataPageIO initIo = null; - long pageId = takePageWithReserve(row.size() - written, row, statHolder, reserve); + long pageId = takePage(row.size() - written, row, statHolder); if (pageId == 0L) { pageId = allocateDataPage(row.partition()); @@ -747,6 +861,9 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder, b initIo = row.ioVersions().latest(); } + if (reserveEnabled) + reservedPages.decrementAndGet(); + written = write(pageId, writeRowHnd, initIo, row, written, FAIL_I, statHolder); assert written != FAIL_I; // We can't fail here. @@ -963,10 +1080,23 @@ public int emptyDataPages() { if (dataRegion.config().isPersistenceEnabled()) return; - if (dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) - checkOomThreshold(rowSize); - else - reserveSpaceForInsert(rowSize); + if (dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) { + long nonEmptyPages = pageMem.loadedPages() - emptyDataPages(); + + // The maximum number of pages that can be allocated (maxSize / systemPageSize) + // should be greater or equal to pages required for inserting a new entry plus + // the number of currently non-empty data pages. Also we account for pages that hold pages-list metadata: + // 8 bytes per each row on average, considering that maybe not all list pages are full, multiply by 1.5. + // Plus, add one page for each bucket in the worst case. + // Note that not the whole page can be used to storing links, + // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, + // in any way, the number of required pages is less than 1 percent. + boolean oomThreshold = (dataRegion.config().getMaxSize() / pageMem.systemPageSize()) < + ((double)rowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); + + if (oomThreshold) + throw outOfMemory(dataRegion.config()); + } } /** {@inheritDoc} */ @@ -986,92 +1116,51 @@ private long allocateDataPage(int part) throws IgniteCheckedException { } /** - * Takes a page, optionally running the size-aware reserve first. - * The reserve targets {@code requiredPages + emptyPagesPoolSize}, so after this thread consumes its pages - * at least {@code emptyPagesPoolSize} pages remain for concurrent writers. - *

    - * The reserve is a check-then-act on shared counters: concurrent writers may drain the free list and the - * allocation headroom between the gate check and {@link #takePage}. If {@code takePage} finds no page, the - * reserve is re-evaluated on fresh state and given a second chance to evict before the caller falls back to - * allocating a brand-new page, which fails with a raw out-of-memory once the region is physically exhausted. - * - * @param size Free space required on the page. - * @param row Row to write. - * @param statHolder Statistics holder to track IO operations. - * @param reserve {@code true} to run the size-aware reserve before taking the page. Must be {@code false} - * when the current thread already holds a page write lock: the reserve may evict, and eviction takes a - * blocking read lock on the candidate data page, while {@link OffheapReadWriteLock} is not reentrant - * (see {@link PageAbstractEvictionTracker}). - * @return Page identifier or 0 if no page could be obtained after reserving. - * @throws IgniteCheckedException If failed. - */ - private long takePageWithReserve(int size, T row, IoStatisticsHolder statHolder, boolean reserve) throws IgniteCheckedException { - if (reserve && !dataRegion.config().isPersistenceEnabled() && - !(dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED)) { - reserveSpaceForInsert(size); - - long pageId = takePage(size, row, statHolder); - - if (pageId == 0L) { - reserveSpaceForInsert(size); - - pageId = takePage(size, row, statHolder); - } - - return pageId; - } - - return takePage(size, row, statHolder); - } - - /** - * Size-aware reserve for an in-memory region with enabled page eviction. Runs eviction until the combined space - * (empty pages in this free list + unallocated headroom) is enough to accommodate the row plus the - * {@link DataRegionConfiguration#getEmptyPagesPoolSize() empty-pages pool}, or throws - * {@link IgniteOutOfMemoryException} if the goal is unreachable / no progress can be made. The pool margin is kept - * on top of the row's own requirement so that, after this thread consumes its pages, at least {@code poolSize} - * pages remain for concurrent writers; this makes the path safe for up to {@code poolSize} concurrent threads. - * Both empty pages and headroom are treated as concurrent resources: the fast path returns as soon as - * their sum covers the target, and the eviction loop only evicts the deficit. + * Reserves the space for a row of the given size in an in-memory region with enabled page eviction: runs eviction + * until {@code freePages - reservedPages} covers the whole demand of the writer — {@code requiredPages} for the row + * itself plus a personal copy of the {@code emptyPagesPoolSize} margin — then atomically adds the demand to + * {@code reservedPages}. * - * @param rowSize Size of the row (or its remaining part) to be inserted. - * @throws IgniteOutOfMemoryException If the target cannot be reached (row too large for the region or eviction - * makes no progress). + * @param rowSize Size of the row to be inserted. + * @return Number of reserved pages (the row's pages plus the claimed margin) to be released via + * {@link #releaseSpace(long)}, or {@code 0} if the reservation is not engaged for this free list (persistent data + * region or disabled page eviction — see {@link #reserveEnabled}). + * @throws IgniteOutOfMemoryException If the row cannot fit into the region, or eviction makes no progress. * @throws IgniteCheckedException If failed to evict data pages. */ - private void reserveSpaceForInsert(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { - if (evictionTracker == null) - return; + private long acquireSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { + if (!reserveEnabled) + return 0; DataRegionConfiguration cfg = dataRegion.config(); - long pagePayload = pageMem.pageSize() - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD; - long totalPages = cfg.getMaxSize() / pageMem.systemPageSize(); - long requiredPages = (rowSize + pagePayload - 1) / pagePayload; + long requiredPages = (rowSize + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; if (requiredPages > totalPages) throw outOfMemory(cfg); - long targetPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); + long claimPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); - long emptyPages = emptyDataPages(); - - // Effective headroom accounts for per-segment allocation overhead (up to one page per segment), - // so the fast path does not trust headroom that cannot actually be allocated. - if (emptyPages + effectiveHeadroom(totalPages, pageMem.loadedPages()) >= targetPages) - return; - - long bestEmptyPages = emptyPages; + long backoffNanos = EVICTION_BACKOFF_START_NANOS; long lastProgressNanos = System.nanoTime(); long cycleDeadlineNanos = lastProgressNanos + TimeUnit.MILLISECONDS.toNanos(EVICTION_MAX_CYCLE_TIME_MILLIS); - long backoffNanos = EVICTION_BACKOFF_START_NANOS; + while (true) { + long deficit = claimPages - (freePages(totalPages) - reservedPages.get()); + + if (deficit <= 0) { + long reserved = reservedPages.get(); + + if (reservedPages.compareAndSet(reserved, reserved + claimPages)) + return claimPages; + + continue; + } - while (emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()) < targetPages) { if (memMetrics.onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + " Consider increasing 'maxSize' on Data Region configuration: " + cfg.getName()); @@ -1081,12 +1170,9 @@ private void reserveSpaceForInsert(int rowSize) throws IgniteOutOfMemoryExceptio memMetrics.updateEvictionRate(); - long curEmptyPages = emptyDataPages(); - - if (evicted || curEmptyPages > bestEmptyPages) { - if (curEmptyPages > bestEmptyPages) - bestEmptyPages = curEmptyPages; + long curDeficit = claimPages - (freePages(totalPages) - reservedPages.get()); + if (evicted || curDeficit < deficit) { lastProgressNanos = System.nanoTime(); backoffNanos = EVICTION_BACKOFF_START_NANOS; @@ -1099,36 +1185,64 @@ private void reserveSpaceForInsert(int rowSize) throws IgniteOutOfMemoryExceptio long nowNanos = System.nanoTime(); - long maxNoProgress = TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS); - - if (nowNanos - lastProgressNanos > maxNoProgress || nowNanos > cycleDeadlineNanos) + if (nowNanos - lastProgressNanos > TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS) || + nowNanos > cycleDeadlineNanos) throw outOfMemory(cfg); } } /** - * Checks that an in-memory region with disabled page eviction has enough pages for a new row, taking into - * account the pages required to support the free list. + * Non-blocking flavor of {@link #acquireSpace(int)} for callers holding a page write lock: claims the space only + * if it is instantly available; never runs the guarded eviction and never parks. * * @param rowSize Size of the row to be inserted. - * @throws IgniteOutOfMemoryException If the region does not have enough free space for the new entry. + * @return Number of reserved pages to be released via {@link #releaseSpace(long)}, {@code 0} if the reservation is + * not engaged for this free list (see {@link #reserveEnabled}), or {@code -1} if the claim cannot be made without + * the guarded eviction or raced with a concurrent one. + */ + private long tryAcquireSpace(int rowSize) { + if (!reserveEnabled) + return 0; + + DataRegionConfiguration cfg = dataRegion.config(); + + long totalPages = cfg.getMaxSize() / pageMem.systemPageSize(); + + long requiredPages = (rowSize + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; + + if (requiredPages > totalPages) + return -1; + + long claimPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); + + long reserved = reservedPages.get(); + + if (claimPages - (freePages(totalPages) - reserved) > 0 || !reservedPages.compareAndSet(reserved, reserved + claimPages)) + return -1; + + return claimPages; + } + + /** + * Releases the reservation made by {@link #acquireSpace(int)}: consumed pages were already decremented + * {@code reservedPages} by {@link #writeSinglePage} and the continuation path of {@link #insertDataRows}, so + * only the unconsumed remainder (the row's personal empty-pages margin) is subtracted here. + * + * @param pages Number of pages to release. + */ + private void releaseSpace(long pages) { + if (reserveEnabled && pages != 0) + reservedPages.addAndGet(-pages); + } + + /** + * @param totalPages Total pages in the region (maxSize / systemPageSize). + * @return Free pages of the region: empty pages in this free list plus unallocated headroom. Effective headroom + * accounts for per-segment allocation overhead (up to one page per segment), so it does not trust headroom + * that cannot actually be allocated. */ - private void checkOomThreshold(int rowSize) { - long nonEmptyPages = pageMem.loadedPages() - emptyDataPages(); - - // The maximum number of pages that can be allocated (maxSize / systemPageSize) - // should be greater or equal to pages required for inserting a new entry plus - // the number of currently non-empty data pages. Also we account for pages that hold pages-list metadata: - // 8 bytes per each row on average, considering that maybe not all list pages are full, multiply by 1.5. - // Plus, add one page for each bucket in the worst case. - // Note that not the whole page can be used to storing links, - // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, - // in any way, the number of required pages is less than 1 percent. - boolean oomThreshold = (dataRegion.config().getMaxSize() / pageMem.systemPageSize()) < - ((double)rowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); - - if (oomThreshold) - throw outOfMemory(dataRegion.config()); + private long freePages(long totalPages) { + return emptyDataPages() + effectiveHeadroom(totalPages, pageMem.loadedPages()); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java new file mode 100644 index 0000000000000..a3262b0297244 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.freelist; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; + +/** + * Tests that the atomic page reservation of an in-memory free list with enabled page eviction is fully released + * after batch inserts: covers the whole-page fragment path, rows consumed by the page fill-up handler, rows left + * unfinished by the fill-up handler and rows sized as an exact multiple of the whole-page fragment size. + */ +public class FreeListReservedPagesSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration(); + + dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(20 * 1024 * 1024L / 2) + .setMaxSize(20 * 1024 * 1024L) + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)); + + cfg.setDataStorageConfiguration(dsCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** */ + @Test + public void testReservedPagesReleasedAfterSmallRowsBatch() throws Exception { + assertReservedPagesReleasedAfterBatch(10); + } + + /** */ + @Test + public void testReservedPagesReleasedAfterLargeRowsBatch() throws Exception { + assertReservedPagesReleasedAfterBatch(10_000); + } + + /** */ + @Test + public void testReservedPagesReleasedAfterWholePageAlignedRowsBatch() throws Exception { + IgniteEx ignite = startGrid(0); + + createCache(ignite); + + FreeList freeList = freeList(ignite); + + int minPageSize = getFieldValue(freeList, "MIN_SIZE_FOR_DATA_PAGE"); + + // Probe the size fixed part (key bytes, version and the link slot) to size the value so that the whole row + // is an exact multiple of the whole-page fragment size. + int fixedSize = row(cacheContext(ignite), 0, 0).size(); + + insertBatches(ignite, freeList, 2 * minPageSize - fixedSize); + + assertEquals(0, reservedPages(freeList)); + } + + /** */ + private void assertReservedPagesReleasedAfterBatch(int payloadSize) throws Exception { + IgniteEx ignite = startGrid(0); + + createCache(ignite); + + insertBatches(ignite, freeList(ignite), payloadSize); + } + + /** */ + private static void insertBatches(IgniteEx ignite, FreeList freeList, int payloadSize) throws IgniteCheckedException { + int batchSize = 500; + + for (int batch = 0; batch < 3; batch++) { + GridCacheContext cctx = cacheContext(ignite); + + Collection rows = new ArrayList<>(batchSize); + + for (int i = 0; i < batchSize; i++) + rows.add(row(cctx, batch * batchSize + i, payloadSize)); + + freeList.insertDataRows(rows, IoStatisticsHolderNoOp.INSTANCE); + + assertEquals(0, reservedPages(freeList)); + } + } + + /** + * @param freeList Free list. + * @return Pages currently reserved by in-flight inserts. + */ + private static long reservedPages(FreeList freeList) { + return ((AtomicLong)getFieldValue(freeList, "reservedPages")).get(); + } + + /** + * @param ignite Ignite. + * @return Free list of the first partition data store. + */ + private static FreeList freeList(IgniteEx ignite) { + return (FreeList)cacheContext(ignite).offheap().cacheDataStores().iterator().next().rowStore().freeList(); + } + + /** + * @param ignite Ignite. + * @return Context of the default cache. + */ + private static GridCacheContext cacheContext(IgniteEx ignite) { + return ignite.context().cache().cache(DEFAULT_CACHE_NAME).context(); + } + + /** + * @param ignite Ignite. + */ + private static void createCache(IgniteEx ignite) { + IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)).setBackups(0)); + + cache.put(0, 0); + } + + /** + * @param cctx Cache context. + * @param key Key. + * @param payloadSize Value size. + * @return Data row with the given key and a byte array value of the given size. + */ + private static CacheDataRow row(GridCacheContext cctx, int key, int payloadSize) throws IgniteCheckedException { + KeyCacheObject keyObj = cctx.toCacheKeyObject(key); + CacheObject valObj = cctx.toCacheObject(new byte[payloadSize]); + + keyObj.valueBytes(cctx.cacheObjectContext()); + valObj.valueBytes(cctx.cacheObjectContext()); + + return new DataRow(keyObj, valObj, new GridCacheVersion(), 0, 0L, cctx.cacheId()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index 0b240cf0f23dd..a6f7f3a631206 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -61,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListReservedPagesSelfTest; import org.apache.ignite.testframework.GridTestUtils; /** @@ -124,6 +125,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, DhtAndNearEvictionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, FreeListReservedPagesSelfTest.class, ignoredTests); + return suite; } } From 4429129820f67bd71f9ec9c9bec3d96e6a7d89a5 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 15 Oct 2026 22:55:09 +0500 Subject: [PATCH 26/26] reserve until entry lock --- .../internal/binary/BinaryObjectImpl.java | 6 + .../processors/cache/GridCacheEntryEx.java | 51 +-- .../processors/cache/GridCacheMapEntry.java | 142 ++++---- .../dht/atomic/GridDhtAtomicCache.java | 75 +++- .../persistence/CacheDataRowAdapter.java | 6 +- .../IgniteCacheDatabaseSharedManager.java | 54 +-- .../cache/persistence/RowStore.java | 36 +- .../evict/FairFifoPageEvictionTracker.java | 4 +- .../evict/NoOpPageEvictionTracker.java | 2 +- .../evict/PageAbstractEvictionTracker.java | 10 +- .../evict/PageEvictionTracker.java | 18 +- .../evict/Random2LruPageEvictionTracker.java | 9 +- .../evict/RandomLruPageEvictionTracker.java | 9 +- .../freelist/AbstractFreeList.java | 319 ++++++++---------- .../cache/persistence/freelist/FreeList.java | 40 ++- .../persistence/tree/io/CacheVersionIO.java | 2 +- .../paged/PageEvictionAbstractTest.java | 42 ++- ...eEvictionConcurrentWritesAbstractTest.java | 140 ++++++-- ...isabledEvictionReadThroughOomSelfTest.java | 158 +++++++++ ...eEvictionEnsureFreeSpaceThresholdTest.java | 85 +++++ .../paged/PageEvictionGuardOomTest.java | 10 +- .../paged/PageEvictionMetricTest.java | 4 +- .../PageEvictionMultinodeAbstractTest.java | 2 +- ...ctionReadThroughReservedPagesSelfTest.java | 140 ++++++++ .../PageEvictionRowSizeBinaryObjectTest.java | 108 ++++++ .../PageEvictionSizeAwareAbstractTest.java | 86 ++--- ...eEvictionWithExpiryPolicyAbstractTest.java | 18 +- .../FreeListReservedPagesAbstractTest.java | 97 ++++++ .../FreeListReservedPagesSelfTest.java | 204 +++++++---- .../IgniteCacheEvictionSelfTestSuite.java | 12 + 30 files changed, 1386 insertions(+), 503 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDisabledEvictionReadThroughOomSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionEnsureFreeSpaceThresholdTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughReservedPagesSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionRowSizeBinaryObjectTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesAbstractTest.java diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index 3359c73459de8..2c99d107627fa 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -218,6 +218,12 @@ public BinaryObjectImpl(BinaryContext ctx, byte[] arr, int start) { /** {@inheritDoc} */ @Override public int valueBytesLength(CacheObjectValueContext ctx) throws IgniteCheckedException { + if (valBytes == null) { + assert arr != null; + + valBytes = valueBytesFromArray(ctx); + } + return CacheObjectUtils.objectPutSize(valBytes.length); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index 042e9f78a323d..9755e56228058 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -201,30 +201,6 @@ public interface GridCacheEntryEx { public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap) throws IgniteCheckedException; - /** - * Same as {@link #evictInternal(GridCacheVersion, CacheEntryPredicate[], boolean)}, but when {@code tryLock} is - * {@code true} the entry lock is acquired non-blockingly: the entry is skipped (this method returns {@code false}) - * whenever its lock is contended or already held by the current thread (the self-hold case), instead of blocking. - * Used by size-aware page eviction which may run while the current thread already holds other entry locks, to - * avoid a lock-ordering deadlock. The default implementation ignores {@code tryLock} and uses the blocking - * variant. - * - * @param obsoleteVer Version for eviction. - * @param filter Optional filter. - * @param evictOffheap Evict offheap value flag. - * @param tryLock {@code true} to acquire the entry lock non-blockingly (skip contended or self-held entries). - * @return {@code True} if entry could be evicted. - * @throws IgniteCheckedException In case of error. - */ - public default boolean evictInternal( - GridCacheVersion obsoleteVer, - @Nullable CacheEntryPredicate[] filter, - boolean evictOffheap, - boolean tryLock - ) throws IgniteCheckedException { - return evictInternal(obsoleteVer, filter, evictOffheap); - } - /** * This method should be called each time entry is marked obsolete * other than by calling {@link #markObsolete(GridCacheVersion)}. @@ -689,6 +665,10 @@ public GridCacheVersionedEntryEx versionedEntry(final boolean keepB * Sets new value if passed in version matches the current version * (used for read-through only). * + *

    This overload reserves page space internally (before acquiring the entry lock). Use + * {@link #versionedValue(CacheObject, GridCacheVersion, GridCacheVersion, IgniteCacheExpiryPolicy, + * ReaderArguments, long)} when the caller has already reserved (or the entry is already locked). + * * @param val New value. * @param curVer Version to match or {@code null} if match is not required. * @param newVer Version to set. @@ -705,6 +685,29 @@ public EntryGetResult versionedValue(CacheObject val, @Nullable ReaderArguments readerArgs) throws IgniteCheckedException, GridCacheEntryRemovedException; + /** + * Sets new value if passed in version matches the current version (used for read-through only). + * + * @param val New value. + * @param curVer Version to match or {@code null} if match is not required. + * @param newVer Version to set. + * @param loadExpiryPlc Expiry policy if entry is loaded from store. + * @param readerArgs Reader will be added if not null. + * @param reserved Number of pages reserved by a prior call to {@code reserveEntrySpace} (before acquiring the + * entry lock), or {@code 0} if no reservation was made. + * @return Current version and value. + * @throws IgniteCheckedException If index could not be updated. + * @throws GridCacheEntryRemovedException If entry was removed. + */ + public EntryGetResult versionedValue( + CacheObject val, + @Nullable GridCacheVersion curVer, + @Nullable GridCacheVersion newVer, + @Nullable IgniteCacheExpiryPolicy loadExpiryPlc, + @Nullable ReaderArguments readerArgs, + long reserved + ) throws IgniteCheckedException, GridCacheEntryRemovedException; + /** * Checks if the candidate is either owner or pending. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 536b72a27648a..ca4ae6327df42 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -55,9 +55,10 @@ import org.apache.ignite.internal.processors.cache.extras.GridCacheTtlEntryExtras; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; -import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.RowStore; import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpEntryChangeListener; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryListener; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; @@ -903,8 +904,10 @@ private EntryGetResult entryGetResult(CacheObject val, GridCacheVersion ver, boo boolean touch = false; + long reserved; + try { - ensureFreeSpace(); + reserved = reserveEntrySpace(rowSize(ret, ver)); lockEntry(); @@ -950,6 +953,8 @@ private EntryGetResult entryGetResult(CacheObject val, GridCacheVersion ver, boo } finally { unlockEntry(); + + releaseReserved(reserved); } touch = true; @@ -1006,7 +1011,7 @@ protected void recordNodeId(UUID nodeId, AffinityTopologyVersion topVer) { long updateCntr0; - ensureFreeSpace(); + long reserved = reserveEntrySpace(rowSize(val, ver)); lockListenerReadLock(); lockEntry(); @@ -1159,6 +1164,8 @@ else if (interceptorVal != val0) finally { unlockEntry(); unlockListenerReadLock(); + + releaseReserved(reserved); } onUpdateFinished(updateCntr0); @@ -1463,8 +1470,12 @@ else if (log.isDebugEnabled()) AtomicCacheUpdateClosure c; + long reserved = 0; + if (!primary && !isNear()) - ensureFreeSpace(); + reserved = reserveEntrySpace(rowSize( + op == GridCacheOperation.UPDATE && writeObj instanceof CacheObject ? (CacheObject)writeObj : null, + newVer)); lockListenerReadLock(); lockEntry(); @@ -1724,6 +1735,8 @@ else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome()) finally { unlockEntry(); unlockListenerReadLock(); + + releaseReserved(reserved); } onUpdateFinished(c.updateRes.updateCounter()); @@ -2473,7 +2486,7 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { ) throws IgniteCheckedException, GridCacheEntryRemovedException { assert !primary || !(preload || fromStore); - ensureFreeSpace(); + long reserved = reserveEntrySpace(rowSize(val, ver)); boolean deferred = false; boolean obsolete = false; @@ -2615,6 +2628,8 @@ else if (deletedUnlocked()) unlockEntry(); unlockListenerReadLock(); + releaseReserved(reserved); + // It is necessary to execute these callbacks outside of lock to avoid deadlocks. if (obsolete) { @@ -2705,11 +2720,26 @@ protected long nextPartitionCounter(IgniteInternalTx tx, @Nullable Long updateCn } /** {@inheritDoc} */ - @Override public EntryGetResult versionedValue(CacheObject val, + @Override public EntryGetResult versionedValue( + CacheObject val, GridCacheVersion curVer, GridCacheVersion newVer, @Nullable IgniteCacheExpiryPolicy loadExpiryPlc, @Nullable ReaderArguments readerArgs + ) throws IgniteCheckedException, GridCacheEntryRemovedException { + long reserved = reserveEntrySpace(rowSize(val, newVer != null ? newVer : ver)); + + return versionedValue(val, curVer, newVer, loadExpiryPlc, readerArgs, reserved); + } + + /** {@inheritDoc} */ + @Override public EntryGetResult versionedValue( + CacheObject val, + GridCacheVersion curVer, + GridCacheVersion newVer, + @Nullable IgniteCacheExpiryPolicy loadExpiryPlc, + @Nullable ReaderArguments readerArgs, + long reserved ) throws IgniteCheckedException, GridCacheEntryRemovedException { lockEntry(); @@ -2765,6 +2795,8 @@ protected long nextPartitionCounter(IgniteInternalTx tx, @Nullable Long updateCn } finally { unlockEntry(); + + releaseReserved(reserved); } } @@ -3619,16 +3651,49 @@ protected void removeValue() throws IgniteCheckedException { } /** - * Evicts necessary number of data pages if per-page eviction is configured in current {@link DataRegion}. + * Reserves pages for the row about to be inserted. Must be called before {@link #lockEntry()} — eviction may + * block on other entries' locks. + * + * @param rowSize Size of the row to be inserted. + * @return Number of reserved pages, or {@code 0} if reservation is not engaged. + * @throws IgniteCheckedException If failed. */ - private void ensureFreeSpace() throws IgniteCheckedException { - // Deadlock alert: evicting a data page removes (and locks) all entries on the page one by one, so this must - // only run while NOT holding this entry's lock (all call sites run before lockEntry()). The size-aware path - // (RowStore.addRow -> FreeList.ensureFreeSpace) runs under the lock and instead uses the non-blocking - // tryLockEntry(0) inside evictInternal. + private long reserveEntrySpace(int rowSize) throws IgniteCheckedException { + // Deadlock alert: evicting a data page locks all entries on it. assert !lock.isHeldByCurrentThread(); - cctx.shared().database().ensureFreeSpace(cctx.dataRegion()); + // Near entries keep values on heap and never write rows to the free list (storeValue is a no-op for them). + if (isNear()) + return 0; + + return cctx.shared().database().reserveInRegion(cctx.dataRegion(), rowSize); + } + + /** + * Releases a reservation made by {@link #reserveEntrySpace(int)}. The whole claim is released as a unit: + * the insert does not consume the reservation page by page, so the release is exact on any outcome + * (including an exception in the middle of the row insert). + * + * @param reserved Number of reserved pages. + */ + private void releaseReserved(long reserved) { + if (reserved > 0) + cctx.shared().database().releaseInRegion(cctx.dataRegion(), reserved); + } + + /** + * Computes the data row size for the given value and version (see {@link RowStore#rowSize}). + * Used to reserve pages before the entry lock is acquired. + * + * @param val Value (maybe {@code null} if unknown — under-reserves safely). + * @param ver Version. + * @return Estimated row size in bytes. + * @throws IgniteCheckedException If failed. + */ + private int rowSize(@Nullable CacheObject val, GridCacheVersion ver) throws IgniteCheckedException { + int cacheId = cctx.group().storeCacheIdInDataPage() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID; + + return RowStore.rowSize(key, val, CacheVersionIO.size(ver, false), cctx.cacheObjectContext(), cacheId); } /** @@ -3651,32 +3716,14 @@ private CacheEntryImplEx wrapVersionedWithValue() { @Override public boolean evictInternal( GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, - boolean evictOffheap - ) throws IgniteCheckedException { - return evictInternal(obsoleteVer, filter, evictOffheap, false); - } + boolean evictOffheap) + throws IgniteCheckedException { - /** {@inheritDoc} */ - @Override public boolean evictInternal( - GridCacheVersion obsoleteVer, - @Nullable CacheEntryPredicate[] filter, - boolean evictOffheap, - boolean tryLock - ) throws IgniteCheckedException { boolean marked = false; try { if (F.isEmptyOrNulls(filter)) { - // With tryLock=true the lock is taken non-blockingly and a contended entry is skipped - // (returns false) to avoid a lock-ordering deadlock; the tracker then picks another page. - // Reentry is forbidden here to prevent self-eviction: if the current thread already holds - // the entry lock (e.g. during a row insert that triggers size-aware eviction), evicting - // the entry from under itself would corrupt in-flight operations. - if (tryLock && lockedByCurrentThread()) - return false; - - if (!lockEntry(tryLock)) - return false; + lockEntry(); try { if (evictionDisabled()) { @@ -3710,15 +3757,10 @@ private CacheEntryImplEx wrapVersionedWithValue() { } else { // For optimistic check. - // Reentry is forbidden when tryLock=true to prevent self-eviction (see the comment above). - if (tryLock && lockedByCurrentThread()) - return false; - while (true) { GridCacheVersion v; - if (!lockEntry(tryLock)) - return false; + lockEntry(); try { v = ver; @@ -3730,8 +3772,7 @@ private CacheEntryImplEx wrapVersionedWithValue() { if (!cctx.isAll(/*version needed for sync evicts*/this, filter)) return false; - if (!lockEntry(tryLock)) - return false; + lockEntry(); try { if (evictionDisabled()) { @@ -4173,23 +4214,6 @@ private int extrasSize() { lock.lock(); } - /** - * Acquires the entry lock either blocking ({@code tryLock == false}) or non-blockingly with an immediate - * {@code tryLock(0)} ({@code tryLock == true}). Used by {@link #evictInternal} to let size-aware - * eviction skip contended entries instead of blocking, avoiding a lock-ordering deadlock. - * - * @param tryLock {@code true} to acquire the lock non-blockingly. - * @return {@code true} if the lock was acquired (always {@code true} when {@code tryLock == false}). - */ - private boolean lockEntry(boolean tryLock) { - if (tryLock) - return tryLockEntry(0); - - lockEntry(); - - return true; - } - /** {@inheritDoc} */ @Override public boolean tryLockEntry(long timeout) { try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 496f579a33fcf..7a125d2bf597b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -55,6 +55,7 @@ import org.apache.ignite.internal.processors.cache.CacheInvokeResult; import org.apache.ignite.internal.processors.cache.CacheLazyEntry; import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; import org.apache.ignite.internal.processors.cache.CacheOperationContext; import org.apache.ignite.internal.processors.cache.CacheStoppedException; import org.apache.ignite.internal.processors.cache.CacheStorePartialUpdateException; @@ -90,7 +91,10 @@ import org.apache.ignite.internal.processors.cache.dr.GridCacheDrExpirationInfo; import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.RowStore; import org.apache.ignite.internal.processors.cache.persistence.StorageException; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext; @@ -162,6 +166,13 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { private static final int DEFERRED_UPDATE_RESPONSE_TIMEOUT = Integer.getInteger(IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT, DFLT_ATOMIC_DEFERRED_ACK_TIMEOUT); + /** + * Upper bound of a single batch reservation in bytes: caps the sum of row sizes passed to + * {@code FreeList.reserve} to keep its page arithmetic within {@code int}. Batches above the cap are silently + * under-reserved (they are pathological — default per-request size limits are far below it). + */ + private static final int MAX_RESERVE_BATCH_BYTES = 1_000_000_000; + /** */ private final ThreadLocal> defRes = new ThreadLocal>() { @@ -1806,9 +1817,13 @@ private void updateAllAsyncInternal0( ctx.shared().database().checkpointReadLock(); + long batchReserved = 0; + try { ctx.shared().database().ensureFreeSpace(ctx.dataRegion()); + batchReserved = reserveBatchSpace(req); + // If batch store update is enabled, we need to lock all entries. // First, need to acquire locks on cache entries, then check filter. List locked = lockEntries(req, req.topologyVersion()); @@ -1949,6 +1964,8 @@ private void updateAllAsyncInternal0( return; } finally { + releaseBatchSpace(batchReserved); + ctx.shared().database().checkpointReadUnlock(); } @@ -2489,7 +2506,7 @@ private void reloadIfNeeded(final List entries) throws Ignite try { GridCacheVersion ver = entry.version(); - entry.versionedValue(ctx.toCacheObject(v), null, ver, null, null); + entry.versionedValue(ctx.toCacheObject(v), null, ver, null, null, 0L); } catch (GridCacheEntryRemovedException e) { assert false : "Entry should not get obsolete while holding lock [entry=" + entry + @@ -2978,6 +2995,62 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { } } + /** + * Reserves free-list pages for the total size of all rows in the batch: size-aware admission control for + * the primary atomic path, which acquires entry locks via {@link #lockEntries} before {@code innerUpdate} and so + * cannot reserve (or evict) per entry there. Reserving before {@link #lockEntries} guarantees every row of a + * batch with known value sizes can be inserted without an out-of-memory in {@code FreeList.insertDataRow}. + * + *

    Best-effort only: {@code TRANSFORM} rows reserve the key sizes (processor results are unknown), and batches + * above {@link #MAX_RESERVE_BATCH_BYTES} are reserved for the capped size. + * + * @param req Update request. + * @return Reserved pages to release via {@link #releaseBatchSpace}, or {@code 0} if reservation is not engaged. + * @throws IgniteCheckedException If failed. + */ + private long reserveBatchSpace(GridNearAtomicAbstractUpdateRequest req) throws IgniteCheckedException { + DataRegion dataRegion = ctx.dataRegion(); + + if (dataRegion == null || dataRegion.config().isPersistenceEnabled()) + return 0; + + CacheObjectValueContext coctx = ctx.cacheObjectContext(); + + int cacheId = ctx.group().storeCacheIdInDataPage() ? ctx.cacheId() : CU.UNDEFINED_CACHE_ID; + + long sumRowSize = 0; + + for (int i = 0; i < req.size(); i++) { + KeyCacheObject key = req.key(i); + + CacheObject val = req.writeValue(i); + + if (val == null && req.operation() == GridCacheOperation.UPDATE) + val = req.value(i); + + // CacheVersionIO.SIZE_V2 is the maximum of both version protocols (V1 is smaller), so this is an upper + // bound: the actual rows can not be larger than estimated. + sumRowSize += RowStore.rowSize(key, val, CacheVersionIO.SIZE_V2, coctx, cacheId); + } + + if (sumRowSize == 0) + return 0; + + int reserveSize = sumRowSize > MAX_RESERVE_BATCH_BYTES ? MAX_RESERVE_BATCH_BYTES : (int)sumRowSize; + + return ctx.shared().database().reserveInRegion(dataRegion, reserveSize); + } + + /** + * Releases the batch-level reservation made by {@link #reserveBatchSpace}. + * + * @param reserved Reserved pages. + */ + private void releaseBatchSpace(long reserved) { + if (reserved != 0) + ctx.shared().database().releaseInRegion(ctx.dataRegion(), reserved); + } + /** * Acquires java-level locks on cache entries. Returns collection of locked entries. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 810423838a3a8..9625bcc91fc3c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -890,11 +890,7 @@ public boolean isReady() { /** {@inheritDoc} */ @Override public int size() throws IgniteCheckedException { - int len = key().valueBytesLength(null); - - len += value().valueBytesLength(null) + CacheVersionIO.size(version(), false) + 8; - - return len + (cacheId() != 0 ? 4 : 0); + return RowStore.rowSize(key(), value(), CacheVersionIO.size(version(), false), null, cacheId()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 9ac6cee62c65c..77046c62df0f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -100,7 +100,6 @@ import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_SUB_INTERVALS; import static org.apache.ignite.configuration.DataStorageConfiguration.HALF_MAX_WAL_ARCHIVE_SIZE; import static org.apache.ignite.configuration.DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE; -import static org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.SEG_CNT; import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.METASTORE_DATA_REGION_NAME; import static org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor.VOLATILE_DATA_REGION_NAME; @@ -1157,21 +1156,10 @@ public WALPointer latestWalPointerReservedForPreloading() { } /** - * Computes the effective headroom (unallocated pages that can actually be allocated), accounting for per-segment - * allocation overhead (up to one page per segment). - * - * @param totalPages Total pages in the region (maxSize / systemPageSize). - * @param loadedPages Currently loaded pages. - * @return Effective headroom, never negative. - */ - private static long effectiveHeadroom(long totalPages, long loadedPages) { - long effectiveTotal = totalPages > SEG_CNT ? totalPages - SEG_CNT : 0; - - return Math.max(0, effectiveTotal - loadedPages); - } - - /** - * See {@code GridCacheMapEntry#ensureFreeSpace()} + * Evicts data pages from the given region until the configured eviction threshold is no longer exceeded or the + * empty-pages pool is refilled. This is the coarse, size-agnostic admission gate called at the batch level (before + * per-entry locks are acquired); size-aware per-row admission is handled separately by + * {@link org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList#reserve(int)}. * * @param memPlc data region. */ @@ -1179,11 +1167,6 @@ public void ensureFreeSpace(DataRegion memPlc) throws IgniteCheckedException { if (memPlc == null) return; - long totalPages = memPlc.config().getMaxSize() / memPlc.pageMemory().systemPageSize(); - - if (effectiveHeadroom(totalPages, memPlc.pageMemory().loadedPages()) > memPlc.config().getEmptyPagesPoolSize()) - return; - while (memPlc.evictionTracker().evictionRequired()) { if (memPlc.metrics().onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + @@ -1196,6 +1179,35 @@ public void ensureFreeSpace(DataRegion memPlc) throws IgniteCheckedException { } } + /** + * Reserves free-list pages in the given data region for a row (or a batch of rows totaling {@code rowSize} bytes) + * about to be inserted. See {@link FreeList#reserve(int)} for the caller contract (no entry/page locks held). + * + * @param memPlc Data region ({@code null} is tolerated). + * @param rowSize Size of the row, or the total size of all rows of a batch. + * @return Number of reserved pages to be passed to {@link FreeList#insertDataRow} and released via + * {@link #releaseInRegion(DataRegion, long)}, or {@code 0} if the reservation is not engaged for this region + * (persistent data region or disabled page eviction). + * @throws IgniteCheckedException If failed. + */ + public long reserveInRegion(DataRegion memPlc, int rowSize) throws IgniteCheckedException { + if (memPlc == null || memPlc.config().isPersistenceEnabled()) + return 0; + + return freeList(memPlc.config().getName()).reserve(rowSize); + } + + /** + * Releases the unconsumed portion of a reservation made by {@link #reserveInRegion(DataRegion, int)}. + * + * @param memPlc Data region the reservation was made in ({@code null} is tolerated). + * @param pages Number of pages to release. + */ + public void releaseInRegion(DataRegion memPlc, long pages) { + if (memPlc != null && pages != 0) + freeList(memPlc.config().getName()).releaseReserved(pages); + } + /** * @param memCfg memory configuration with common parameters. * @param plcCfg data region with PageMemory specific parameters. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java index 15cdb52067745..a9af7a2c80f82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java @@ -24,11 +24,17 @@ import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; +import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner; import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; /** * Data store for H2 rows. @@ -101,11 +107,12 @@ public void removeRow(long link, IoStatisticsHolder statHolder) throws IgniteChe /** * @param row Row. + * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. */ public void addRow(CacheDataRow row, IoStatisticsHolder statHolder) throws IgniteCheckedException { if (!persistenceEnabled) { - freeList.ensureFreeSpace(row.size()); + freeList.checkFitsForInsert(row.size()); freeList.insertDataRow(row, statHolder); } @@ -172,4 +179,31 @@ public void setRowCacheCleaner(Supplier rowCacheCleane this.rowCacheCleaner = rowCacheCleaner; } + + /** + * Computes the size a row with the given key and value occupies in page memory. Single source of truth for the + * row size: {@link CacheDataRowAdapter#size()} delegates here, and callers use it to reserve free-list pages + * before the actual row is created. + * + * @param key Row key. + * @param val Row value ({@code null} if unknown — under-reserves safely). + * @param verSize Serialized version size (see {@link CacheVersionIO#size(GridCacheVersion, boolean)}). + * @param coctx Cache object context to compute key/value bytes lengths. + * @param cacheId Cache ID stored in the data page. + * @return Row size in bytes. + * @throws IgniteCheckedException If failed to compute key or value bytes length. + */ + public static int rowSize( + KeyCacheObject key, + @Nullable CacheObject val, + int verSize, + CacheObjectValueContext coctx, + int cacheId + ) throws IgniteCheckedException { + int len = key.valueBytesLength(coctx); + + len += (val != null ? val.valueBytesLength(coctx) : 0) + verSize + 8; + + return len + (cacheId != 0 ? 4 : 0); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java index 26035641d5539..53b0609b670a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java @@ -65,10 +65,10 @@ public FairFifoPageEvictionTracker( } /** {@inheritDoc} */ - @Override public synchronized boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { + @Override public synchronized boolean evictDataPage() throws IgniteCheckedException { Integer pageIdx = pageUsageList.pollFirst(); - return pageIdx != null && evictDataPage(pageIdx, tryLock); + return pageIdx != null && evictDataPage(pageIdx); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java index 0e85f17148bf4..7f11ffd4df9ef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java @@ -38,7 +38,7 @@ public class NoOpPageEvictionTracker implements PageEvictionTracker { } /** {@inheritDoc} */ - @Override public boolean evictDataPage(boolean tryLock) { + @Override public boolean evictDataPage() { return false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java index d97a2b95e1c64..2330c0942662d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java @@ -32,8 +32,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; /** - * Base for page eviction trackers sharing the data-page eviction logic - * ({@link #evictDataPage(int, boolean)}). + * */ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker { /** This number of least significant bits is dropped from timestamp. */ @@ -90,13 +89,10 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker /** * @param pageIdx Page index. - * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries - * (e.g. when size-aware eviction runs while the current thread already holds entry locks), avoiding a - * lock-ordering deadlock. * @return true if at least one data row has been evicted * @throws IgniteCheckedException If failed. */ - final boolean evictDataPage(int pageIdx, boolean tryLock) throws IgniteCheckedException { + final boolean evictDataPage(int pageIdx) throws IgniteCheckedException { long fakePageId = PageIdUtils.pageId(0, (byte)0, pageIdx); long page = pageMem.acquirePage(0, fakePageId); @@ -148,7 +144,7 @@ final boolean evictDataPage(int pageIdx, boolean tryLock) throws IgniteCheckedEx GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) : cacheCtx.cache().entryEx(dataRow.key()); - evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true, tryLock); + evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true); } return evictionDone; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java index 49d6760608541..ff5cb78966d5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java @@ -38,32 +38,16 @@ public interface PageEvictionTracker extends LifecycleAware { */ public boolean evictionRequired(); - /** - * Evicts one data page (blocking flavor, equivalent to {@link #evictDataPage(boolean) evictDataPage(false)}). - * In most cases, all entries will be removed from the page. - * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be - * not possible, as some of them can be used by active transactions. - * - * @return {@code true} if at least one data row has been evicted. - * @throws IgniteCheckedException In case of page memory error. - */ - public default boolean evictDataPage() throws IgniteCheckedException { - return evictDataPage(false); - } - /** * Evicts one data page. * In most cases, all entries will be removed from the page. * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be * not possible, as some of them can be used by active transactions. * - * @param tryLock {@code true} to acquire entry locks non-blockingly, skipping contended or already-held entries - * instead of blocking on them. Used by size-aware eviction that may run while the current thread already - * holds other entry locks, to avoid a lock-ordering deadlock. * @return {@code true} if at least one data row has been evicted. * @throws IgniteCheckedException In case of page memory error. */ - public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException; + public boolean evictDataPage() throws IgniteCheckedException; /** * Call this method when last entry is removed from data page. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java index 88c0d046fb371..e075ec1f573c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java @@ -123,7 +123,7 @@ public Random2LruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { + @Override public boolean evictDataPage() throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -161,9 +161,8 @@ public Random2LruPageEvictionTracker( firstTs = first(trackingData); - // The link read above may be stale: a concurrent row deletion nulls the head slot first, and the freed - // head page can be re-used as an intermediate fragment of a new row, so its slot holds a link again - // and the two-hop walk can end on a link. This is a legal state: skip and resample. + // The link read above may be stale: a concurrently deleted row frees the head page, which can be + // re-used as an intermediate fragment holding a link again. Legal state: skip and resample. if (firstTs < 0) { sampleSpinCnt++; @@ -204,7 +203,7 @@ public Random2LruPageEvictionTracker( } } - if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) + if (evictDataPage(pageIdx(lruTrackingIdx))) return true; evictAttemptsCnt++; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java index e775d557c06dd..217abc826d6d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java @@ -109,7 +109,7 @@ public RandomLruPageEvictionTracker( } /** {@inheritDoc} */ - @Override public boolean evictDataPage(boolean tryLock) throws IgniteCheckedException { + @Override public boolean evictDataPage() throws IgniteCheckedException { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int evictAttemptsCnt = 0; @@ -141,9 +141,8 @@ public RandomLruPageEvictionTracker( compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4L); - // The link read above may be stale: a concurrent row deletion nulls the head slot first, and the freed - // head page can be re-used as an intermediate fragment of a new row, so its slot holds a link again - // and the two-hop walk can end on a link. This is a legal state: skip and resample. + // The link read above may be stale: a concurrently deleted row frees the head page, which can be + // re-used as an intermediate fragment holding a link again. Legal state: skip and resample. if (compactTs < 0) { sampleSpinCnt++; @@ -178,7 +177,7 @@ public RandomLruPageEvictionTracker( } } - if (evictDataPage(pageIdx(lruTrackingIdx), tryLock)) + if (evictDataPage(pageIdx(lruTrackingIdx))) return true; evictAttemptsCnt++; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 70f018bb6c586..028fdd2989bf4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -100,6 +100,13 @@ public abstract class AbstractFreeList extends PagesList imp */ private static final long EVICTION_MAX_CYCLE_TIME_MILLIS = 5 * EVICTION_NO_PROGRESS_TIMEOUT_MILLIS; + /** + * Number of compare-and-set attempts in {@link #tryReserve(int)}. A deficit (the space is genuinely unavailable) + * defers the row immediately; only a lost CAS race (concurrent reservers) is retried, up to this many attempts, + * before the row is deferred to the blocking context as well. + */ + private static final int TRY_RESERVE_CAS_ATTEMPTS = 4; + /** * Step between buckets in free list, measured in powers of two. * For example, for page size 4096 and 256 buckets, shift is 4 and step is 16 bytes. @@ -133,7 +140,7 @@ public abstract class AbstractFreeList extends PagesList imp /** Kernal context. */ private final GridKernalContext ctx; - /** Pages reserved by in-flight inserts (see {@link #acquireSpace(int)}). */ + /** Pages reserved by in-flight inserts (see {@link #reserve(int)}). */ private final AtomicLong reservedPages = new AtomicLong(); /** Page list cache limit. */ @@ -359,29 +366,21 @@ protected void putPage(int freeSpace, long pageId, long page, long pageAddr, IoS /** * Per-call write context of {@link #insertDataRows}: the rows cursor plus the reservation accounting of the row - * that is currently being written. The context is shared with {@link WriteRowsHandler}, which may consume the - * following rows from the cursor to fill up the page being written (acquiring their reservations itself), and - * with the outer loop of {@link #insertDataRows}, which continues a row left unfinished by the handler. + * currently being written, shared between {@link WriteRowsHandler} and the outer loop of {@link #insertDataRows}. */ private static final class RowsWriteContext { /** Rows cursor. */ private GridCursor cur; - /** Reservation made by {@link #acquireSpace(int)} for the current row. */ + /** Reservation made by {@link #reserve(int)} for the current row. */ private long reserved; /** Number of the current row's pages that have already consumed the reservation. */ private int consumed; /** - * Set by {@link #reserveAndWriteWholePages} when it could not claim the space without the guarded eviction - * while the caller holds a page write lock: the row is left to the outer loop of {@link #insertDataRows}. - */ - private boolean deferred; - - /** - * Set when a row was deferred by the page fill-up handler: the cursor stays positioned on it, so the outer - * loop of {@link #insertDataRows} must process it without advancing the cursor. + * Set when {@link WriteRowsHandler} could not reserve space for a row without eviction (forbidden under the + * page write lock); the cursor is positioned on that row, and the outer loop must continue without advancing. */ private boolean pending; } @@ -407,16 +406,17 @@ private final class WriteRowsHandler extends PageHandler, In T row = ctx.cur.get(); if (written == COMPLETE) { - int rowWritten = reserveAndWriteWholePages(ctx, row, statHolder, true); + long tryClaim = tryReserve(row.size()); - if (ctx.deferred) { - ctx.deferred = false; + if (tryClaim < 0) { ctx.pending = true; break; } - written = rowWritten; + claim(ctx, tryClaim); + + written = writeWholePages(ctx, row, statHolder); // If the data row was completely written without remainder, proceed to the next. if (written == COMPLETE) @@ -668,18 +668,12 @@ private int bucket(int freeSpace, boolean allowReuse) { @Override public void insertDataRow(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { int written = 0; - long reserved = acquireSpace(row.size()); - - int consumed = 0; - try { do { if (written != 0) memMetrics.incrementLargeEntriesPages(); written = writeSinglePage(row, written, statHolder); - - consumed++; } while (written != COMPLETE); } @@ -689,9 +683,6 @@ private int bucket(int freeSpace, boolean allowReuse) { catch (Throwable t) { throw new CorruptedFreeListException("Failed to insert data row", t, grpId); } - finally { - releaseSpace(reserved - consumed); - } } /** @@ -719,20 +710,8 @@ private int bucket(int freeSpace, boolean allowReuse) { T row = ctx.cur.get(); - // If eviction is required - free up memory before locking the next page. - while (evictionTracker.evictionRequired()) { - if (memMetrics.onPageEvictionsStarted()) { - U.warn(log, "Page-based evictions started." + - " Consider increasing 'maxSize' on Data Region configuration: " + dataRegion.config().getName()); - } - - evictionTracker.evictDataPage(); - - memMetrics.updateEvictionRate(); - } - if (written == COMPLETE) { - written = reserveAndWriteWholePages(ctx, row, statHolder, false); + written = reserveAndWriteWholePages(ctx, row, statHolder); continue; } @@ -747,10 +726,11 @@ private int bucket(int freeSpace, boolean allowReuse) { initIo = row.ioVersions().latest(); } - if (reserveEnabled) + if (reserveEnabled) { reservedPages.decrementAndGet(); - ctx.consumed++; + ctx.consumed++; + } written = write(pageId, writeRowsHnd, initIo, ctx, written, FAIL_I, statHolder); @@ -758,7 +738,7 @@ private int bucket(int freeSpace, boolean allowReuse) { } } finally { - releaseSpace(ctx.reserved - ctx.consumed); + releaseReserved(ctx.reserved - ctx.consumed); } } catch (IgniteOutOfMemoryException e) { @@ -770,63 +750,47 @@ private int bucket(int freeSpace, boolean allowReuse) { } /** - * Reserves the space for the next row and writes its fragments, which occupy the whole memory page. Every - * obtained page consumes the reservation made here. Releases the leftover reservation of the previous row (the - * personal empty-pages margin it was holding) before making a new one. + * Reserves the space for the next row (releasing the leftover reservation of the previous one) and writes its + * whole-page fragments, each consuming the reservation. * * @param ctx Write context holding the reservation accounting of the current row. * @param row Row to process. * @param statHolder Statistics holder to track IO operations. - * @param underPageLock {@code true} when the caller holds a page write lock: the guarded eviction is never run in - * this case — if the claim cannot be made without it, the row is deferred to the outer loop of - * {@link #insertDataRows} ({@code ctx.deferred} is set and {@code 0} is returned, the return value is to be - * ignored by the caller). * @return Number of bytes written, {@link #COMPLETE} if the row was fully written by whole-page fragments. * @throws IgniteCheckedException If failed. */ - private int reserveAndWriteWholePages( - RowsWriteContext ctx, - T row, - IoStatisticsHolder statHolder, - boolean underPageLock - ) throws IgniteCheckedException { - releaseSpace(ctx.reserved - ctx.consumed); - - ctx.reserved = 0; - ctx.consumed = 0; - - ctx.reserved = underPageLock ? tryAcquireSpace(row.size()) : acquireSpace(row.size()); + private int reserveAndWriteWholePages(RowsWriteContext ctx, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { + claim(ctx, reserve(row.size())); - if (ctx.reserved < 0) { - ctx.deferred = true; - - return 0; - } - - int written = writeWholePages(row, statHolder); + return writeWholePages(ctx, row, statHolder); + } - // Each whole-page fragment consumes exactly MIN_SIZE_FOR_DATA_PAGE bytes, and writeSinglePage has already - // decremented reservedPages for each of them: account the fragments in the local counter. If writeWholePages - // fails midway, the fragments it has already written are over-consumed here - bounded by the row size and - // acceptable for a failing insert. - ctx.consumed = written != COMPLETE - ? written / MIN_SIZE_FOR_DATA_PAGE - : (row.size() + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; + /** + * Releases the unconsumed remainder of the previous row's reservation and stores the new claim for the current row. + * + * @param ctx Write context holding the reservation accounting of the current row. + * @param claimPages Pages claimed for the current row. + */ + private void claim(RowsWriteContext ctx, long claimPages) { + releaseReserved(ctx.reserved - ctx.consumed); - return written; + ctx.reserved = claimPages; + ctx.consumed = 0; } /** * Write fragments of the row, which occupy the whole memory page. A data row is ignored if it is less than the max - * payload of an empty data page. + * payload of an empty data page. Every written fragment consumes one reserved page (accounted in {@code ctx} + * right next to the {@code reservedPages} decrement), so the accounting is exact on any outcome. * + * @param ctx Write context holding the reservation accounting of the current row. * @param row Row to process. * @param statHolder Statistics holder to track IO operations. * @return Number of bytes written, {@link #COMPLETE} if the row was fully written, {@code 0} if data row was * ignored because it is less than the max payload of an empty data page. * @throws IgniteCheckedException If failed. */ - private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { + private int writeWholePages(RowsWriteContext ctx, T row, IoStatisticsHolder statHolder) throws IgniteCheckedException { assert row.link() == 0 : row.link(); int written = 0; @@ -835,6 +799,12 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC while (rowSize - written >= MIN_SIZE_FOR_DATA_PAGE) { written = writeSinglePage(row, written, statHolder); + if (reserveEnabled) { + reservedPages.decrementAndGet(); + + ctx.consumed++; + } + memMetrics.incrementLargeEntriesPages(); } @@ -842,7 +812,9 @@ private int writeWholePages(T row, IoStatisticsHolder statHolder) throws IgniteC } /** - * Take a page and write row on it. Every obtained page consumes the reservation made by {@link #acquireSpace(int)}. + * Take a page and write row on it. The reservation made by {@link #reserve(int)} (when engaged for the row's + * insert) is released as a whole by the caller once the insert completes, so pages taken here are not + * individually accounted against it. * * @param row Row to write. * @param written Written size. @@ -861,9 +833,6 @@ private int writeSinglePage(T row, int written, IoStatisticsHolder statHolder) t initIo = row.ioVersions().latest(); } - if (reserveEnabled) - reservedPages.decrementAndGet(); - written = write(pageId, writeRowHnd, initIo, row, written, FAIL_I, statHolder); assert written != FAIL_I; // We can't fail here. @@ -1076,72 +1045,14 @@ public int emptyDataPages() { } /** {@inheritDoc} */ - @Override public void ensureFreeSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { - if (dataRegion.config().isPersistenceEnabled()) - return; - - if (dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) { - long nonEmptyPages = pageMem.loadedPages() - emptyDataPages(); - - // The maximum number of pages that can be allocated (maxSize / systemPageSize) - // should be greater or equal to pages required for inserting a new entry plus - // the number of currently non-empty data pages. Also we account for pages that hold pages-list metadata: - // 8 bytes per each row on average, considering that maybe not all list pages are full, multiply by 1.5. - // Plus, add one page for each bucket in the worst case. - // Note that not the whole page can be used to storing links, - // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5, - // in any way, the number of required pages is less than 1 percent. - boolean oomThreshold = (dataRegion.config().getMaxSize() / pageMem.systemPageSize()) < - ((double)rowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256 /*one page per bucket*/); - - if (oomThreshold) - throw outOfMemory(dataRegion.config()); - } - } - - /** {@inheritDoc} */ - @Override public String toString() { - return "FreeList [name=" + name() + ']'; - } - - /** - * @param part Partition. - * @return Page ID. - * @throws IgniteCheckedException If failed. - */ - private long allocateDataPage(int part) throws IgniteCheckedException { - assert part <= PageIdAllocator.MAX_PARTITION_ID; + @Override public long reserve(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { + if (!reserveEnabled) { + checkDisabledEvictionOom(rowSize); - return pageMem.allocatePage(grpId, part, FLAG_DATA); - } - - /** - * Reserves the space for a row of the given size in an in-memory region with enabled page eviction: runs eviction - * until {@code freePages - reservedPages} covers the whole demand of the writer — {@code requiredPages} for the row - * itself plus a personal copy of the {@code emptyPagesPoolSize} margin — then atomically adds the demand to - * {@code reservedPages}. - * - * @param rowSize Size of the row to be inserted. - * @return Number of reserved pages (the row's pages plus the claimed margin) to be released via - * {@link #releaseSpace(long)}, or {@code 0} if the reservation is not engaged for this free list (persistent data - * region or disabled page eviction — see {@link #reserveEnabled}). - * @throws IgniteOutOfMemoryException If the row cannot fit into the region, or eviction makes no progress. - * @throws IgniteCheckedException If failed to evict data pages. - */ - private long acquireSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException { - if (!reserveEnabled) return 0; + } - DataRegionConfiguration cfg = dataRegion.config(); - - long totalPages = cfg.getMaxSize() / pageMem.systemPageSize(); - - long requiredPages = (rowSize + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; - - if (requiredPages > totalPages) - throw outOfMemory(cfg); - - long claimPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); + long claimPages = pagesToReserve(rowSize); long backoffNanos = EVICTION_BACKOFF_START_NANOS; @@ -1149,6 +1060,8 @@ private long acquireSpace(int rowSize) throws IgniteOutOfMemoryException, Ignite long cycleDeadlineNanos = lastProgressNanos + TimeUnit.MILLISECONDS.toNanos(EVICTION_MAX_CYCLE_TIME_MILLIS); + long totalPages = totalPages(); + while (true) { long deficit = claimPages - (freePages(totalPages) - reservedPages.get()); @@ -1163,10 +1076,10 @@ private long acquireSpace(int rowSize) throws IgniteOutOfMemoryException, Ignite if (memMetrics.onPageEvictionsStarted()) { U.warn(log, "Page-based evictions started." + - " Consider increasing 'maxSize' on Data Region configuration: " + cfg.getName()); + " Consider increasing 'maxSize' on Data Region configuration: " + dataRegion.config().getName()); } - boolean evicted = evictionTracker.evictDataPage(true); + boolean evicted = evictionTracker.evictDataPage(); memMetrics.updateEvictionRate(); @@ -1187,52 +1100,107 @@ private long acquireSpace(int rowSize) throws IgniteOutOfMemoryException, Ignite if (nowNanos - lastProgressNanos > TimeUnit.MILLISECONDS.toNanos(EVICTION_NO_PROGRESS_TIMEOUT_MILLIS) || nowNanos > cycleDeadlineNanos) - throw outOfMemory(cfg); + throw outOfMemory(dataRegion.config()); } } + /** {@inheritDoc} */ + @Override public void releaseReserved(long pages) { + assert pages >= 0; + + if (reserveEnabled && pages != 0) + reservedPages.addAndGet(-pages); + } + + /** {@inheritDoc} */ + @Override public void checkFitsForInsert(int rowSize) throws IgniteOutOfMemoryException { + checkDisabledEvictionOom(rowSize); + } + /** - * Non-blocking flavor of {@link #acquireSpace(int)} for callers holding a page write lock: claims the space only - * if it is instantly available; never runs the guarded eviction and never parks. + * Attempts to reserve space for a row without blocking on eviction. Unlike {@link #reserve(int)}, this method + * never calls {@code evictionTracker.evictDataPage()} (which would take an entry lock), so it is safe to call + * while holding a page write lock. * - * @param rowSize Size of the row to be inserted. - * @return Number of reserved pages to be released via {@link #releaseSpace(long)}, {@code 0} if the reservation is - * not engaged for this free list (see {@link #reserveEnabled}), or {@code -1} if the claim cannot be made without - * the guarded eviction or raced with a concurrent one. + * @param rowSize Row size in bytes. + * @return The number of claimed pages (>= 0) if the reservation succeeded without eviction, or {@code -1} if + * the claim would require eviction (the caller should defer the row to a context where blocking is safe). + * @throws IgniteOutOfMemoryException If the row is larger than the region or the region is exhausted (disabled + * eviction in-memory OOM check). */ - private long tryAcquireSpace(int rowSize) { - if (!reserveEnabled) + private long tryReserve(int rowSize) throws IgniteOutOfMemoryException { + if (!reserveEnabled) { + checkDisabledEvictionOom(rowSize); + return 0; + } + + long claimPages = pagesToReserve(rowSize); + long totalPages = totalPages(); + + for (int i = 0; i < TRY_RESERVE_CAS_ATTEMPTS; i++) { + long reserved = reservedPages.get(); - DataRegionConfiguration cfg = dataRegion.config(); + if (claimPages - (freePages(totalPages) - reserved) > 0) + return -1; - long totalPages = cfg.getMaxSize() / pageMem.systemPageSize(); + if (reservedPages.compareAndSet(reserved, reserved + claimPages)) + return claimPages; + } + + return -1; + } + + /** + * Computes the number of pages to reserve for a row, validating that the row can fit into the data region. + * + * @param rowSize Row size in bytes. + * @return Number of pages to claim: the pages required for the row plus the empty pages pool, capped at the region size. + * @throws IgniteOutOfMemoryException If the row is larger than the whole data region. + */ + private long pagesToReserve(int rowSize) throws IgniteOutOfMemoryException { + long totalPages = totalPages(); long requiredPages = (rowSize + MIN_SIZE_FOR_DATA_PAGE - 1) / MIN_SIZE_FOR_DATA_PAGE; if (requiredPages > totalPages) - return -1; + throw outOfMemory(dataRegion.config()); - long claimPages = Math.min(requiredPages + cfg.getEmptyPagesPoolSize(), totalPages); + return Math.min(requiredPages + dataRegion.config().getEmptyPagesPoolSize(), totalPages); + } + + /** @return Total number of pages in the data region (maxSize / systemPageSize). */ + private long totalPages() { + return dataRegion.config().getMaxSize() / pageMem.systemPageSize(); + } - long reserved = reservedPages.get(); + /** + * OOM threshold check for the disabled-eviction in-memory case (previously in {@code ensureFreeSpace}). + * + * @param rowSize Row size in bytes. + * @throws IgniteOutOfMemoryException If the region is near exhaustion. + */ + private void checkDisabledEvictionOom(int rowSize) throws IgniteOutOfMemoryException { + if (!dataRegion.config().isPersistenceEnabled() && dataRegion.config().getPageEvictionMode() == DataPageEvictionMode.DISABLED) { + long nonEmptyPages = pageMem.loadedPages() - emptyDataPages(); - if (claimPages - (freePages(totalPages) - reserved) > 0 || !reservedPages.compareAndSet(reserved, reserved + claimPages)) - return -1; + boolean oomThreshold = + totalPages() < ((double)rowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256); - return claimPages; + if (oomThreshold) + throw outOfMemory(dataRegion.config()); + } } /** - * Releases the reservation made by {@link #acquireSpace(int)}: consumed pages were already decremented - * {@code reservedPages} by {@link #writeSinglePage} and the continuation path of {@link #insertDataRows}, so - * only the unconsumed remainder (the row's personal empty-pages margin) is subtracted here. - * - * @param pages Number of pages to release. + * @param part Partition. + * @return Page ID. + * @throws IgniteCheckedException If failed. */ - private void releaseSpace(long pages) { - if (reserveEnabled && pages != 0) - reservedPages.addAndGet(-pages); + private long allocateDataPage(int part) throws IgniteCheckedException { + assert part <= PageIdAllocator.MAX_PARTITION_ID; + + return pageMem.allocatePage(grpId, part, FLAG_DATA); } /** @@ -1247,17 +1215,14 @@ private long freePages(long totalPages) { /** * Computes the effective headroom (unallocated pages that can actually be allocated), accounting for per-segment - * allocation overhead (up to one page per segment). Mirrors the same-named helper in - * IgniteCacheDatabaseSharedManager. + * allocation overhead (up to one page per segment). * * @param totalPages Total pages in the region (maxSize / systemPageSize). * @param loadedPages Currently loaded pages. * @return Effective headroom, never negative. */ private static long effectiveHeadroom(long totalPages, long loadedPages) { - long effectiveTotal = totalPages > SEG_CNT ? totalPages - SEG_CNT : 0; - - return Math.max(0, effectiveTotal - loadedPages); + return Math.max(0, (totalPages > SEG_CNT ? totalPages - SEG_CNT : 0) - loadedPages); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java index 2ea0e4a55d2b2..38902e6cf321f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java @@ -24,10 +24,15 @@ import org.apache.ignite.internal.metric.IoStatisticsHolder; import org.apache.ignite.internal.processors.cache.persistence.Storable; -/** - */ +/** */ public interface FreeList { /** + * Inserts a data page by page. When the caller reserved space via {@link #reserve(int)} before acquiring the + * entry lock, the whole reservation is released by the caller via {@link #releaseReserved(long)} once the + * insert completes (normally or not): pages taken here are not individually accounted against it, so an + * exception in the middle of the insert cannot drift the reservation counter. Does not run eviction or acquire + * space. + * * @param row Row. * @param statHolder Statistics holder to track IO operations. * @throws IgniteCheckedException If failed. @@ -58,16 +63,37 @@ public interface FreeList { public void removeDataRowByLink(long link, IoStatisticsHolder statHolder) throws IgniteCheckedException; /** - * Ensures the data region backing this free list can accommodate a row of the given size. + * @param log Logger. + */ + public void dumpStatistics(IgniteLogger log); + + /** + * Reserves pages for a row of the given size before the entry lock is acquired. Runs blocking eviction when + * necessary (the caller must not hold any entry or page locks). * * @param rowSize Size of the row to be inserted. - * @throws IgniteOutOfMemoryException If the row cannot be accommodated even after eviction. + * @return Number of reserved pages to be released via {@link #releaseReserved(long)} once the row insert + * completes, or {@code 0} if the reservation is not engaged for this free list (persistent + * data region or disabled page eviction). + * @throws IgniteOutOfMemoryException If the row cannot fit into the region, or eviction makes no progress. * @throws IgniteCheckedException If failed to evict data pages. */ - public void ensureFreeSpace(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException; + public long reserve(int rowSize) throws IgniteOutOfMemoryException, IgniteCheckedException; /** - * @param log Logger. + * Releases the unconsumed portion of a reservation made by {@link #reserve(int)}. + * + * @param pages Number of pages to release (must not be negative). */ - public void dumpStatistics(IgniteLogger log); + public void releaseReserved(long pages); + + /** + * Early out-of-memory check for an in-memory data region with disabled page eviction: verifies that the region + * still has room for a row of the given size with a safety margin for free-list bookkeeping pages. No-op for + * persistent regions or regions with eviction enabled. + * + * @param rowSize Row size in bytes. + * @throws IgniteOutOfMemoryException If the region is near exhaustion. + */ + public void checkFitsForInsert(int rowSize) throws IgniteOutOfMemoryException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java index fd7f2f5db8691..c2ba4e0c998ed 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java @@ -40,7 +40,7 @@ public class CacheVersionIO { private static final int SIZE_V1 = 17; /** Serialized size in bytes. */ - private static final int SIZE_V2 = 33; + public static final int SIZE_V2 = 33; /** * @param ver Version. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java index 3541aa97b63cd..f7f278bd7ca15 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java @@ -59,6 +59,16 @@ public class PageEvictionAbstractTest extends GridCommonAbstractTest { /** Default policy name. */ private static final String DEFAULT_POLICY_NAME = "dfltPlc"; + /** @return Deterministic per-key value content. */ + public static byte[] valueFor(int key, int size) { + byte[] val = new byte[size]; + + for (int i = 0; i < size; i++) + val[i] = (byte)(key * 31 + i); + + return val; + } + /** * @param mode Eviction mode. * @param configuration Configuration. @@ -139,20 +149,14 @@ protected CacheConfiguration cacheConfig( return cacheConfiguration; } - /** - * @param ignite Ignite node. - * @param cacheName Cache name. - * @return Cache with a small partition count. - */ + /** @param cacheName Cache name. */ protected static IgniteCache createCache(IgniteEx ignite, String cacheName) { return createCache(ignite, cacheName, 0, false); } /** - * @param ignite Ignite node. - * @param cacheName Cache name. * @param ttl TTL in milliseconds ({@code 0} for no expiry). - * @return Cache with a small partition count and, if {@code ttl > 0}, eager TTL expiry. + * @return Cache with eager TTL expiry if {@code ttl > 0}. */ protected static IgniteCache createCache(IgniteEx ignite, String cacheName, long ttl, boolean transactional) { CacheConfiguration ccfg = new CacheConfiguration(cacheName) @@ -191,4 +195,26 @@ protected static long totalPages(IgniteEx ignite) throws IgniteCheckedException protected static boolean isEvictionsStarted(IgniteEx ignite) throws IgniteCheckedException { return defaultRegion(ignite).metrics().isEvictionsStarted(); } + + /** + * Pre-fills the cache with page-sized entries until the loaded page count reaches {@code fraction} of the total + * pages in the default data region. Entries are written with zero-filled {@code pageSize - 200} byte values at + * consecutive keys starting from {@code 0}. + * + * @param ignite Node. + * @param cache Cache. + * @param fraction Target fraction of total pages to load (e.g. {@code 0.85}). + * @return Number of entries written. + * @throws Exception If failed. + */ + protected static int fillRegion(IgniteEx ignite, IgniteCache cache, double fraction) throws Exception { + int entries = (int)(totalPages(ignite) * fraction); + + byte[] small = new byte[pageSize(ignite) - 200]; + + for (int i = 0; i < entries; i++) + cache.put(i, small); + + return entries; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java index 90eb9b7099c8b..8c42fd7e93c30 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionConcurrentWritesAbstractTest.java @@ -18,10 +18,13 @@ package org.apache.ignite.internal.processors.cache.eviction.paged; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -38,8 +41,8 @@ public abstract class PageEvictionConcurrentWritesAbstractTest extends PageEvict } /** - * Concurrent large inserts into a region pre-filled with small entries must complete within the deadline without - * deadlock, and without corrupting the free list (eviction frees small entries rather than overrunning the region). + * Concurrent large inserts into a region pre-filled with small entries must complete without deadlock and without + * corrupting the free list. * * @throws Exception If failed. */ @@ -55,7 +58,7 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { long regionMax = regionMaxSize(ignite); - // Small value: sized so that 'smallEntries' of them fill ~80% of the region, staying under the 90% eviction threshold. + // Small values fill ~80% of the region, below the 0.9 eviction threshold. int smallValSize = (int)(regionMax * 0.8 / smallEntries); for (int i = 0; i < smallEntries; i++) @@ -66,7 +69,7 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { CountDownLatch startLatch = new CountDownLatch(1); AtomicInteger threadIdx = new AtomicInteger(); - // Large value: 5% of the region — large enough to trigger eviction during concurrent writes. + // Large values (5% of the region) trigger eviction during concurrent writes. int largeValSize = (int)(regionMax / 20); IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { @@ -87,7 +90,7 @@ public void testConcurrentLargeWritesNoDeadlock() throws Exception { assertTrue("Eviction must have started during concurrent large writes", isEvictionsStarted(ignite)); - // Verify the free list is not corrupted: a new put+get must succeed. + // Probe put+get: the free list must not be corrupted. int probeKey = smallEntries + threads * largeRowsPerThread + 1; byte[] probeVal = valueFor(probeKey, pageSize(ignite) - 200); @@ -177,29 +180,126 @@ public void testRebalanceBatchInsertsWithEviction() throws Exception { } /** - * @param key Key the value is stored under. - * @param size Value size in bytes. - * @return Deterministic per-key value content. + * Concurrent {@code putAll} batches on a near-full region with enabled eviction must complete without deadlock. + * + * @throws Exception If failed. */ - private static byte[] valueFor(int key, int size) { - byte[] val = new byte[size]; + @Test + public void testConcurrentPutAllNoDeadlock() throws Exception { + IgniteEx ignite = startGrid(0); + + IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(0) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC)); + + int pSize = pageSize(ignite); + + // Pre-fill to ~70% of the region. + int smallValSize = pSize / 4 - 50; + int smallEntries = (int)(regionMaxSize(ignite) * 0.7 / smallValSize); + + for (int i = 0; i < smallEntries; i++) + cache.put(i, valueFor(i, smallValSize)); + + int valSize = pSize / 4 - 50; + + CountDownLatch startLatch = new CountDownLatch(1); + AtomicInteger keyGen = new AtomicInteger(smallEntries); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { + U.awaitQuiet(startLatch); - for (int i = 0; i < size; i++) - val[i] = (byte)(key * 31 + i); + for (int b = 0; b < 10; b++) { + Map batch = new HashMap<>(); - return val; + for (int i = 0; i < 50; i++) { + int key = keyGen.incrementAndGet(); + + batch.put(key, valueFor(key, valSize)); + } + + cache.putAll(batch); + } + }, 4, "batch-writer"); + + startLatch.countDown(); + + fut.get(TimeUnit.MINUTES.toMillis(3)); + + assertTrue("Eviction must have started during concurrent batch writes", isEvictionsStarted(ignite)); + + int probeKey = keyGen.incrementAndGet(); + byte[] probeVal = valueFor(probeKey, smallValSize); + + cache.put(probeKey, probeVal); + + byte[] read = cache.get(probeKey); + + assertNotNull(read); + assertTrue(Arrays.equals(probeVal, read)); } /** - * Reads {@code cnt} entries starting from {@code base} and asserts that every non-null value is byte-identical - * to the deterministic content written for its key. + * Concurrent single-row puts interleaved with {@code putAll} batches on a near-full region. * - * @param cache Cache to read from. - * @param base First key. - * @param cnt Number of keys to check. - * @param valSize Size of the values written for these keys. - * @return Number of entries that survived eviction. + * @throws Exception If failed. */ + @Test + public void testConcurrentPutsAndPutAllNoDeadlock() throws Exception { + IgniteEx ignite = startGrid(0); + + IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(0) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC)); + + int valSize = pageSize(ignite) / 4 - 50; + int smallEntries = (int)(regionMaxSize(ignite) * 0.7 / valSize); + + for (int i = 0; i < smallEntries; i++) + cache.put(i, valueFor(i, valSize)); + + AtomicInteger keyGen = new AtomicInteger(smallEntries); + + CountDownLatch startLatch = new CountDownLatch(1); + + AtomicInteger threadIdx = new AtomicInteger(); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { + U.awaitQuiet(startLatch); + + // Even threads do putAll batches, odd threads do single puts. + boolean isBatch = (threadIdx.getAndIncrement() & 1) == 0; + + for (int op = 0; op < 50; op++) { + if (isBatch) { + Map batch = new HashMap<>(); + + for (int i = 0; i < 20; i++) { + int key = keyGen.incrementAndGet(); + + batch.put(key, valueFor(key, valSize)); + } + + cache.putAll(batch); + } + else { + int key = keyGen.incrementAndGet(); + + cache.put(key, valueFor(key, valSize)); + } + } + }, 6, "mixed-writer"); + + startLatch.countDown(); + + fut.get(TimeUnit.MINUTES.toMillis(3)); + + assertTrue("Eviction must have started", isEvictionsStarted(ignite)); + } + + /** Asserts every non-null value in {@code [base, base+cnt)} is byte-identical to {@code valueFor(k, valSize)}. */ private static int verifyContent(IgniteCache cache, int base, int cnt, int valSize) { int survived = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDisabledEvictionReadThroughOomSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDisabledEvictionReadThroughOomSelfTest.java new file mode 100644 index 0000000000000..f1967cd8f5fa6 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDisabledEvictionReadThroughOomSelfTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import javax.cache.Cache; +import javax.cache.configuration.Factory; +import javax.cache.integration.CacheLoaderException; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.store.CacheStoreAdapter; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; + +/** + * Regression test for the disabled-eviction in-memory OOM guard on insert paths that bypass {@code FreeList.reserve}: + * read-through inside {@code get()} runs under the entry lock and cannot reserve, so the per-insert guard previously + * living in {@code RowStore.addRow} (now {@code FreeList.checkFitsForInsert}) must still fire and reject the insert + * with {@link IgniteOutOfMemoryException} when the region is at the threshold — rather than silently consuming the + * free-list bookkeeping margin. + */ +public class PageEvictionDisabledEvictionReadThroughOomSelfTest extends PageEvictionAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.DISABLED, super.getConfiguration(gridName)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * Fills the region until a {@code put} hits the disabled-eviction OOM guard, then verifies that a read-through + * {@code get} on a missing key also throws {@link IgniteOutOfMemoryException} from the same guard (the bypass + * path must not silently consume the remaining free-list margin). + * + * @throws Exception If failed. + */ + @Test(timeout = 120_000) + public void testReadThroughGetOomAtThreshold() throws Exception { + IgniteEx ignite = startGrid(0); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC) + .setReadThrough(true) + .setCacheStoreFactory(new TestStoreFactory()); + + IgniteCache cache = grid(0).getOrCreateCache(cfg); + + byte[] small = new byte[pageSize(ignite) - 200]; + + // Fill the region until the disabled-eviction OOM guard fires on a put. + int filled = 0; + + for (int i = 0; i < 100_000; i++) { + try { + cache.put(i, small); + + filled++; + } + catch (Exception e) { + assertTrue("Expected IgniteOutOfMemoryException while filling the region, got: " + e, + X.hasCause(e, IgniteOutOfMemoryException.class)); + + break; + } + } + + assertTrue("Region must be filled before OOM (filled=" + filled + ")", filled > 0); + + // Read-through on a missing key: the store returns a page-sized value, which must be rejected by the + // disabled-eviction OOM guard at the RowStore level (the bypass path cannot reserve). + int missingKey = filled + 1; + + GridTestUtils.assertThrowsWithCause(() -> cache.get(missingKey), IgniteOutOfMemoryException.class); + } + + /** + * Below the threshold, read-through {@code get} must succeed — the restored guard must not spuriously reject. + * + * @throws Exception If failed. + */ + @Test + public void testReadThroughGetBelowThresholdSucceeds() throws Exception { + startGrid(0); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC) + .setReadThrough(true) + .setCacheStoreFactory(new TestStoreFactory()); + + IgniteCache cache = grid(0).getOrCreateCache(cfg); + + // A single read-through load well below capacity must succeed. + byte[] val = cache.get(1); + + assertNotNull("Read-through load must succeed below threshold", val); + assertEquals(PAGE_SIZE - 200, val.length); + } + + /** */ + private static class TestStoreFactory implements Factory { + /** {@inheritDoc} */ + @Override public TestCacheStore create() { + return new TestCacheStore(); + } + } + + /** */ + private static class TestCacheStore extends CacheStoreAdapter { + /** {@inheritDoc} */ + @Override public byte[] load(Integer key) throws CacheLoaderException { + return new byte[PAGE_SIZE - 200]; + } + + /** {@inheritDoc} */ + @Override public void write(Cache.Entry entry) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void delete(Object key) { + // No-op. + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionEnsureFreeSpaceThresholdTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionEnsureFreeSpaceThresholdTest.java new file mode 100644 index 0000000000000..240a38118cba9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionEnsureFreeSpaceThresholdTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.junit.Test; + +/** + * Regression test for C2: the batch-level {@code ensureFreeSpace} must evict at the configured + * {@code evictionThreshold} (default 0.9), not silently skip eviction when the headroom exceeds the empty-pages pool. + */ +public class PageEvictionEnsureFreeSpaceThresholdTest extends PageEvictionAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_2_LRU, super.getConfiguration(gridName)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** */ + @Test + public void testSequentialLargePutsEvictAtThreshold() throws Exception { + IgniteEx ignite = startGrid(1); + + IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); + + long regionMax = regionMaxSize(ignite); + + int largeValSize = (int)(regionMax / 20); + + int largeRows = 25; + + for (int k = 0; k < largeRows; k++) { + byte[] val = new byte[largeValSize]; + + for (int i = 0; i < val.length; i++) + val[i] = (byte)(k * 31 + i); + + cache.put(k, val); + } + + assertTrue("Eviction must have started", isEvictionsStarted(ignite)); + + int survived = 0; + + for (int k = 0; k < largeRows; k++) { + byte[] read = (byte[])cache.get(k); + + if (read == null) + continue; + + survived++; + + assertEquals(largeValSize, read.length); + + for (int i = 0; i < read.length; i++) + assertEquals((byte)(k * 31 + i), read[i]); + } + + assertTrue("At least one large entry must survive", survived > 0); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java index 4bb8d20842685..1de971a5ce2df 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionGuardOomTest.java @@ -44,8 +44,8 @@ public class PageEvictionGuardOomTest extends PageEvictionAbstractTest { } /** - * Filling the region with locked entries and then writing a row that needs more free pages than remain must fail - * with OOM (bounded time), not hang: eviction cannot free any page because every candidate entry is locked. + * With every candidate entry locked, eviction cannot free any page, so a row needing more free pages than remain + * must fail with OOM (bounded time), not hang. * * @throws Exception If failed. */ @@ -55,7 +55,7 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME, 0, true); - // Fill to ~85% of total pages, below the 0.9 eviction threshold. Each entry is ~one page. + // Fill to ~85% of total pages (below the 0.9 threshold), ~1 page per entry. int fillEntries = (int)(totalPages(ignite) * 0.85); byte[] fillVal = new byte[pageSize(ignite) - 200]; @@ -72,7 +72,7 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { CountDownLatch release = new CountDownLatch(1); - // Hold entry locks on every resident key from a background thread so that eviction has no evictable page. + // Hold entry locks on every resident key from a background thread so eviction has no evictable page. IgniteInternalFuture lockerFut = GridTestUtils.runAsync(() -> { Lock lock = cache.lockAll(keys); @@ -91,7 +91,7 @@ public void testGuardOomWhenAllEntriesLocked() throws Exception { try { assertTrue("Timed out waiting for entries to be locked", ready.await(60, TimeUnit.SECONDS)); - // Large row that needs more free pages than remain — eviction cannot free any page (all locked) → OOM. + // All pages locked → eviction cannot free anything → OOM. GridTestUtils.assertThrowsWithCause( () -> cache.put(fillEntries + 1, new byte[4 * 1024 * 1024]), IgniteOutOfMemoryException.class diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java index 1e637f81985da..fa774690ea372 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java @@ -50,8 +50,7 @@ public void testPageEvictionMetric() throws Exception { } /** - * Regression test: ordinary small records that keep the region below the eviction threshold must not trigger page - * eviction at all (eviction is not started, eviction rate stays zero). + * Small records below the eviction threshold must not trigger page eviction. * * @throws Exception If failed. */ @@ -68,7 +67,6 @@ public void testNoEvictionBelowThreshold() throws Exception { IgniteCache cache = ignite.getOrCreateCache(cfg); - // A small number of records far below the eviction threshold and empty-pages pool pressure. for (int i = 1; i <= 500; i++) cache.put(i, new TestObject(PAGE_SIZE / 6)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java index b7138964e9572..898cb287fc98f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java @@ -30,7 +30,7 @@ * */ public abstract class PageEvictionMultinodeAbstractTest extends PageEvictionAbstractTest { - /** Number of entries, sized to fill the data region past the headroom gate in {@code ensureFreeSpace}. */ + /** Number of entries, sized to fill the data region past the eviction threshold. */ private static final int MULTINODE_ENTRIES = 16_000; /** Cache modes. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughReservedPagesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughReservedPagesSelfTest.java new file mode 100644 index 0000000000000..e94ce5198fe0e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughReservedPagesSelfTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import javax.cache.Cache; +import javax.cache.configuration.Factory; +import javax.cache.integration.CacheLoaderException; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.store.CacheStoreAdapter; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListReservedPagesAbstractTest; +import org.junit.Test; + +/** + * Read-through loads go through {@code versionedValue}, which must reserve pages before the entry lock and release the + * remainder after the insert. After loads complete and the cluster quiesces, {@code reservedPages} must be 0. + */ +public class PageEvictionReadThroughReservedPagesSelfTest extends PageEvictionAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * Read-through loads trigger {@code versionedValue}; {@code reservedPages} must be released. + * + * @throws Exception If failed. + */ + @Test + public void testReservedPagesReleasedAfterReadThrough() throws Exception { + startGrid(0); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC) + .setReadThrough(true) + .setCacheStoreFactory(new TestStoreFactory()); + + IgniteCache cache = grid(0).getOrCreateCache(cfg); + + for (int i = 1; i <= 20_000; i++) + cache.get(i); + + assertTrue("Eviction must have started during read-through loads", isEvictionsStarted(grid(0))); + + byte[] val = cache.get(1); + + assertNotNull("Value for key 1 must not be null", val); + assertEquals(PAGE_SIZE - 200, val.length); + + FreeListReservedPagesAbstractTest.assertAllReleased(DEFAULT_CACHE_NAME, 10_000); + } + + /** + * Read-through on a 2-node cluster with backups; {@code reservedPages} must be released on every node. + * + * @throws Exception If failed. + */ + @Test + public void testReservedPagesReleasedAfterReadThroughOnBackup() throws Exception { + startGrids(2); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setBackups(1) + .setReadThrough(true) + .setCacheStoreFactory(new TestStoreFactory()); + + IgniteCache cache = grid(0).getOrCreateCache(cfg); + + awaitPartitionMapExchange(); + + for (int i = 1; i <= 15_000; i++) + cache.get(i); + + assertTrue("Eviction must have started", isEvictionsStarted(grid(0))); + + FreeListReservedPagesAbstractTest.assertAllReleased(DEFAULT_CACHE_NAME, 10_000); + } + + /** */ + private static class TestStoreFactory implements Factory { + /** {@inheritDoc} */ + @Override public TestCacheStore create() { + return new TestCacheStore(); + } + } + + /** */ + private static class TestCacheStore extends CacheStoreAdapter { + /** {@inheritDoc} */ + @Override public byte[] load(Integer key) throws CacheLoaderException { + // Fills most of a page so ~20k entries exceed the 20 MB region and trigger eviction. + return new byte[PAGE_SIZE - 200]; + } + + /** {@inheritDoc} */ + @Override public void write(Cache.Entry entry) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void delete(Object key) { + // No-op. + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionRowSizeBinaryObjectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionRowSizeBinaryObjectTest.java new file mode 100644 index 0000000000000..8878e868d39ba --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionRowSizeBinaryObjectTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.junit.Test; + +/** + * Regression test for C1: {@code GridCacheMapEntry#rowSize()} must not throw NPE when the value is an unmarshalled + * {@code BinaryObjectImpl} or a POJO whose key has not yet been marshalled on the reservation path. + */ +public class PageEvictionRowSizeBinaryObjectTest extends PageEvictionAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_2_LRU, super.getConfiguration(gridName)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * POJO puts on the transactional commit path must not throw NPE inside {@code rowSize}. + * + * @throws Exception If failed. + */ + @Test + public void testPojoPutTxDoesNotNpeInRowSize() throws Exception { + IgniteEx ignite = startGrid(0); + + IgniteCache cache = ignite.createCache( + new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setCacheMode(CacheMode.REPLICATED) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC)); + + // ~1 row per page; fill past the eviction threshold. + int entries = 15_000; + + for (int i = 1; i <= entries; i++) + cache.put(i, new TestObject(PAGE_SIZE / 6)); + + assertTrue("Eviction must have started", isEvictionsStarted(ignite)); + + int probeKey = entries + 1; + + cache.put(probeKey, new TestObject(PAGE_SIZE / 6)); + + assertNotNull(cache.get(probeKey)); + } + + /** + * POJO puts via a {@code DataStreamer} must not throw NPE inside {@code rowSize}. + * + * @throws Exception If failed. + */ + @Test + public void testPojoDataStreamDoesNotNpeInRowSize() throws Exception { + IgniteEx ignite = startGrid(0); + + IgniteCache cache = ignite.createCache( + new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32))); + + int entries = 15_000; + + try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { + streamer.allowOverwrite(true); + + for (int i = 1; i <= entries; i++) + streamer.addData(i, new TestObject(PAGE_SIZE / 6)); + } + + assertTrue("Eviction must have started", isEvictionsStarted(ignite)); + + int probeKey = entries + 1; + + cache.put(probeKey, new TestObject(PAGE_SIZE / 6)); + + assertNotNull(cache.get(probeKey)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java index 86865b56e6137..87a3d32018778 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionSizeAwareAbstractTest.java @@ -35,10 +35,8 @@ public abstract class PageEvictionSizeAwareAbstractTest extends PageEvictionAbst } /** - * A record larger than the whole region must fail (not hang) even when size-aware eviction is enabled. - * A {@code putAll} of records whose total size equals the region capacity must also fail with OOM because - * structural pages (free list, index tree) leave no room for the data. {@code putAll} writes entries individually - * via {@code RowStore.addRow} → {@code FreeList.insertDataRow} (single-row path). + * A record larger than the whole region, and a {@code putAll} whose total size equals the region capacity, must + * both fail with OOM (structural pages leave no room for the data). * * @throws Exception If failed. */ @@ -71,9 +69,7 @@ public void testRecordLargerThanRegionOom() throws Exception { } /** - * A {@code putAll} of several large records (each larger than the empty-pages pool) must be stored successfully - * when page eviction is enabled. Exercises the size-aware reserve in the single-row path - * ({@code RowStore.addRow} → {@code FreeList.insertDataRow}). + * A {@code putAll} of large records (each larger than the empty-pages pool) must succeed with eviction enabled. * * @throws Exception If failed. */ @@ -83,7 +79,7 @@ public void testPutAllLargeRows() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill to ~85% of capacity so that less than one large row remains available. + // Pre-fill to ~85% of capacity so less than one large row remains available. int smallEntries = (int)(totalPages(ignite) * 0.85); byte[] small = new byte[pageSize(ignite) - 200]; @@ -112,9 +108,7 @@ public void testPutAllLargeRows() throws Exception { } /** - * Updating a record from a small to a large value (larger than the empty-pages pool) must succeed with page - * eviction enabled: the update goes through the same size-aware reserve as an insert. The region is pre-filled - * to near capacity so that the grown value cannot fit without evicting pre-filled entries. + * Updating a small record to a large value near capacity must succeed via the size-aware reserve. * * @throws Exception If failed. */ @@ -131,7 +125,6 @@ public void testUpdateRowGrows() throws Exception { for (int i = 0; i < smallEntries; i++) cache.put(smallEntries + i, small); - // Insert key 1 with a small value, then update it to a large value that requires size-aware eviction. cache.put(1, new byte[1024]); byte[] big = new byte[(int)(regionMaxSize(ignite) / 4)]; @@ -149,11 +142,8 @@ public void testUpdateRowGrows() throws Exception { } /** - * Verifies the fast path in {@code ensureFreeSpaceForEviction}: when the region is below the - * eviction threshold, a large row (exceeding the empty-pages pool) that fits in the remaining headroom is written - * successfully without triggering eviction — the size-aware reserve trusts headroom - * and skips the eviction loop. If the fast path were broken, size-aware eviction would run unnecessarily, - * evicting pre-filled entries and setting the evictions-started flag. + * Fast path: below the eviction threshold, a large row that fits in the remaining headroom is written without + * triggering eviction. * * @throws Exception If failed. */ @@ -163,13 +153,8 @@ public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill to ~40% of total pages — well below the eviction threshold, leaving ~50% headroom. - int preFill = (int)(totalPages(ignite) * 0.4); - - byte[] small = new byte[pageSize(ignite) - 200]; - - for (int i = 0; i < preFill; i++) - cache.put(i, small); + // Pre-fill to ~40% of total pages, leaving ~50% headroom. + int preFill = fillRegion(ignite, cache, 0.4); long loadedPages = loadedPages(ignite); long totalPages = totalPages(ignite); @@ -177,8 +162,7 @@ public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception assertTrue("Pre-fill must load ~40% of pages", loadedPages > totalPages * 0.30 && loadedPages < totalPages * 0.50); assertFalse("Eviction must not have started during pre-fill below threshold", isEvictionsStarted(ignite)); - // Write a large row that fits in the remaining headroom. After pre-fill at ~40%, ~60% remains. - // Use ~30% of region so total ~70% stays below the 90% threshold. + // ~30% of the region fits in the remaining headroom (total ~70% < 0.9 threshold). byte[] val = new byte[(int)(regionMaxSize(ignite) * 0.3)]; Arrays.fill(val, (byte)1); @@ -197,9 +181,8 @@ public void testLargeRowBelowThresholdUsesHeadroomNoEviction() throws Exception } /** - * Verifies that a large row (exceeding the empty-pages pool) is written successfully when the region is already - * at or above the eviction threshold — the size-aware eviction loop evicts enough - * pre-filled entries to free the required pages, and to write succeeds. + * Above the eviction threshold, a large row exceeding the headroom + empty-pages pool is written by evicting + * pre-filled entries. * * @throws Exception If failed. */ @@ -209,18 +192,12 @@ public void testLargeRowAboveThresholdEvictsAndSucceeds() throws Exception { IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); - // Pre-fill to ~85% of capacity — above the 0.9 eviction threshold (with page overhead). - int smallEntries = (int)(totalPages(ignite) * 0.85); - - byte[] small = new byte[pageSize(ignite) - 200]; - - for (int i = 0; i < smallEntries; i++) - cache.put(i, small); + // Pre-fill to ~85% of capacity (above the 0.9 threshold with page overhead). + int smallEntries = fillRegion(ignite, cache, 0.85); assertTrue("Pre-fill must load >80% of pages", loadedPages(ignite) > totalPages(ignite) * 0.80); - // Write a large row that exceeds the combined headroom + empty-pages pool, so the size-aware - // reserve must actually evict pre-filled entries to make room. + // Large row exceeding headroom + empty-pages pool → reserve must evict to make room. byte[] val = new byte[(int)(regionMaxSize(ignite) / 4)]; Arrays.fill(val, (byte)1); @@ -238,31 +215,33 @@ public void testLargeRowAboveThresholdEvictsAndSucceeds() throws Exception { } /** - * Verifies that the data region can be filled beyond the eviction threshold (default 0.9) — at least to - * 95% of total pages — without triggering eviction. + * Threshold-gate semantics: once loaded pages cross the 0.9 threshold, the gate evicts on every put, so the region + * caps near the threshold. Exercises both single-put and {@code putAll} paths. * * @throws Exception If failed. */ @Test - public void testRegionFillsBeyondEvictionThreshold() throws Exception { + public void testRegionEvictsAtThreshold() throws Exception { IgniteEx ignite = startGrid(1); IgniteCache cache = createCache(ignite, DEFAULT_CACHE_NAME); long total = totalPages(ignite); + int cap = (int)total * 2; + byte[] small = new byte[pageSize(ignite) - 200]; - // Put entries until the region is at least 95% full. - for (int i = 0; i < total; i++) { + // Fill until eviction starts. + for (int i = 0; i < cap; i++) { cache.put(i, small); - if (loadedPages(ignite) >= total * 0.95) + if (isEvictionsStarted(ignite)) break; } - assertTrue("Region must fill beyond the 0.9 eviction threshold", loadedPages(ignite) >= total * 0.95); - assertFalse("Eviction must not start when headroom is still available", isEvictionsStarted(ignite)); + assertTrue("Eviction must start once the region crosses the 0.9 threshold", isEvictionsStarted(ignite)); + assertTrue("Region must reach at least ~85% before eviction engages", loadedPages(ignite) >= total * 0.85); stopGrid(1); ignite = startGrid(1); @@ -273,21 +252,20 @@ public void testRegionFillsBeyondEvictionThreshold() throws Exception { int batchSize = 100; - // Put entries in batches until the region is at least 95% full. - for (int base = 0; base < total; base += batchSize) { + // Fill via putAll batches until eviction starts. + for (int base = 0; base < cap && !isEvictionsStarted(ignite); base += batchSize) { Map batch = new HashMap<>(); - for (int i = 0; i < batchSize && base + i < total; i++) + for (int i = 0; i < batchSize; i++) batch.put(base + i, small); cache.putAll(batch); - - if (loadedPages(ignite) >= total * 0.95) - break; } - assertTrue("Region must fill beyond the 0.9 eviction threshold via putAll", loadedPages(ignite) >= total * 0.95); - assertFalse("Eviction must not start when headroom is still available", isEvictionsStarted(ignite)); + assertTrue("Eviction must start via putAll once the region crosses the 0.9 threshold", + isEvictionsStarted(ignite)); + assertTrue("Region must reach at least ~85% before eviction engages via putAll", + loadedPages(ignite) >= total * 0.85); } /** @return Loaded pages. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java index b701cb74dc9ef..37ab38b0afb54 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithExpiryPolicyAbstractTest.java @@ -41,10 +41,8 @@ public abstract class PageEvictionWithExpiryPolicyAbstractTest extends PageEvict } /** - * Concurrent page eviction and TTL cleanup on the same data region must not deadlock. - * Multiple threads insert large records into a plain cache (triggering eviction) while page-sized - * entries in a separate TTL cache (sharing the same data region) expire and are cleaned up - * by the eager-TTL background worker, freeing real pages that the eviction machinery may race with. + * Concurrent page eviction and TTL cleanup on the same data region must not deadlock: large inserts trigger + * eviction while page-sized TTL entries expire and free real pages concurrently. * * @throws Exception If failed. */ @@ -62,7 +60,7 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { assertFalse("Eviction must not have started during pre-fill", isEvictionsStarted(ignite)); - // Page-sized TTL entries so their cleanup frees real pages concurrently with eviction. + // Page-sized TTL entries: their cleanup frees real pages concurrently with eviction. int ttlEntries = 100; byte[] ttlVal = new byte[pageSize(ignite) - 200]; @@ -102,8 +100,7 @@ public void testLargePutWithExpiryNoDeadlock() throws Exception { } /** - * After short-TTL entries expire and free their pages, putting the same amount of data to a non-expiring cache - * must succeed, and the new data must be readable. + * After short-TTL entries expire and free their pages, putting the same data to a non-expiring cache must succeed. * * @throws Exception If failed. */ @@ -114,12 +111,7 @@ public void testTtlFreedSpaceAccountedForByEviction() throws Exception { IgniteCache plainCache = createCache(ignite, "plain-cache"); IgniteCache ttlCache = createCache(ignite, "ttl-cache", TTL, false); - int smallEntries = (int)(totalPages(ignite) * 0.80); - - byte[] small = new byte[pageSize(ignite) - 200]; - - for (int i = 0; i < smallEntries; i++) - plainCache.put(i, small); + int smallEntries = fillRegion(ignite, plainCache, 0.80); assertFalse("Eviction must not start during pre-fill", isEvictionsStarted(ignite)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesAbstractTest.java new file mode 100644 index 0000000000000..0e9d83b3b1056 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesAbstractTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.freelist; + +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; + +/** + * Base for tests verifying that the atomic page reservation of an in-memory free list with enabled page eviction is + * fully released after inserts. + */ +public abstract class FreeListReservedPagesAbstractTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(10 * 1024 * 1024L) + .setMaxSize(20 * 1024 * 1024L) + .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU))); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * @return Pages currently reserved by in-flight inserts on the given free list. + */ + protected static long reservedPages(FreeList freeList) { + return ((AtomicLong)getFieldValue(freeList, "reservedPages")).get(); + } + + /** + * @param ignite Node. + * @param cacheName Cache name. + * @return The free list of the first partition data store of the named cache on the given node. + */ + protected static FreeList freeList(IgniteEx ignite, String cacheName) { + return (FreeList)ignite.context().cache().cache(cacheName).context().offheap() + .cacheDataStores().iterator().next().rowStore().freeList(); + } + + /** + * Asserts that {@code reservedPages} is {@code 0} on every data store of every started node, waiting up to + * {@code timeoutMs} for the reservation to drain. + * + * @param cacheName Cache name. + * @param timeoutMs Wait timeout in milliseconds. + * @throws Exception If the reservation does not drain in time. + */ + public static void assertAllReleased(String cacheName, long timeoutMs) throws Exception { + for (Ignite ignite : Ignition.allGrids()) { + for (CacheDataStore store : ((IgniteEx)ignite).context().cache().cache(cacheName).context().offheap().cacheDataStores()) { + FreeList freeList = (FreeList)store.rowStore().freeList(); + + if (reservedPages(freeList) == 0) + continue; + + assertTrue(GridTestUtils.waitForCondition(() -> reservedPages(freeList) == 0, timeoutMs)); + } + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java index a3262b0297244..08845e27713b1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListReservedPagesSelfTest.java @@ -19,15 +19,14 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.concurrent.atomic.AtomicLong; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataPageEvictionMode; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; import org.apache.ignite.internal.processors.cache.CacheObject; @@ -36,40 +35,17 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.tree.DataRow; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; /** - * Tests that the atomic page reservation of an in-memory free list with enabled page eviction is fully released - * after batch inserts: covers the whole-page fragment path, rows consumed by the page fill-up handler, rows left - * unfinished by the fill-up handler and rows sized as an exact multiple of the whole-page fragment size. + * Tests that the atomic page reservation of an in-memory free list with enabled page eviction is fully released after + * batch inserts: whole-page fragments, fill-up-handler rows, whole-page-aligned rows, concurrent batch inserts, and + * the end-to-end backup entry path. */ -public class FreeListReservedPagesSelfTest extends GridCommonAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - DataStorageConfiguration dsCfg = new DataStorageConfiguration(); - - dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setInitialSize(20 * 1024 * 1024L / 2) - .setMaxSize(20 * 1024 * 1024L) - .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU)); - - cfg.setDataStorageConfiguration(dsCfg); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); - - super.afterTest(); - } - +public class FreeListReservedPagesSelfTest extends FreeListReservedPagesAbstractTest { /** */ @Test public void testReservedPagesReleasedAfterSmallRowsBatch() throws Exception { @@ -89,12 +65,11 @@ public void testReservedPagesReleasedAfterWholePageAlignedRowsBatch() throws Exc createCache(ignite); - FreeList freeList = freeList(ignite); + FreeList freeList = freeList(ignite, DEFAULT_CACHE_NAME); int minPageSize = getFieldValue(freeList, "MIN_SIZE_FOR_DATA_PAGE"); - // Probe the size fixed part (key bytes, version and the link slot) to size the value so that the whole row - // is an exact multiple of the whole-page fragment size. + // Size the value so the whole row is an exact multiple of the whole-page fragment size. int fixedSize = row(cacheContext(ignite), 0, 0).size(); insertBatches(ignite, freeList, 2 * minPageSize - fixedSize); @@ -102,13 +77,136 @@ public void testReservedPagesReleasedAfterWholePageAlignedRowsBatch() throws Exc assertEquals(0, reservedPages(freeList)); } + /** */ + @Test + public void testReservedPagesReleasedAfterConcurrentBatchInserts() throws Exception { + IgniteEx ignite = startGrid(0); + + createCache(ignite); + + final FreeList freeList = freeList(ignite, DEFAULT_CACHE_NAME); + + final GridCacheContext cctx = cacheContext(ignite); + + int rowsPerThread = 200; + + final AtomicInteger keyGen = new AtomicInteger(); + + GridTestUtils.runMultiThreaded(() -> { + try { + Collection rows = new ArrayList<>(rowsPerThread); + + for (int i = 0; i < rowsPerThread; i++) + rows.add(row(cctx, keyGen.incrementAndGet(), 100)); + + freeList.insertDataRows(rows, IoStatisticsHolderNoOp.INSTANCE); + } + catch (IgniteCheckedException e) { + throw new RuntimeException(e); + } + }, 8, "concurrent-batch-insert"); + + assertTrue(GridTestUtils.waitForCondition(() -> reservedPages(freeList) == 0, 10_000)); + } + + /** + * End-to-end entry path: a 2-node cluster with {@code backups=1} so the backup node reserves pages before the + * entry lock. After puts and quiescence, {@code reservedPages} must be 0 on both nodes. + * + * @throws Exception If failed. + */ + @Test + public void testReservedPagesReleasedAfterEntryPathPutsOnBackup() throws Exception { + startGrids(2); + + IgniteCache cache = grid(0).createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)); + + awaitPartitionMapExchange(); + + int keyCnt = 500; + int payloadSize = 200; + + for (int i = 0; i < keyCnt; i++) + cache.put(i, new byte[payloadSize]); + + assertAllReleased(DEFAULT_CACHE_NAME, 10_000); + } + + /** + * Concurrent single-row puts on a 2-node cluster with backups; {@code reservedPages} must be 0 after quiescence. + * + * @throws Exception If failed. + */ + @Test + public void testReservedPagesReleasedAfterConcurrentPuts() throws Exception { + startGrids(2); + + IgniteCache cache = grid(0).createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)); + + awaitPartitionMapExchange(); + + AtomicInteger keyGen = new AtomicInteger(); + + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < 300; i++) { + int key = keyGen.incrementAndGet(); + + cache.put(key, new byte[500]); + } + }, 8, "concurrent-put"); + + assertAllReleased(DEFAULT_CACHE_NAME, 10_000); + } + + /** + * Concurrent {@code putAll} batches on a 2-node cluster with backups; {@code reservedPages} must be 0 after + * quiescence. + * + * @throws Exception If failed. + */ + @Test + public void testReservedPagesReleasedAfterConcurrentPutAll() throws Exception { + startGrids(2); + + IgniteCache cache = grid(0).createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)); + + awaitPartitionMapExchange(); + + AtomicInteger keyGen = new AtomicInteger(); + + GridTestUtils.runMultiThreaded(() -> { + for (int b = 0; b < 10; b++) { + Map batch = new HashMap<>(); + + for (int i = 0; i < 100; i++) { + int key = keyGen.incrementAndGet(); + + batch.put(key, new byte[300]); + } + + cache.putAll(batch); + } + }, 4, "concurrent-putAll"); + + assertAllReleased(DEFAULT_CACHE_NAME, 10_000); + } + /** */ private void assertReservedPagesReleasedAfterBatch(int payloadSize) throws Exception { IgniteEx ignite = startGrid(0); createCache(ignite); - insertBatches(ignite, freeList(ignite), payloadSize); + insertBatches(ignite, freeList(ignite, DEFAULT_CACHE_NAME), payloadSize); } /** */ @@ -129,33 +227,12 @@ private static void insertBatches(IgniteEx ignite, FreeList freeLi } } - /** - * @param freeList Free list. - * @return Pages currently reserved by in-flight inserts. - */ - private static long reservedPages(FreeList freeList) { - return ((AtomicLong)getFieldValue(freeList, "reservedPages")).get(); - } - - /** - * @param ignite Ignite. - * @return Free list of the first partition data store. - */ - private static FreeList freeList(IgniteEx ignite) { - return (FreeList)cacheContext(ignite).offheap().cacheDataStores().iterator().next().rowStore().freeList(); - } - - /** - * @param ignite Ignite. - * @return Context of the default cache. - */ + /** @return Context of the default cache. */ private static GridCacheContext cacheContext(IgniteEx ignite) { return ignite.context().cache().cache(DEFAULT_CACHE_NAME).context(); } - /** - * @param ignite Ignite. - */ + /** */ private static void createCache(IgniteEx ignite) { IgniteCache cache = ignite.createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) .setAffinity(new RendezvousAffinityFunction(false, 32)).setBackups(0)); @@ -163,12 +240,7 @@ private static void createCache(IgniteEx ignite) { cache.put(0, 0); } - /** - * @param cctx Cache context. - * @param key Key. - * @param payloadSize Value size. - * @return Data row with the given key and a byte array value of the given size. - */ + /** @return Data row with the given key and a byte array value of the given size. */ private static CacheDataRow row(GridCacheContext cctx, int key, int payloadSize) throws IgniteCheckedException { KeyCacheObject keyObj = cctx.toCacheKeyObject(key); CacheObject valObj = cctx.toCacheObject(new byte[payloadSize]); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index a6f7f3a631206..dc325d3e9865b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -38,10 +38,14 @@ import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionDisabledEvictionReadThroughOomSelfTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionEnsureFreeSpaceThresholdTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionGuardOomTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMetricTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionPagesRecyclingAndReusingTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughReservedPagesSelfTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionRowSizeBinaryObjectTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruNearEnabledPageEvictionMultinodeTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionConcurrentWritesTest; @@ -127,6 +131,14 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, FreeListReservedPagesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionReadThroughReservedPagesSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, PageEvictionRowSizeBinaryObjectTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, PageEvictionEnsureFreeSpaceThresholdTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, PageEvictionDisabledEvictionReadThroughOomSelfTest.class, ignoredTests); + return suite; } }