From 8bf92cb178c3dda48fff7cc162d91a9a74b5e4f2 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 9 Oct 2026 18:57:15 +0300 Subject: [PATCH 1/5] IGNITE-29127 Cache locks usage may lead to node failure with assertion error during stopping --- .../processors/cache/GridCacheAdapter.java | 4 +- .../cache/GridCacheExplicitLockSpan.java | 7 +- .../colocated/GridDhtColocatedLockFuture.java | 8 +- .../dht/ExplicitLockCancelOnNodeStopTest.java | 126 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite14.java | 2 + 5 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 6b32f9408fb02..74285ab5cc957 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -2991,7 +2991,9 @@ public CacheMetricsImpl metrics0() { e = ex; } - throw new NodeStoppingException(e); + throw e == null + ? new NodeStoppingException("Failed to acquire lock (node is stopping) [keys=" + keys + ']') + : new NodeStoppingException(e); } finally { if (isInterrupted) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java index 53c8ad6f5774d..d7dc968ae2b78 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java @@ -234,6 +234,10 @@ public Collection candidates() { /** * Marks all candidates added for given key as owned. + *

+ * Candidates may have been already removed from the span if the lock future was cancelled + * (e.g. on node stop or client disconnect) while the lock acquisition was still in progress. + * In this case there is nothing to mark and the call is a no-op. * * @param key Key. */ @@ -243,7 +247,8 @@ public void markOwned(IgniteTxKey key) { try { Deque deque = cands.get(key); - assert deque != null; + if (deque == null) + return; for (GridCacheMvccCandidate cand : deque) cand.setOwner(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index 3b7158b4e5dd6..ede00c8d9724a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -1295,8 +1295,14 @@ private void lockLocally( ", mappedKeys=" + keys + ", fut=" + this + ']'); try { - if (timeoutObj == null) + if (timeoutObj == null) { + // Future may be already completed (e.g. cancelled on node stop) while + // local DHT lock acquisition was still in progress. + if (isDone()) + return false; + markLocalDhtLocksAcquired(keys); + } else { synchronized (timeoutObj) { if (isDone()) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java new file mode 100644 index 0000000000000..00c3b1b5a9360 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureHandler; +import org.apache.ignite.failure.TestFailureHandler; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.util.lang.RunnableX; +import org.apache.ignite.lifecycle.LifecycleEventType; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; +import static org.apache.ignite.internal.util.typedef.X.hasCause; + +/** + * Tests that an explicit lock request cancelled on a stopping node does not cause a critical failure + * when the response for the in-flight local DHT lock arrives after the cancellation. + */ +public class ExplicitLockCancelOnNodeStopTest extends GridCommonAbstractTest { + /** */ + private volatile RunnableX beforeStop; + + /** */ + private final TestFailureHandler failureHnd = new TestFailureHandler(false); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(TRANSACTIONAL) + .setBackups(1)) + .setLifecycleBeans(evt -> { + if (evt == LifecycleEventType.BEFORE_NODE_STOP && getTestIgniteInstanceName(0).equals(igniteInstanceName)) + beforeStop.run(); + }); + } + + /** {@inheritDoc} */ + @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { + return failureHnd; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * Scenario: + *

    + *
  1. A thread holds an explicit lock on one key so its explicit lock span stays non-empty.
    2. + *
  2. The node starts stopping and, from the {@code BEFORE_NODE_STOP} callback, the same thread tries to + * lock another key. The local DHT lock is acquired and a lock request is sent to the backup.
    3. + *
  3. Since the node is already stopping, the lock future is cancelled immediately which removes + * the explicit lock candidate from the span.
    4. + *
  4. The backup response arrives afterwards and the completed DHT lock future tries to mark + * the already removed candidate as owned.
    5. + *
+ */ + @Test + public void testLockOnStoppingNode() throws Exception { + Ignite srv = startGrid(0); + Ignite backup = startGrid(1); + + awaitPartitionMapExchange(); + + IgniteCache cache = srv.cache(DEFAULT_CACHE_NAME); + + List keys = primaryKeys(cache, 2); + + cache.lock(keys.get(0)).lock(); + + spi(backup).blockMessages(GridDhtLockResponse.class, srv.name()); + + AtomicReference lockErr = new AtomicReference<>(); + + beforeStop = () -> { + try { + cache.lock(keys.get(1)).tryLock(); + } + catch (Throwable e) { + lockErr.set(e); + } + + spi(backup).waitForBlocked(); + spi(backup).stopBlock(); + }; + + stopGrid(0); + + assertTrue("Lock on stopping node must fail: " + lockErr.get(), + hasCause(lockErr.get(), NodeStoppingException.class)); + + FailureContext failureCtx = failureHnd.failureContext(); + + assertNull("Unexpected critical failure: " + failureCtx, failureCtx); + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java index 2f8fc5a17dc71..21e88dbbe1130 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java @@ -48,6 +48,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClientNodePartitionsExchangeTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheServerNodeConcurrentStart; import org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockCancelOnNodeStopTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedDebugTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPreloadRestartSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPrimarySyncSelfTest; @@ -125,6 +126,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedMultiNodeSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedExplicitLockNodeFailureSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheLockReleaseNodeLeaveTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, ExplicitLockCancelOnNodeStopTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNestedTxTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxConcurrentGetTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxReadTest.class, ignoredTests); From c74fb37d17106a59bbff3d3491288522654bbfdd Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 9 Oct 2026 19:41:29 +0300 Subject: [PATCH 2/5] IGNITE-29127 Cache locks usage may lead to node failure with assertion error during stopping --- .../dht/colocated/GridDhtColocatedLockFuture.java | 8 +------- .../dht/ExplicitLockCancelOnNodeStopTest.java | 11 +++-------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index ede00c8d9724a..3b7158b4e5dd6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -1295,14 +1295,8 @@ private void lockLocally( ", mappedKeys=" + keys + ", fut=" + this + ']'); try { - if (timeoutObj == null) { - // Future may be already completed (e.g. cancelled on node stop) while - // local DHT lock acquisition was still in progress. - if (isDone()) - return false; - + if (timeoutObj == null) markLocalDhtLocksAcquired(keys); - } else { synchronized (timeoutObj) { if (isDone()) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java index 00c3b1b5a9360..077df257f4612 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java @@ -23,19 +23,18 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureHandler; import org.apache.ignite.failure.TestFailureHandler; import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.util.lang.RunnableX; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.lifecycle.LifecycleEventType; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; -import static org.apache.ignite.internal.util.typedef.X.hasCause; /** * Tests that an explicit lock request cancelled on a stopping node does not cause a critical failure @@ -116,11 +115,7 @@ public void testLockOnStoppingNode() throws Exception { stopGrid(0); - assertTrue("Lock on stopping node must fail: " + lockErr.get(), - hasCause(lockErr.get(), NodeStoppingException.class)); - - FailureContext failureCtx = failureHnd.failureContext(); - - assertNull("Unexpected critical failure: " + failureCtx, failureCtx); + assertTrue("Lock on stopping node must fail", X.hasCause(lockErr.get(), NodeStoppingException.class)); + assertNull("No failures", failureHnd.failureContext()); } } \ No newline at end of file From 21f0ed40996841c7f02b76f08ec6716f78564865 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 9 Oct 2026 19:43:18 +0300 Subject: [PATCH 3/5] IGNITE-29127 Cache locks usage may lead to node failure with assertion error during stopping --- .../internal/processors/cache/GridCacheExplicitLockSpan.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java index d7dc968ae2b78..dc49befe62aa5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java @@ -234,10 +234,6 @@ public Collection candidates() { /** * Marks all candidates added for given key as owned. - *

- * Candidates may have been already removed from the span if the lock future was cancelled - * (e.g. on node stop or client disconnect) while the lock acquisition was still in progress. - * In this case there is nothing to mark and the call is a no-op. * * @param key Key. */ @@ -247,6 +243,7 @@ public void markOwned(IgniteTxKey key) { try { Deque deque = cands.get(key); + // Candidates may have been already removed from the span if the lock future was cancelled on node stop or client disconnect. if (deque == null) return; From 38c5cca0b4febad6ef1226c61e3f58ec5f83cd7c Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 9 Oct 2026 20:17:23 +0300 Subject: [PATCH 4/5] IGNITE-29127 Cache locks usage may lead to node failure with assertion error during stopping --- .../cache/GridCacheExplicitLockSpan.java | 8 +++++--- .../processors/cache/GridCacheMvccManager.java | 6 +++--- .../colocated/GridDhtColocatedLockFuture.java | 16 ++++++++++++++-- .../dht/ExplicitLockCancelOnNodeStopTest.java | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java index dc49befe62aa5..3d4bff80a533b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java @@ -236,19 +236,21 @@ public Collection candidates() { * Marks all candidates added for given key as owned. * * @param key Key. + * @return {@code True} if candidate owned, {@code false} otherwise. */ - public void markOwned(IgniteTxKey key) { + public boolean markOwned(IgniteTxKey key) { lock(); try { Deque deque = cands.get(key); - // Candidates may have been already removed from the span if the lock future was cancelled on node stop or client disconnect. if (deque == null) - return; + return false; for (GridCacheMvccCandidate cand : deque) cand.setOwner(); + + return true; } finally { unlock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java index 9fe3462af0546..90f19c05463b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java @@ -1031,14 +1031,14 @@ public boolean isLockedByThread(IgniteTxKey key, long threadId) { * * @param key Key. * @param threadId Thread id. + * @return {@code False} if there is no explicit lock candidate for the given thread and key. */ - public void markExplicitOwner(IgniteTxKey key, long threadId) { + public boolean markExplicitOwner(IgniteTxKey key, long threadId) { assert threadId > 0; GridCacheExplicitLockSpan span = pendingExplicit.get(threadId); - if (span != null) - span.markOwned(key); + return span != null && span.markOwned(key); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index 3b7158b4e5dd6..4229376f756aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -1329,10 +1329,22 @@ private void markLocalDhtLocksAcquired(Collection keys) { } else { for (KeyCacheObject key : keys) - cctx.mvcc().markExplicitOwner(cctx.txKey(key), threadId); + markExplicitOwner(key); } } + /** + * Marks explicit lock candidate for the given key as owned. + * + * @param key Locked key. + */ + private void markExplicitOwner(KeyCacheObject key) { + boolean marked = cctx.mvcc().markExplicitOwner(cctx.txKey(key), threadId); + + assert marked || isDone() : "Explicit lock candidate not found for active lock future [key=" + key + + ", fut=" + this + ']'; + } + /** * Tries to map this future in assumption that local node is primary for all keys passed in. * If node is not primary for one of the keys, then mapping is reverted and full remote mapping is performed. @@ -1775,7 +1787,7 @@ void onResult(GridNearLockResponse res) { log.debug("Processed response for entry [res=" + res + ", entry=" + entry + ']'); } else - cctx.mvcc().markExplicitOwner(cctx.txKey(k), threadId); + markExplicitOwner(k); if (retval && cctx.events().isRecordable(EVT_CACHE_OBJECT_READ)) { cctx.events().addEvent(cctx.affinity().partition(k), diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java index 077df257f4612..7caf82227057d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockCancelOnNodeStopTest.java @@ -118,4 +118,4 @@ public void testLockOnStoppingNode() throws Exception { assertTrue("Lock on stopping node must fail", X.hasCause(lockErr.get(), NodeStoppingException.class)); assertNull("No failures", failureHnd.failureContext()); } -} \ No newline at end of file +} From f2489f72587bd1a0947d14cf38f0a9a271e7a306 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 9 Oct 2026 20:18:08 +0300 Subject: [PATCH 5/5] IGNITE-29127 Cache locks usage may lead to node failure with assertion error during stopping --- .../dht/colocated/GridDhtColocatedLockFuture.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index 4229376f756aa..9d4e07a3070fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -1333,11 +1333,7 @@ private void markLocalDhtLocksAcquired(Collection keys) { } } - /** - * Marks explicit lock candidate for the given key as owned. - * - * @param key Locked key. - */ + /** */ private void markExplicitOwner(KeyCacheObject key) { boolean marked = cctx.mvcc().markExplicitOwner(cctx.txKey(key), threadId);