From 095124b985e126205530e36690fc397e85255dcc Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Mon, 2 Nov 2026 11:24:16 +0100 Subject: [PATCH] [IT] Scale the DataNode heap with the CPU cores in IoTDBMultiTAGsWithAttributesTableIT With the default AUTO extension policy, the number of DataRegionGroups follows the CPU cores of the DataNodes (0.5 per core): 2 on a 4-core CI runner, 6 on a 12-core machine. The number of fragment instances of fourTableJoinTest grows with it, and many of them reserve about 1 MB of operator memory because of the 1 MB sort buffer the test sets. The DataNode heap doesn't grow with it. With 6 regions, the 25 fragment instances don't fit into the about 16 MB of operator memory of a 256 MB heap, and the test fails with "There is not enough memory to execute current fragment instance". Give the test DataNodes 128 MB of heap per expected DataRegion, but never less than the configured heap. On a 4-core machine this is the default 256 MB, so CI runs with the same settings as before. --- .../IoTDBMultiTAGsWithAttributesTableIT.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java index 1b293f75865c3..3f59c88c568f6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -21,6 +21,8 @@ import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.ClusterConstant; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -53,6 +55,11 @@ @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBMultiTAGsWithAttributesTableIT { private static final String DATABASE_NAME = "db"; + // Default heap of the test DataNodes (see DataNodeWrapper#initVMConfig). + private static final int DEFAULT_DATANODE_MAX_HEAP_SIZE_IN_MB = 256; + // fourTableJoinTest needs about 3.5 MB of operator memory per DataRegion. A 256 MB heap has + // about 16 MB of it, enough for about 4 regions; 128 MB of heap per region leaves a 2x margin. + private static final int DATANODE_HEAP_SIZE_PER_DATA_REGION_IN_MB = 128; private static final String[] sql1 = new String[] { @@ -185,6 +192,22 @@ public class IoTDBMultiTAGsWithAttributesTableIT { @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L); + // The number of DataRegionGroups follows the CPU cores (0.5 per core), and with it the number + // of FragmentInstances of fourTableJoinTest, each reserving up to the 1 MB sort buffer. Scale + // the DataNode heap, and so its operator memory, with the regions. On a 4-core machine + // (2 regions) this keeps the default heap. + final int expectedDataRegionsPerDataNode = + Math.max(1, Runtime.getRuntime().availableProcessors() / 2); + EnvFactory.getEnv() + .getConfig() + .getDataNodeJVMConfig() + .setMaxHeapSize( + Math.max( + EnvUtils.getIntFromSysVar( + ClusterConstant.DATANODE_MAX_HEAP_SIZE, + DEFAULT_DATANODE_MAX_HEAP_SIZE_IN_MB, + 0), + expectedDataRegionsPerDataNode * DATANODE_HEAP_SIZE_PER_DATA_REGION_IN_MB)); EnvFactory.getEnv().initClusterEnvironment(); EnvFactory.getEnv() .getConfig()