diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml
index 51f4046ae8880..2fb7ca46dc295 100644
--- a/.github/workflows/pipe-it.yml
+++ b/.github/workflows/pipe-it.yml
@@ -30,6 +30,11 @@ on:
- "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine
# allow manually run the action:
workflow_dispatch:
+ inputs:
+ pipe_p0_only:
+ description: Run only the six Pipe P0 recovery integration tests
+ type: boolean
+ default: false
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -41,7 +46,74 @@ env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
jobs:
+ pipe-p0:
+ if: github.event_name == 'workflow_dispatch' && inputs.pipe_p0_only
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ permissions:
+ contents: read
+ env:
+ MAVEN_OPTS: -Xmx2g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+ steps:
+ - uses: actions/checkout@v5
+ - name: Set up JDK 17
+ uses: actions/setup-java@v5
+ with:
+ distribution: corretto
+ java-version: '17'
+ cache: maven
+ - name: Run Pipe P0 recovery tests
+ shell: bash
+ run: |
+ mvn -B -ntp -T 2 clean verify \
+ -Pwith-integration-tests -PMultiClusterIT2DualTreeAutoEnhanced \
+ -pl integration-test -am \
+ -DskipUTs -Dtest=NoSuchSurefireTest \
+ -Dsurefire.failIfNoSpecifiedTests=false \
+ -Dit.test=IoTDBPipeRecoveryIT \
+ -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false \
+ -DintegrationTest.clusterReadyRetryCount=90 \
+ -DintegrationTest.forkCount=1 \
+ -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 \
+ -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=HighPerformanceMode,HighPerformanceMode
+ - name: Verify all six tests ran successfully
+ if: always()
+ shell: bash
+ run: |
+ python3 - <<'PY'
+ from pathlib import Path
+ import xml.etree.ElementTree as ET
+
+ report = Path('integration-test/target/failsafe-reports/TEST-org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced.IoTDBPipeRecoveryIT.xml')
+ expected = {
+ 'testAllReceiverDataNodesDownRetainAndDrainBacklogWithoutNewWrites',
+ 'testHistoryRealtimeHandoffSurvivesReceiverRestart',
+ 'testMultipleDataRegionsAndWritersDrainAfterLeaderTransfer',
+ 'testRequestNotDeliveredRetriesWithoutStarvingAnotherPipe',
+ 'testCommittedWriteWithLostResponseRetriesIdempotently',
+ 'testRuntimeQueueBackpressureDrainsAndReleasesMemory',
+ }
+ cases = list(ET.parse(report).getroot().iter('testcase'))
+ actual = {case.attrib['name'] for case in cases}
+ if len(cases) != 6 or actual != expected:
+ raise SystemExit(f'Expected all six P0 cases; got {sorted(actual)}')
+ for case in cases:
+ if any(child.tag in {'failure', 'error', 'skipped'} for child in case):
+ raise SystemExit(f"P0 case did not pass: {case.attrib['name']}")
+ print('All six Pipe P0 recovery integration tests passed')
+ PY
+ - name: Upload Pipe P0 reports and cluster logs
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: pipe-p0-recovery-reports
+ path: |
+ integration-test/target/failsafe-reports
+ integration-test/target/cluster-logs
+ retention-days: 14
single:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -131,6 +203,7 @@ jobs:
# 12 IT classes split across 3 parallel shards to cut the historical ~42 min
# wall clock to ~14 min. See cluster-it-1c1d.yml for the shard pattern.
dual-tree-auto-basic:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -226,6 +299,7 @@ jobs:
# 9 IT classes split across 3 parallel shards to cut the historical ~51 min
# wall clock to ~17 min. See cluster-it-1c1d.yml for the shard pattern.
dual-tree-auto-enhanced:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -322,6 +396,7 @@ jobs:
# 11 IT classes split across 3 parallel shards to cut the historical ~27 min
# wall clock to ~9 min. See cluster-it-1c1d.yml for the shard pattern.
dual-tree-manual:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -416,6 +491,7 @@ jobs:
path: integration-test/target/cluster-logs
retention-days: 30
dual-table-manual-basic:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -511,6 +587,7 @@ jobs:
# 11 IT classes split across 3 parallel shards to cut the historical ~62 min
# wall clock to ~22 min. See cluster-it-1c1d.yml for the shard pattern.
dual-table-manual-enhanced:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 15
@@ -604,6 +681,7 @@ jobs:
path: integration-test/target/cluster-logs
retention-days: 30
triple:
+ if: github.event_name != 'workflow_dispatch' || !inputs.pipe_p0_only
strategy:
fail-fast: false
max-parallel: 1
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 05b8ce8b1d13c..18300e08b7ac5 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -92,6 +92,12 @@
3
+
+ org.javassist
+ javassist
+ 3.30.2-GA
+ test
+
org.apache.iotdb
iotdb-server
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
index d9c51717cf8e1..7304e3024cd14 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
@@ -23,12 +23,26 @@
import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class MppJVMConfig implements JVMConfig {
private int initHeapSize;
private int maxHeapSize;
private int maxDirectMemorySize;
private String timezone;
+ private List jvmOptions = Collections.emptyList();
+
+ /** Additional arguments for child test-node JVMs, for example a test-only Java agent. */
+ public void setJvmOptions(List options) {
+ this.jvmOptions = new ArrayList<>(options);
+ }
+
+ public List getJvmOptions() {
+ return Collections.unmodifiableList(jvmOptions);
+ }
@Override
public JVMConfig setInitHeapSize(int initSize) {
@@ -94,6 +108,7 @@ public void override(@Nullable MppJVMConfig config) {
this.setMaxHeapSize(config.getMaxHeapSize());
this.setMaxDirectMemorySize(config.getMaxDirectMemorySize());
this.setTimezone(config.getTimezone());
+ this.setJvmOptions(config.getJvmOptions());
}
public static Builder builder() {
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index 8d3f5f2cc9d9e..a5bc823eafda2 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -483,6 +483,7 @@ public void start() {
File stdoutFile = new File(getLogPath());
List startCmd = new ArrayList<>();
startCmd.add(JAVA_CMD);
+ startCmd.addAll(jvmConfig.getJvmOptions());
if (!SystemUtils.IS_JAVA_1_8) {
startCmd.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED");
startCmd.add("--add-opens=java.base/java.lang=ALL-UNNAMED");
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBMetadataCatalogSnapshotRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBMetadataCatalogSnapshotRecoveryIT.java
new file mode 100644
index 0000000000000..23f7e56d16026
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBMetadataCatalogSnapshotRecoveryIT.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.consensus.ConfigRegionId;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
+import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.consensus.ratis.utils.Utils;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.client.RaftClientConfigKeys;
+import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.protocol.exceptions.RaftRetryFailureException;
+import org.apache.ratis.retry.RetryPolicies;
+import org.apache.ratis.util.TimeDuration;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.assertRows;
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.connection;
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.execute;
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.property;
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.schemaRegion;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+/** P1-8: restore an in-use catalog snapshot and its later DDL, beyond partition-only snapshots. */
+@RunWith(IoTDBTestRunner.class)
+@Category(ClusterIT.class)
+public class IoTDBMetadataCatalogSnapshotRecoveryIT {
+ private static final String DATABASE = "root.p1_catalog";
+ private static final String TABLE_DATABASE = "p1_catalog_table";
+
+ @Before
+ public void before() {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaEngineMode("Memory")
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaReplicationFactor(3)
+ .setDataReplicationFactor(3)
+ .setConfigNodeRatisSnapshotTriggerThreshold(Integer.MAX_VALUE);
+ property(
+ EnvFactory.getEnv().getConfig().getConfigNodeCommonConfig(),
+ "config_node_ratis_preserve_logs_num_when_purge",
+ "10000");
+ EnvFactory.getEnv().initClusterEnvironment(3, 3);
+ }
+
+ @After
+ public void after() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testCatalogSnapshotAndSubsequentDdlSurviveLeaderRecovery() throws Exception {
+ long time = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1);
+ int oldLeaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ ConfigNodeWrapper recovering = EnvFactory.getEnv().getConfigNodeWrapper(oldLeaderIndex);
+ try (Connection tree =
+ connection(EnvFactory.getEnv().getDataNodeWrapper(0), BaseEnv.TREE_SQL_DIALECT);
+ Connection table =
+ connection(EnvFactory.getEnv().getDataNodeWrapper(0), BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(
+ tree,
+ "CREATE DATABASE " + DATABASE,
+ "CREATE DEVICE TEMPLATE p1_catalog_template (s INT64 ENCODING=PLAIN COMPRESSION=LZ4)",
+ "SET DEVICE TEMPLATE p1_catalog_template TO " + DATABASE + ".templ",
+ "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + DATABASE + ".templ.d",
+ "INSERT INTO " + DATABASE + ".templ.d(time,s) VALUES(" + time + ",10)",
+ "CREATE TIMESERIES " + DATABASE + ".source.d.s INT32",
+ "INSERT INTO " + DATABASE + ".source.d(time,s) VALUES(" + time + ",11)",
+ "CREATE DEVICE TEMPLATE p1_catalog_deleted (s INT64)",
+ "SET DEVICE TEMPLATE p1_catalog_deleted TO " + DATABASE + ".unused",
+ "SET TTL TO " + DATABASE + " 604800000");
+ execute(
+ table,
+ "CREATE DATABASE " + TABLE_DATABASE,
+ "USE " + TABLE_DATABASE,
+ "CREATE TABLE catalog_table(device STRING TAG,owner STRING ATTRIBUTE,value INT32 FIELD,removed INT32 FIELD) WITH (ttl=604800000)",
+ "INSERT INTO catalog_table(time,device,owner,value,removed) VALUES("
+ + time
+ + ",'d1','before',1,100)",
+ "CREATE TABLE deleted_table(device STRING TAG,value INT32 FIELD)",
+ "CREATE VIEW catalog_view(device STRING TAG) AS " + DATABASE + ".source.**",
+ "CREATE VIEW deleted_view(device STRING TAG) AS " + DATABASE + ".source.**");
+ int partition = schemaRegion(DATABASE, DATABASE + ".templ.d.s");
+ TShowClusterResp cluster;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ cluster = client.showCluster();
+ }
+ RaftPeerId recoveredPeer =
+ Utils.fromNodeIdToRaftPeerId(
+ cluster.getConfigNodeList().stream()
+ .filter(node -> node.getInternalEndPoint().getPort() == recovering.getPort())
+ .findFirst()
+ .orElseThrow(AssertionError::new)
+ .getConfigNodeId());
+ try (RaftClient raft = raftClient(cluster)) {
+ assertTrue(
+ "Snapshot must complete before the following DDL",
+ raft.getSnapshotManagementApi(recoveredPeer).create(true, 30000).isSuccess());
+ recovering.stopForcibly();
+ assertNotEquals(oldLeaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ execute(
+ tree,
+ "ALTER DEVICE TEMPLATE p1_catalog_template ADD(extra INT64 ENCODING=PLAIN COMPRESSION=LZ4)",
+ "INSERT INTO " + DATABASE + ".templ.d(time,extra) VALUES(" + (time + 1) + ",20)",
+ "UNSET DEVICE TEMPLATE p1_catalog_deleted FROM " + DATABASE + ".unused",
+ "DROP DEVICE TEMPLATE p1_catalog_deleted",
+ "SET TTL TO " + DATABASE + " 86400000");
+ execute(
+ table,
+ "ALTER TABLE catalog_table ADD COLUMN after_snapshot DOUBLE FIELD",
+ "ALTER TABLE catalog_table DROP COLUMN removed",
+ "ALTER TABLE catalog_table SET PROPERTIES ttl=86400000",
+ "INSERT INTO catalog_table(time,device,owner,value,after_snapshot) VALUES("
+ + (time + 1)
+ + ",'d2','after',2,2.5)",
+ "ALTER VIEW catalog_view RENAME TO current_view",
+ "DROP VIEW deleted_view",
+ "DROP TABLE deleted_table");
+ SQLException renameFailure =
+ assertThrows(
+ SQLException.class,
+ () ->
+ execute(table, "ALTER TABLE catalog_table RENAME COLUMN owner TO owner_after"));
+ assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), renameFailure.getErrorCode());
+ // The peer is stopped, so clearing its log excludes evidence from any earlier startup.
+ recovering.clearLogContent();
+ recovering.start();
+ String snapshotLoaded =
+ ConfigNodeMessages.CONFIGNODESNAPSHOT_LOAD_SNAPSHOT_SUCCESS_LATESTSNAPSHOTROOTDIR
+ .replace("{}", "");
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .until(() -> recovering.logContains(snapshotLoaded));
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ // Keep the other two peers alive for quorum while this particular recovered peer takes
+ // over. The client may first contact its cached old leader before learning the new one.
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .ignoreException(RaftRetryFailureException.class)
+ .untilAsserted(
+ () ->
+ assertTrue(raft.admin().transferLeadership(recoveredPeer, 10000).isSuccess()));
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> assertEquals(oldLeaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()));
+ }
+ assertEquals(partition, schemaRegion(DATABASE, DATABASE + ".templ.d.s"));
+ for (DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ try (Connection readTree = connection(node, BaseEnv.TREE_SQL_DIALECT);
+ Connection readTable = connection(node, BaseEnv.TABLE_SQL_DIALECT)) {
+ assertRows(readTree, "SHOW DEVICE TEMPLATES", "TemplateName,", "p1_catalog_template,");
+ assertRows(
+ readTree,
+ "SHOW NODES IN DEVICE TEMPLATE p1_catalog_template",
+ "ChildNodes,DataType,Encoding,Compression,",
+ "s,INT64,PLAIN,LZ4,",
+ "extra,INT64,PLAIN,LZ4,");
+ assertRows(
+ readTree,
+ "SHOW PATHS SET DEVICE TEMPLATE p1_catalog_template",
+ "Paths,",
+ DATABASE + ".templ,");
+ assertRows(
+ readTree,
+ "SHOW PATHS USING DEVICE TEMPLATE p1_catalog_template",
+ "Paths,",
+ DATABASE + ".templ.d,");
+ assertRows(
+ readTree, "SHOW TTL ON " + DATABASE, "Device,TTL(ms),", DATABASE + ",86400000,");
+ assertRows(
+ readTree,
+ "SELECT s,extra FROM " + DATABASE + ".templ.d",
+ "Time," + DATABASE + ".templ.d.s," + DATABASE + ".templ.d.extra,",
+ time + ",10,null,",
+ (time + 1) + ",null,20,");
+ execute(readTable, "USE " + TABLE_DATABASE);
+ assertRows(
+ readTable,
+ "DESCRIBE catalog_table",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "device,STRING,TAG,",
+ "owner,STRING,ATTRIBUTE,",
+ "value,INT32,FIELD,",
+ "after_snapshot,DOUBLE,FIELD,");
+ assertRows(
+ readTable,
+ "SHOW TABLES",
+ "TableName,TTL(ms),",
+ "catalog_table,86400000,",
+ "current_view,INF,");
+ assertRows(
+ readTable,
+ "SELECT CAST(time AS INT64) AS ts,device,owner,value,after_snapshot FROM catalog_table",
+ "ts,device,owner,value,after_snapshot,",
+ time + ",d1,before,1,null,",
+ (time + 1) + ",d2,after,2,2.5,");
+ assertRows(
+ readTable,
+ "SELECT CAST(time AS INT64) AS ts,device,s FROM current_view",
+ "ts,device,s,",
+ time + ",d,11,");
+ }
+ }
+ execute(
+ table,
+ "ALTER TABLE catalog_table ADD COLUMN continued BOOLEAN FIELD",
+ "INSERT INTO catalog_table(time,device,continued) VALUES(" + (time + 2) + ",'d1',true)");
+ assertRows(
+ table,
+ "SELECT CAST(time AS INT64) AS ts,device,continued FROM catalog_table WHERE continued IS NOT NULL",
+ "ts,device,continued,",
+ (time + 2) + ",d1,true,");
+ execute(
+ tree,
+ "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + DATABASE + ".templ.fresh",
+ "INSERT INTO " + DATABASE + ".templ.fresh(time,extra) VALUES(" + (time + 2) + ",30)");
+ assertRows(
+ tree,
+ "SELECT extra FROM " + DATABASE + ".templ.fresh",
+ "Time," + DATABASE + ".templ.fresh.extra,",
+ (time + 2) + ",30,");
+ } finally {
+ if (!recovering.isAlive()) {
+ recovering.start();
+ }
+ }
+ }
+
+ private RaftClient raftClient(TShowClusterResp cluster) {
+ RaftProperties properties = new RaftProperties();
+ RaftClientConfigKeys.Rpc.setRequestTimeout(
+ properties, TimeDuration.valueOf(35, TimeUnit.SECONDS));
+ return RaftClient.newBuilder()
+ .setRaftGroup(
+ RaftGroup.valueOf(
+ Utils.fromConsensusGroupIdToRaftGroupId(
+ new ConfigRegionId(
+ ConfigNodeDescriptor.getInstance().getConf().getConfigRegionId())),
+ cluster.getConfigNodeList().stream()
+ .map(
+ node ->
+ Utils.fromNodeInfoAndPriorityToRaftPeer(
+ node.getConfigNodeId(), node.getConsensusEndPoint(), 0))
+ .collect(Collectors.toList())))
+ .setProperties(properties)
+ .setRetryPolicy(RetryPolicies.noRetry())
+ .build();
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java
index 8f8ec7daa64b6..eb6e361589d86 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.it;
+import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -144,6 +145,20 @@ public void testUnsupportedValueFilter() throws SQLException {
}
assertEquals(1, cnt);
}
+ try (ResultSet rows = statement.executeQuery("SELECT s0,s3,s4 FROM root.vehicle1.d0")) {
+ TestUtils.assertResultSetEqual(
+ rows,
+ "Time,root.vehicle1.d0.s0,root.vehicle1.d0.s3,root.vehicle1.d0.s4,",
+ new String[] {"10,310.0,text,true,"});
+ }
+ statement.execute("DELETE FROM root.vehicle1.d0.s0 WHERE time=10");
+ statement.execute("FLUSH");
+ try (ResultSet rows = statement.executeQuery("SELECT s0,s3,s4 FROM root.vehicle1.d0")) {
+ TestUtils.assertResultSetEqual(
+ rows,
+ "Time,root.vehicle1.d0.s0,root.vehicle1.d0.s3,root.vehicle1.d0.s4,",
+ new String[] {"10,null,text,true,"});
+ }
}
}
@@ -513,6 +528,101 @@ public void testDeleteByRangeComparison() throws SQLException {
}
}
+ @Test
+ public void testOverlappingDeleteRangesAtTimePartitionBoundaries() throws SQLException {
+ final String database = "root.delete_partition_boundary";
+ final long[] timestamps = {
+ -1, 0, 1, 998, 999, 1000, 1001, 1002, 1003, 1998, 1999, 2000, 2001, 4102444800000L
+ };
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s1 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s2 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d2.s1 WITH DATATYPE=INT64");
+
+ for (long timestamp : timestamps) {
+ statement.execute(
+ "INSERT INTO "
+ + database
+ + ".d1(time,s1,s2) VALUES("
+ + timestamp
+ + ","
+ + (timestamp + 100)
+ + ","
+ + (timestamp + 10000)
+ + ")");
+ statement.execute(
+ "INSERT INTO "
+ + database
+ + ".d2(time,s1) VALUES("
+ + timestamp
+ + ","
+ + (timestamp + 30000)
+ + ")");
+ }
+ statement.execute("FLUSH");
+
+ // Leave unflushed values on both sides of the boundary as well as the closed-file data.
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(999,9999),(1001,10001)");
+ // The interval is 1000. Overlapping, adjacent and disjoint deletions must preserve the gap
+ // at 1003..1998, the unselected measurement and the other device.
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time >= 999 AND time <= 1000");
+ try (ResultSet rows =
+ statement.executeQuery("SELECT s1 FROM " + database + ".d1 WHERE time=1000")) {
+ assertFalse(rows.next());
+ }
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time >= 1000 AND time <= 1001");
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time = 1002");
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time > 1998 AND time < 2001");
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(1500,15000)");
+ // No writes within this range occur between retries. The disjoint new point must survive.
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time >= 999 AND time <= 1000");
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(1000,900)");
+ statement.execute("FLUSH");
+ }
+
+ TestUtils.resultSetEqualTest(
+ "SELECT s1,s2 FROM " + database + ".d1 ORDER BY time",
+ new String[] {"Time", database + ".d1.s1", database + ".d1.s2"},
+ new String[] {
+ "-1,99,9999,",
+ "0,100,10000,",
+ "1,101,10001,",
+ "998,1098,10998,",
+ "999,null,10999,",
+ "1000,900,11000,",
+ "1001,null,11001,",
+ "1002,null,11002,",
+ "1003,1103,11003,",
+ "1500,15000,null,",
+ "1998,2098,11998,",
+ "1999,null,11999,",
+ "2000,null,12000,",
+ "2001,2101,12001,",
+ "4102444800000,4102444800100,4102444810000,"
+ });
+ TestUtils.resultSetEqualTest(
+ "SELECT s1 FROM " + database + ".d2 ORDER BY time",
+ new String[] {"Time", database + ".d2.s1"},
+ new String[] {
+ "-1,29999,",
+ "0,30000,",
+ "1,30001,",
+ "998,30998,",
+ "999,30999,",
+ "1000,31000,",
+ "1001,31001,",
+ "1002,31002,",
+ "1003,31003,",
+ "1998,31998,",
+ "1999,31999,",
+ "2000,32000,",
+ "2001,32001,",
+ "4102444800000,4102444830000,"
+ });
+ }
+
private static void prepareSeries() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
index 3827615fe24d8..e8fcd550bb4b5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
@@ -19,208 +19,548 @@
package org.apache.iotdb.db.it;
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
-import org.apache.iotdb.it.utils.TsFileGenerator;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.external.commons.io.FileUtils;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import org.slf4j.helpers.MessageFormatter;
+
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
import java.io.File;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
+import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege;
+import static org.apache.iotdb.db.it.utils.TestUtils.revokeUserSeriesPrivilege;
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBLoadTsFileActiveRetryIT {
-
- private static final String DATABASE = "root.sg.test_0";
- private static final String DEVICE = DATABASE + ".d_0";
- private static final String MEASUREMENT = "sensor_00";
- private static final long UNALLOCATABLE_TABLET_CONVERSION_BATCH_MEMORY_SIZE_IN_BYTES =
- Long.MAX_VALUE / 4;
+ private static final String DATABASE = "root.sg.active_retry";
+ private static final String DEVICE = DATABASE + ".d0";
+ private static final String PERMANENT_DEVICE = DATABASE + ".permanent";
+ private static final String MEASUREMENT = "s0";
+ private static final int ROWS_PER_FILE = 12;
+ private static final String USER = "active_retry_user";
+ private static final String IDENTITY_USER = "active_retry_identity";
+ private static final String IDENTITY_FILE_NAME = "identity-restart.tsfile";
+ private static final String PASSWORD = "Retry_Load#2026";
private static final MeasurementSchema TSFILE_SCHEMA =
- new MeasurementSchema(MEASUREMENT, TSDataType.INT32, TSEncoding.RLE);
+ new MeasurementSchema(MEASUREMENT, TSDataType.INT64, TSEncoding.RLE);
private File tmpDir;
- @Before
- public void setUp() throws Exception {
- tmpDir = new File(Files.createTempDirectory("load-active-retry").toUri());
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ configureRetryMetrics();
EnvFactory.getEnv().getConfig().getCommonConfig().setPipeMemoryManagementEnabled(false);
EnvFactory.getEnv()
.getConfig()
.getDataNodeConfig()
.setMaxAllocateMemoryRatioForLoad(1.0)
.setLoadTsFileAnalyzeSchemaMemorySizeInBytes(1)
- .setLoadTsFileTabletConversionBatchMemorySizeInBytes(
- UNALLOCATABLE_TABLET_CONVERSION_BATCH_MEMORY_SIZE_IN_BYTES)
+ .setLoadTsFileTabletConversionBatchMemorySizeInBytes(Long.MAX_VALUE / 4)
.setLoadActiveListeningCheckIntervalSeconds(1);
-
EnvFactory.getEnv().initClusterEnvironment();
+ createUser(USER, PASSWORD);
+ for (final PrivilegeType privilege :
+ Arrays.asList(
+ PrivilegeType.WRITE_DATA,
+ PrivilegeType.READ_DATA,
+ PrivilegeType.READ_SCHEMA,
+ PrivilegeType.WRITE_SCHEMA)) {
+ grantUserSeriesPrivilege(USER, privilege, DATABASE + ".**");
+ }
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ // A filesystem cleanup failure must not leave the remaining test processes running.
+ EnvFactory.getEnv().shutdownForciblyAllDataNodes();
+ EnvFactory.getEnv().shutdownForciblyAllConfigNodes();
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ tmpDir = Files.createTempDirectory("load-active-retry").toFile();
}
@After
public void tearDown() throws Exception {
- try (final Connection connection = EnvFactory.getEnv().getConnection();
- final Statement statement = connection.createStatement()) {
- statement.execute("delete database " + DATABASE);
- } catch (final Exception ignored) {
- // ignore cleanup failure
+ final DataNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
+ if (!node.isAlive()) {
+ node.start();
+ }
+ awaitNode(node);
+ try {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("delete database " + DATABASE);
+ }
} finally {
- EnvFactory.getEnv().cleanClusterEnvironment();
- deleteRecursively(tmpDir);
+ FileUtils.deleteDirectory(activeDir(node));
+ FileUtils.deleteDirectory(failedDir(node));
+ FileUtils.deleteDirectory(tmpDir);
}
}
+ /**
+ * LOAD-P0-06: remove the cause of a witnessed temporary failure and require the original handoffs
+ * to finish without another LOAD. Different mods on same-named files protect pairing; a permanent
+ * failure must not starve recoverable work. The old case only checked retention.
+ */
+ @Test
+ public void testTemporaryFailureRecoversWithoutResubmission() throws Exception {
+ assertRetryCompletes(false);
+ }
+
+ /**
+ * LOAD-P0-06: restart with the original pending paths on disk, then repair the schema. The new
+ * process must rediscover those paths and preserve database-level, verify and on-success. A
+ * separate handoff loses WRITE_DATA after restart and must fail under its stored user, while the
+ * other original handoffs finish. Its disjoint timestamps must never be imported.
+ */
@Test
- public void testActiveLoadTemporaryUnavailableShouldKeepFileForRetry() throws Exception {
- final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0);
- final File retryTsFile = new File(tmpDir, "1-0-0-0.tsfile");
- final File permanentFailureTsFile = new File(tmpDir, "2-0-0-0.tsfile");
- generateTsFile(retryTsFile);
- generateTsFile(permanentFailureTsFile);
+ public void testPendingFilesRecoverAfterDataNodeRestart() throws Exception {
+ assertRetryCompletes(true);
+ }
+
+ private void assertRetryCompletes(final boolean restart) throws Exception {
+ final DataNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
+ final List sourceFiles = new ArrayList<>();
+ for (int fileIndex = 0; fileIndex < 2; fileIndex++) {
+ final File directory = new File(tmpDir, "source-" + fileIndex);
+ Assert.assertTrue(directory.mkdirs());
+ final File file = new File(directory, "1-0-0-0.tsfile");
+ generateTsFile(file, DEVICE, fileIndex);
+ sourceFiles.add(file);
+ }
+ // Class-scoped nodes retain their logs between methods. Give each permanent failure its own
+ // name so an earlier method's log cannot satisfy this method's failure-reason assertion.
+ final File permanentFile =
+ new File(tmpDir, restart ? "permanent-restart.tsfile" : "permanent-live.tsfile");
+ generateTsFile(permanentFile, PERMANENT_DEVICE, 2);
+ final File identitySource = restart ? new File(tmpDir, IDENTITY_FILE_NAME) : null;
+ if (restart) {
+ generateTsFile(identitySource, DEVICE, 3);
+ createUser(IDENTITY_USER, PASSWORD);
+ for (final PrivilegeType privilege :
+ Arrays.asList(
+ PrivilegeType.WRITE_DATA,
+ PrivilegeType.READ_DATA,
+ PrivilegeType.READ_SCHEMA,
+ PrivilegeType.WRITE_SCHEMA)) {
+ grantUserSeriesPrivilege(IDENTITY_USER, privilege, DATABASE + ".**");
+ }
+ }
try (final Connection connection =
- EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
+ EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(node);
final Statement statement = connection.createStatement()) {
statement.execute("create database " + DATABASE);
- statement.execute(
- String.format(
- "create timeseries %s.%s %s", DEVICE, MEASUREMENT, TSDataType.INT64.name()));
-
- statement.execute(
- String.format(
- "load \"%s\" with ('database-level'='3', 'async'='true', 'on-success'='none', "
- + "'convert-on-type-mismatch'='true')",
- retryTsFile.getAbsolutePath()));
- statement.execute(
- String.format(
- "load \"%s\" with ('database-level'='3', 'async'='true', 'on-success'='none', "
- + "'convert-on-type-mismatch'='false')",
- permanentFailureTsFile.getAbsolutePath()));
-
- final File activeDir = getActiveLoadDir(dataNodeWrapper);
- final File failDir = getActiveLoadFailDir(dataNodeWrapper);
- final File activeTsFile = waitForFile(activeDir, retryTsFile.getName(), 30_000L);
-
- Assert.assertNotNull(
- "Async load should copy tsfile into active load directory", activeTsFile);
-
- Assert.assertNotNull(
- "Permanent active load failure should be moved to fail dir",
- waitForFile(failDir, permanentFailureTsFile.getName(), TimeUnit.SECONDS.toMillis(60)));
-
- assertFileKeptForRetry(
- activeDir, failDir, retryTsFile.getName(), TimeUnit.SECONDS.toMillis(12));
- }
- }
-
- private void generateTsFile(final File tsFile) throws Exception {
- try (final TsFileGenerator generator = new TsFileGenerator(tsFile)) {
- generator.registerTimeseries(DEVICE, Collections.singletonList(TSFILE_SCHEMA));
- generator.generateData(DEVICE, 10, 1, false);
- }
- }
-
- private File getActiveLoadDir(final DataNodeWrapper dataNodeWrapper) {
- return new File(
- dataNodeWrapper.getNodePath()
- + File.separator
- + "ext"
- + File.separator
- + "load"
- + File.separator
- + "pending");
- }
-
- private File getActiveLoadFailDir(final DataNodeWrapper dataNodeWrapper) {
- return new File(
- dataNodeWrapper.getNodePath()
- + File.separator
- + "ext"
- + File.separator
- + "load"
- + File.separator
- + "failed");
- }
-
- private File waitForFile(final File root, final String fileName, final long timeoutMs)
- throws InterruptedException {
- final long deadline = System.currentTimeMillis() + timeoutMs;
- while (System.currentTimeMillis() < deadline) {
- final File file = findFile(root, fileName);
- if (file != null) {
- return file;
+ statement.execute("create timeseries " + DEVICE + "." + MEASUREMENT + " INT32");
+ statement.execute("create timeseries " + PERMANENT_DEVICE + "." + MEASUREMENT + " INT32");
+ }
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(node, USER, PASSWORD);
+ final Statement statement = connection.createStatement()) {
+ for (int i = 0; i < sourceFiles.size(); i++) {
+ statement.execute(loadSql(sourceFiles.get(i), i == 0 ? "none" : "delete", true));
+ }
+ statement.execute(loadSql(permanentFile, "none", false));
+ }
+ if (restart) {
+ try (final Connection connection =
+ EnvFactory.getEnv()
+ .getConnectionWithSpecifiedDataNode(node, IDENTITY_USER, PASSWORD);
+ final Statement statement = connection.createStatement()) {
+ statement.execute(loadSql(identitySource, "none", true));
}
- Thread.sleep(500L);
}
- return null;
+
+ final List pendingFiles = new ArrayList<>();
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ pendingFiles.clear();
+ pendingFiles.addAll(findTsFiles(activeDir(node)));
+ Assert.assertEquals(
+ "all temporarily rejected handoffs must remain pending",
+ restart ? 3 : 2,
+ pendingFiles.size());
+ final List failedFiles = findTsFiles(failedDir(node));
+ Assert.assertEquals(1, failedFiles.size());
+ Assert.assertEquals(permanentFile.getName(), failedFiles.get(0).getName());
+ Assert.assertTrue(
+ new File(failedFiles.get(0) + ModificationFile.FILE_SUFFIX).isFile());
+ assertPermanentFailureWasLogged(node, permanentFile.getName());
+ for (final File file : pendingFiles) {
+ Assert.assertTrue(new File(file + ModificationFile.FILE_SUFFIX).isFile());
+ assertTemporaryFailureWasLogged(node, file);
+ assertHandoffAttributes(file, node);
+ }
+ });
+
+ Assert.assertTrue(sourceFiles.get(0).isFile());
+ Assert.assertTrue(new File(sourceFiles.get(0) + ModificationFile.FILE_SUFFIX).isFile());
+ // async on-success=delete transfers ownership at handoff, before background completion.
+ Assert.assertFalse(sourceFiles.get(1).exists());
+ Assert.assertFalse(new File(sourceFiles.get(1) + ModificationFile.FILE_SUFFIX).exists());
+ Assert.assertTrue(permanentFile.isFile());
+ assertRows(DEVICE, Collections.emptyList());
+
+ if (restart) {
+ node.stopForcibly();
+ Assert.assertFalse(node.isAlive());
+ for (final File file : pendingFiles) {
+ Assert.assertTrue("a process crash must preserve the handoff", file.isFile());
+ }
+ node.start();
+ awaitNode(node);
+ for (final File file : pendingFiles) {
+ assertHandoffAttributes(file, node);
+ }
+ // Changing only this user's grant distinguishes restoring the saved identity from running
+ // every recovered handoff as the superuser. The two other handoffs keep their write access.
+ revokeUserSeriesPrivilege(IDENTITY_USER, PrivilegeType.WRITE_DATA, DATABASE + ".**");
+ final File originalIdentityHandoff =
+ pendingFiles.stream()
+ .filter(file -> IDENTITY_FILE_NAME.equals(file.getName()))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ // Permission checks precede schema verification. A retry that passed authorization before
+ // REVOKE must finish against the conflicting schema before we widen it; otherwise it may
+ // legitimately import the identity file and make this test depend on thread scheduling.
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertPermissionFailureWasLogged(node, originalIdentityHandoff);
+ Assert.assertFalse(originalIdentityHandoff.exists());
+ final File failedIdentity = new File(failedDir(node), IDENTITY_FILE_NAME);
+ Assert.assertTrue(failedIdentity.isFile());
+ Assert.assertTrue(new File(failedIdentity + ModificationFile.FILE_SUFFIX).isFile());
+ });
+ }
+
+ // Widen the empty schema in place so it matches the pending INT64 files without conversion.
+ // Dropping the series instead races the scanner: a retry during deletion can be classified as
+ // permanent and move the original handoff to failed before a matching schema can be created.
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("alter timeseries " + DEVICE + "." + MEASUREMENT + " set data type INT64");
+ }
+
+ final List expected = new ArrayList<>();
+ for (int fileIndex = 0; fileIndex < 2; fileIndex++) {
+ for (int row = 0; row < ROWS_PER_FILE; row++) {
+ if (row != fileIndex + 2) {
+ expected.add(timestamp(fileIndex, row) + "," + value(fileIndex, row) + ",");
+ }
+ }
+ }
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertRows(DEVICE, expected);
+ assertRetryQueueEmpty(node);
+ for (final File file : pendingFiles) {
+ Assert.assertFalse(file.exists());
+ Assert.assertFalse(new File(file + ModificationFile.FILE_SUFFIX).exists());
+ }
+ });
+ Assert.assertEquals(
+ "only permanent failures belong in failed",
+ restart ? 2 : 1,
+ findTsFiles(failedDir(node)).size());
+ Assert.assertTrue(new File(failedDir(node), permanentFile.getName()).isFile());
+ assertPermanentFailureWasLogged(node, permanentFile.getName());
+ if (restart) {
+ final File failedIdentity = new File(failedDir(node), IDENTITY_FILE_NAME);
+ Assert.assertTrue(failedIdentity.isFile());
+ Assert.assertTrue(new File(failedIdentity + ModificationFile.FILE_SUFFIX).isFile());
+ final File originalIdentityHandoff =
+ pendingFiles.stream()
+ .filter(file -> IDENTITY_FILE_NAME.equals(file.getName()))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ assertPermissionFailureWasLogged(node, originalIdentityHandoff);
+ Assert.assertTrue(identitySource.isFile());
+ Assert.assertTrue(new File(identitySource + ModificationFile.FILE_SUFFIX).isFile());
+ }
+ Assert.assertTrue(sourceFiles.get(0).isFile());
+ Assert.assertTrue(new File(sourceFiles.get(0) + ModificationFile.FILE_SUFFIX).isFile());
+ Assert.assertFalse(sourceFiles.get(1).exists());
+ assertRows(PERMANENT_DEVICE, Collections.emptyList());
+ }
+
+ private static String loadSql(final File file, final String onSuccess, final boolean convert) {
+ return String.format(
+ "load \"%s\" with ('database-level'='3', 'async'='true', 'on-success'='%s', "
+ + "'verify'='true', 'tablet-conversion-threshold'='0', 'convert-on-type-mismatch'='%s')",
+ file.getAbsolutePath(), onSuccess, convert);
+ }
+
+ private static void assertHandoffAttributes(final File file, final DataNodeWrapper node) {
+ final Map attributes =
+ ActiveLoadPathHelper.parseAttributes(file, activeDir(node));
+ Assert.assertEquals(
+ IDENTITY_FILE_NAME.equals(file.getName()) ? IDENTITY_USER : USER,
+ attributes.get(ActiveLoadPathHelper.USER_KEY));
+ Assert.assertEquals("3", attributes.get("database-level"));
+ Assert.assertEquals("true", attributes.get("convert-on-type-mismatch"));
+ Assert.assertEquals("true", attributes.get("verify"));
+ Assert.assertEquals("0", attributes.get("tablet-conversion-threshold"));
}
- private boolean containsFile(final File root, final String fileName) {
- return findFile(root, fileName) != null;
+ private static void assertPermanentFailureWasLogged(
+ final DataNodeWrapper node, final String fileName) throws Exception {
+ final String expectedMismatch =
+ String.format(
+ DataNodeQueryMessages
+ .EXCEPTION_DATA_TYPE_MISMATCH_FOR_MEASUREMENT_ARGARGARG_TYPE_IN_TSFILE_ARG_TYPE_IN_IOTDB_ARG_C5BA7DBD,
+ PERMANENT_DEVICE,
+ ".",
+ MEASUREMENT,
+ TSDataType.INT64,
+ TSDataType.INT32);
+ final java.nio.file.Path log = new File(node.getLogDirPath(), node.getId() + ".log").toPath();
+ try (final Stream lines = Files.lines(log, StandardCharsets.UTF_8)) {
+ Assert.assertTrue(
+ "no permanent failure was logged for " + fileName,
+ lines.anyMatch(
+ line ->
+ line.contains(fileName)
+ && line.contains("ActiveLoadTsFileLoader")
+ && line.contains(expectedMismatch)
+ && line.contains("code:" + TSStatusCode.LOAD_FILE_ERROR.getStatusCode())));
+ }
}
- private void assertFileKeptForRetry(
- final File activeDir, final File failDir, final String fileName, final long observationMs)
- throws InterruptedException {
- final long deadline = System.currentTimeMillis() + observationMs;
- while (System.currentTimeMillis() < deadline) {
+ private static void assertPermissionFailureWasLogged(
+ final DataNodeWrapper node, final File originalHandoff) throws Exception {
+ final java.nio.file.Path log = new File(node.getLogDirPath(), node.getId() + ".log").toPath();
+ try (final Stream lines = Files.lines(log, StandardCharsets.UTF_8)) {
Assert.assertTrue(
- "Temporary unavailable active load should keep tsfile for retry",
- containsFile(activeDir, fileName));
- Assert.assertFalse(
- "Temporary unavailable active load must not move tsfile to fail dir",
- containsFile(failDir, fileName));
- Thread.sleep(500L);
+ "no stored-user permission failure was logged for " + originalHandoff,
+ lines.anyMatch(
+ line ->
+ line.contains(originalHandoff.getAbsolutePath())
+ && line.contains("ActiveLoadTsFileLoader")
+ && line.contains("WRITE_DATA")
+ && line.contains("code:" + TSStatusCode.NO_PERMISSION.getStatusCode())));
}
}
- private File findFile(final File root, final String fileName) {
- if (root == null || !root.exists()) {
- return null;
+ private static long timestamp(final int fileIndex, final int row) {
+ return 100L * fileIndex + row + 1;
+ }
+
+ private static int value(final int fileIndex, final int row) {
+ return 1000 * (fileIndex + 1) + 7 * row;
+ }
+
+ private static void generateTsFile(final File file, final String device, final int fileIndex)
+ throws Exception {
+ try (final TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTimeseries(new Path(device), Collections.singletonList(TSFILE_SCHEMA));
+ final Tablet tablet = new Tablet(device, Collections.singletonList(TSFILE_SCHEMA));
+ for (int row = 0; row < ROWS_PER_FILE; row++) {
+ tablet.addTimestamp(row, timestamp(fileIndex, row));
+ tablet.addValue(MEASUREMENT, row, (long) value(fileIndex, row));
+ }
+ writer.writeTree(tablet);
}
- if (root.isFile()) {
- return root.getName().equals(fileName) ? root : null;
+ try (final ModificationFile mods =
+ new ModificationFile(new File(file + ModificationFile.FILE_SUFFIX), false)) {
+ final long deletedTime = timestamp(fileIndex, fileIndex + 2);
+ mods.write(
+ new TreeDeletionEntry(
+ new MeasurementPath(device, MEASUREMENT), deletedTime, deletedTime));
}
+ }
- final File[] children = root.listFiles();
- if (children == null) {
- return null;
+ private static void assertRows(final String device, final List expected)
+ throws Exception {
+ TestUtils.treeResultSetEqualTestUnordered(
+ "select " + MEASUREMENT + " from " + device,
+ new String[] {"Time", device + "." + MEASUREMENT},
+ expected.toArray(new String[0]));
+ if (!expected.isEmpty()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement();
+ final ResultSet result =
+ statement.executeQuery("select " + MEASUREMENT + " from " + device + " limit 1")) {
+ Assert.assertEquals(TSDataType.INT64.name(), result.getMetaData().getColumnTypeName(2));
+ }
}
- for (final File child : children) {
- final File file = findFile(child, fileName);
- if (file != null) {
- return file;
+ }
+
+ static void configureRetryMetrics() {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Collections.singletonList("JMX"));
+ final Properties properties = new Properties();
+ properties.setProperty("dn_metric_level", "IMPORTANT");
+ ((MppDataNodeConfig) EnvFactory.getEnv().getConfig().getDataNodeConfig())
+ .updateProperties(properties);
+ }
+
+ static void assertRetryQueueEmpty(final DataNodeWrapper node) throws Exception {
+ if (activeDir(node).exists()) {
+ try (final Stream paths = Files.walk(activeDir(node).toPath())) {
+ Assert.assertFalse(
+ "pending copies, mods and resources must be gone",
+ paths.anyMatch(Files::isRegularFile));
+ }
+ }
+ final int[] ports = node.getPortList();
+ final JMXServiceURL url =
+ new JMXServiceURL(
+ "service:jmx:rmi:///jndi/rmi://"
+ + node.getIp()
+ + ":"
+ + ports[ports.length - 1]
+ + "/jmxrmi");
+ try (final JMXConnector connector = JMXConnectorFactory.connect(url)) {
+ for (final String type : Arrays.asList("queuing", "loading")) {
+ final ObjectName pattern =
+ new ObjectName(
+ "org.apache.iotdb.metrics:name=active_loading_files_number,tag.type="
+ + ObjectName.quote(type)
+ + ",*");
+ final java.util.Set names =
+ connector.getMBeanServerConnection().queryNames(pattern, null);
+ Assert.assertEquals("missing active load metric " + type, 1, names.size());
+ Assert.assertEquals(
+ "the original handoffs must leave " + type,
+ 0L,
+ ((Number)
+ connector
+ .getMBeanServerConnection()
+ .getAttribute(names.iterator().next(), "Count"))
+ .longValue());
}
}
- return null;
}
- private void deleteRecursively(final File file) {
- if (file == null || !file.exists()) {
- return;
+ static void assertSchemaMemoryRejectionWasLogged(final DataNodeWrapper node, final File file)
+ throws Exception {
+ // Schema-analysis allocation uses the legacy memory-message retry classification. Tablet
+ // conversion has a separate typed LOAD_TEMPORARY_UNAVAILABLE status checked below.
+ final String expected =
+ MessageFormatter.format(
+ StorageEngineMessages
+ .STORAGE_LOG_REJECTING_AUTO_LOAD_TSFILE_ISGENERATEDBYPIPE_DUE_TO_MEMORY_9A60DF29,
+ file.getAbsolutePath(),
+ false)
+ .getMessage();
+ final java.nio.file.Path log = new File(node.getLogDirPath(), node.getId() + ".log").toPath();
+ try (final Stream lines = Files.lines(log, StandardCharsets.UTF_8)) {
+ Assert.assertTrue(
+ "no retryable schema-memory rejection observed for " + file,
+ lines.anyMatch(
+ line -> line.contains("ActiveLoadFailedMessageHandler") && line.contains(expected)));
}
- final File[] children = file.listFiles();
- if (children != null) {
- for (final File child : children) {
- deleteRecursively(child);
+ }
+
+ static void assertTemporaryFailureWasLogged(final DataNodeWrapper node, final File file)
+ throws Exception {
+ final java.nio.file.Path log = new File(node.getLogDirPath(), node.getId() + ".log").toPath();
+ try (final Stream lines = Files.lines(log, StandardCharsets.UTF_8)) {
+ Assert.assertTrue(
+ "no temporary failure observed for " + file,
+ lines.anyMatch(
+ line ->
+ line.contains(file.getAbsolutePath())
+ && line.contains("ActiveLoadFailedMessageHandler")
+ && line.contains(
+ "code:"
+ + TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION
+ .getStatusCode())));
+ }
+ }
+
+ static List findTsFiles(final File directory) throws Exception {
+ final List files = new ArrayList<>();
+ if (directory.exists()) {
+ try (final Stream paths = Files.walk(directory.toPath())) {
+ paths
+ .filter(path -> path.toString().endsWith(".tsfile"))
+ .map(java.nio.file.Path::toFile)
+ .forEach(files::add);
}
}
- Assert.assertTrue(file.delete());
+ return files;
+ }
+
+ static File activeDir(final DataNodeWrapper node) {
+ return new File(node.getNodePath(), "ext/load/pending");
+ }
+
+ static File failedDir(final DataNodeWrapper node) {
+ return new File(node.getNodePath(), "ext/load/failed");
+ }
+
+ static void awaitNode(final DataNodeWrapper node) {
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .until(
+ () -> {
+ try (final Connection connection = EnvFactory.getEnv().getConnection(node);
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("show version")) {
+ return result.next();
+ }
+ });
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryTableIT.java
new file mode 100644
index 0000000000000..bf041301f4d1c
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryTableIT.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
+import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.external.commons.io.FileUtils;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.activeDir;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.assertRetryQueueEmpty;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.assertSchemaMemoryRejectionWasLogged;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.awaitNode;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.configureRetryMetrics;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.failedDir;
+import static org.apache.iotdb.db.it.IoTDBLoadTsFileActiveRetryIT.findTsFiles;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLoadTsFileActiveRetryTableIT {
+ private static final String TABLE = "retry_table";
+ private static final String[] DATABASES = {"load_retry_a", "load_retry_b"};
+ private static File tmpDir;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ tmpDir = Files.createTempDirectory("load-active-retry-table").toFile();
+ configureRetryMetrics();
+ // This fixture needs a different startup budget from the tree conversion-rejection cases.
+ EnvFactory.getEnv().getConfig().getCommonConfig().setPipeMemoryManagementEnabled(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(Long.MAX_VALUE / 4)
+ .setLoadActiveListeningCheckIntervalSeconds(1);
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ try {
+ // Stop every process before filesystem cleanup can fail on an earlier node's directory.
+ EnvFactory.getEnv().shutdownForciblyAllDataNodes();
+ EnvFactory.getEnv().shutdownForciblyAllConfigNodes();
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ } finally {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+ }
+
+ /**
+ * LOAD-P0-06 Table difference: two accepted same-named files must keep their explicit target
+ * databases and TAG/FIELD mapping, including null TAGs and all-null FIELD rows, across a process
+ * crash. A logged memory rejection proves a background attempt, rather than just an idle queue.
+ * Restoring the startup budget really removes the failure; no handoff is resubmitted or copied.
+ */
+ @Test
+ public void testPendingTableLoadsRestoreTheirDatabasesAfterRestart() throws Exception {
+ final DataNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(0);
+ final List sources = new ArrayList<>();
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ final Statement statement = connection.createStatement()) {
+ for (int i = 0; i < DATABASES.length; i++) {
+ statement.execute("create database " + DATABASES[i]);
+ final File directory = new File(tmpDir, "source-" + i);
+ Assert.assertTrue(directory.mkdirs());
+ final File source = new File(directory, "1-0-0-0.tsfile");
+ generateTableFile(source, i);
+ sources.add(source);
+ }
+ }
+ // A pinned connection makes the node that owns both handoffs unambiguous.
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(node);
+ final Statement statement = connection.createStatement()) {
+ statement.execute("set sql_dialect=table");
+ for (int i = 0; i < sources.size(); i++) {
+ statement.execute(
+ String.format(
+ "load '%s' with ('async'='true', 'database'='%s', 'on-success'='%s', 'verify'='true', 'tablet-conversion-threshold'='0')",
+ sources.get(i).getAbsolutePath(), DATABASES[i], i == 0 ? "none" : "delete"));
+ }
+ }
+ final List pending = new ArrayList<>();
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ pending.clear();
+ pending.addAll(findTsFiles(activeDir(node)));
+ Assert.assertEquals(2, pending.size());
+ final Set pendingDatabases = new HashSet<>();
+ for (final File file : pending) {
+ assertSchemaMemoryRejectionWasLogged(node, file);
+ final Map attributes =
+ ActiveLoadPathHelper.parseAttributes(file, activeDir(node));
+ final String database = attributes.get(LoadTsFileConfigurator.DATABASE_NAME_KEY);
+ Assert.assertTrue(Arrays.asList(DATABASES).contains(database));
+ Assert.assertEquals("true", attributes.get("verify"));
+ pendingDatabases.add(database);
+ }
+ Assert.assertEquals(2, pendingDatabases.size());
+ Assert.assertTrue(findTsFiles(failedDir(node)).isEmpty());
+ });
+ Assert.assertTrue(sources.get(0).isFile());
+ Assert.assertFalse(sources.get(1).exists());
+ node.stopForcibly();
+ Assert.assertFalse(node.isAlive());
+ for (final File file : pending) {
+ Assert.assertTrue(file.isFile());
+ }
+ final MppDataNodeConfig config =
+ (MppDataNodeConfig) EnvFactory.getEnv().getConfig().getDataNodeConfig();
+ config.setLoadTsFileAnalyzeSchemaMemorySizeInBytes(8 * 1024);
+ node.changeConfig(
+ config, (MppCommonConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig(), null);
+ node.start();
+ awaitNode(node);
+
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ for (int i = 0; i < DATABASES.length; i++) {
+ final List expected = new ArrayList<>();
+ for (int row = 0; row < 6; row++) {
+ expected.add(
+ (row + 1)
+ + ",site_"
+ + i
+ + ","
+ + (row % 2 == 0 ? "null" : "sensor_" + row)
+ + ","
+ + (row == 4 ? "null" : Integer.toString(1000 * i + row))
+ + ","
+ + (row % 2 == 0 ? "null" : "note_" + i + "_" + row)
+ + ",");
+ }
+ TestUtils.tableResultSetEqualTestUnordered(
+ "select cast(time as int64) as t,site,sensor,v,note from " + TABLE,
+ new String[] {"t", "site", "sensor", "v", "note"},
+ expected.toArray(new String[0]),
+ DATABASES[i]);
+ }
+ assertRetryQueueEmpty(node);
+ Assert.assertTrue(findTsFiles(failedDir(node)).isEmpty());
+ });
+ for (final File file : pending) {
+ Assert.assertFalse("the original handoff must be cleaned", file.exists());
+ }
+ for (final String database : DATABASES) {
+ // Values and JDBC types cannot distinguish TAGs from FIELDs or swapped TAG ordinals.
+ // DESCRIBE emits schema order, including the TAG order that defines device identity.
+ TestUtils.tableResultSetEqualTest(
+ "describe " + TABLE,
+ new String[] {"ColumnName", "DataType", "Category"},
+ new String[] {
+ "time,TIMESTAMP,TIME,",
+ "site,STRING,TAG,",
+ "sensor,STRING,TAG,",
+ "v,INT64,FIELD,",
+ "note,STRING,FIELD,"
+ },
+ database);
+ }
+ try (final Connection connection = EnvFactory.getEnv().getTableConnection();
+ final Statement statement = connection.createStatement()) {
+ for (final String database : DATABASES) {
+ statement.execute("use " + database);
+ try (final ResultSet result =
+ statement.executeQuery("select site,sensor,v,note from " + TABLE + " limit 1")) {
+ Assert.assertEquals(TSDataType.STRING.name(), result.getMetaData().getColumnTypeName(1));
+ Assert.assertEquals(TSDataType.STRING.name(), result.getMetaData().getColumnTypeName(2));
+ Assert.assertEquals(TSDataType.INT64.name(), result.getMetaData().getColumnTypeName(3));
+ Assert.assertEquals(TSDataType.STRING.name(), result.getMetaData().getColumnTypeName(4));
+ }
+ }
+ }
+ Assert.assertTrue(sources.get(0).isFile());
+ Assert.assertFalse(sources.get(1).exists());
+ }
+
+ private static void generateTableFile(final File file, final int fileIndex) throws Exception {
+ final List names = Arrays.asList("site", "sensor", "v", "note");
+ final List types =
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.INT64, TSDataType.STRING);
+ final List categories =
+ Arrays.asList(
+ ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD);
+ final List schemas = new ArrayList<>();
+ for (int i = 0; i < names.size(); i++) {
+ schemas.add(new MeasurementSchema(names.get(i), types.get(i)));
+ }
+ try (final TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTableSchema(new TableSchema(TABLE, schemas, categories));
+ final Tablet tablet = new Tablet(TABLE, names, types, categories);
+ for (int row = 0; row < 6; row++) {
+ tablet.addTimestamp(row, row + 1);
+ tablet.addValue("site", row, "site_" + fileIndex);
+ tablet.addValue("sensor", row, row % 2 == 0 ? null : "sensor_" + row);
+ tablet.addValue("v", row, row == 4 ? null : 1000L * fileIndex + row);
+ tablet.addValue("note", row, row % 2 == 0 ? null : "note_" + fileIndex + "_" + row);
+ }
+ writer.writeTable(tablet);
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
index dadd5a7b4bae8..43079a82af945 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
@@ -19,34 +19,65 @@
package org.apache.iotdb.db.it;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils;
+import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdReq;
+import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdResp;
+import org.apache.iotdb.confignode.rpc.thrift.TRegionRouteMapResp;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileConsensusOp;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader;
+import org.apache.iotdb.db.storageengine.load.LoadTsFileProgress;
+import org.apache.iotdb.db.storageengine.load.splitter.ChunkData;
+import org.apache.iotdb.db.storageengine.load.splitter.ChunkPayloadRef;
+import org.apache.iotdb.db.storageengine.load.splitter.TsFileData;
+import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.ClusterConstant;
+import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
+import org.apache.iotdb.it.env.cluster.config.MppDataNodeConfig;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.it.utils.TsFileGenerator;
import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.runtime.ClusterTestResultSet;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.rpc.UrlUtils;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TsFileReader;
import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.read.reader.chunk.ChunkReader;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -61,9 +92,20 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.sql.Connection;
@@ -73,11 +115,13 @@
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.CountDownLatch;
@@ -85,6 +129,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
@@ -134,7 +179,9 @@ public static void setUpClass() {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(REPLICATION_FACTOR)
- .setDataReplicationFactor(REPLICATION_FACTOR);
+ .setDataReplicationFactor(REPLICATION_FACTOR)
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1);
// Keep every kind of merge off for this class: the restart case hard-links the replicas'
// TsFiles out of the DataNode directory and compares their points while the restarted DataNode
// catches up, so compaction must not rewrite those files in the meantime.
@@ -144,11 +191,33 @@ public static void setUpClass() {
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Arrays.asList("JMX", "PROMETHEUS"))
+ .setDeleteWalFilesPeriodInMs(500);
+ final Properties metrics = new Properties();
+ metrics.setProperty("prometheus_reporter_async_update", "false");
+ ((MppCommonConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig())
+ .updateProperties(metrics);
+ final Properties retentionProperties = new Properties();
+ retentionProperties.setProperty("dn_metric_level", "IMPORTANT");
+ retentionProperties.setProperty("load_clean_up_task_execution_delay_time_seconds", "1");
+ retentionProperties.setProperty("load_clean_up_task_idle_timeout_seconds", "5");
+ // The shared executor otherwise rounds the cleaner's one-second period up to twenty seconds.
+ retentionProperties.setProperty(
+ "pipe_subtask_executor_cron_heartbeat_event_interval_seconds", "1");
+ retentionProperties.setProperty("wal_file_size_threshold_in_byte", "16384");
+ ((MppDataNodeConfig) EnvFactory.getEnv().getConfig().getDataNodeConfig())
+ .updateProperties(retentionProperties);
EnvFactory.getEnv().initClusterEnvironment(CONFIG_NODE_NUM, DATA_NODE_NUM);
}
@AfterClass
public static void tearDownClass() {
+ // Stop every process before filesystem cleanup can fail on an earlier node's directory.
+ EnvFactory.getEnv().shutdownForciblyAllDataNodes();
+ EnvFactory.getEnv().shutdownForciblyAllConfigNodes();
// Every waiting attempt deletes its own snapshot directory; this also removes what a JVM that
// died in the middle of an attempt left behind next to the DataNode directories.
final File targetDir =
@@ -178,19 +247,28 @@ public void tearDown() throws Exception {
}
waitUntilClusterAnswers();
- try (final Connection connection = EnvFactory.getEnv().getConnection();
- final Statement statement = connection.createStatement()) {
- statement.execute("delete database " + DATABASE);
- } catch (final Exception ignored) {
- }
-
- final File[] files = tmpDir.listFiles();
- if (files != null) {
- for (final File file : files) {
- Assert.assertTrue(deleteRecursively(file));
+ try {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ boolean exists = false;
+ try (final ResultSet databases = statement.executeQuery("show databases")) {
+ while (databases.next()) {
+ exists |= DATABASE.equals(databases.getString(1));
+ }
+ }
+ if (exists) {
+ statement.execute("delete database " + DATABASE);
+ }
}
+ } finally {
+ final File[] files = tmpDir.listFiles();
+ if (files != null) {
+ for (final File file : files) {
+ Assert.assertTrue(deleteRecursively(file));
+ }
+ }
+ Assert.assertTrue(tmpDir.delete());
}
- Assert.assertTrue(tmpDir.delete());
}
@Test
@@ -304,8 +382,8 @@ public void testLoadAfterOneDataNodeDown() throws Exception {
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+ EnvFactory.getEnv().getDataNodeWrapper(1).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
@@ -347,8 +425,8 @@ public void testRestartedDataNodeIsSyncedWithTheLoadedFile() throws Exception {
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+ EnvFactory.getEnv().getDataNodeWrapper(1).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
awaitPointCount(statement, expectedPointCount);
@@ -363,6 +441,685 @@ public void testRestartedDataNodeIsSyncedWithTheLoadedFile() throws Exception {
awaitPointCount(restartedDataNode, expectedPointCount);
}
+ /**
+ * LOAD-P0-04: a completed task still owns WAL payloads while a real Region replica is offline.
+ * Witness COMMIT above the replication watermark, restart the payload holder, and require the
+ * same retained task to survive until the follower replays it. No new writes trigger cleanup.
+ * Local TsFile values (including physical duplicate detection) supplement routed SQL reads.
+ */
+ @Test
+ public void testRetainedPayloadSurvivesRestartUntilSlowReplicaCatchesUp() throws Exception {
+ final Map expectedS1 = new TreeMap<>();
+ final Map expectedS2 = new TreeMap<>();
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("create database " + DATABASE + " with max_data_region_group_num=1");
+ statement.execute("create timeseries " + DEVICES.get(0) + ".s1 INT64");
+ statement.execute("create timeseries " + DEVICES.get(0) + ".s2 INT64");
+ }
+ // Allocate the sole DataRegion through the existing partition fixture. A seed INSERT would
+ // otherwise be replayed after a crash from the sender's older durable checkpoint, introducing
+ // unrelated unsealed files and duplicate INSERT points into the LOAD-only physical checks.
+ ConfigNodeTestUtils.getOrCreateDataPartitionWithRetry(DATABASE, 0, 1, 0, 1, PARTITION_INTERVAL);
+ final Map> initialRegions =
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .until(IoTDBLoadTsFileClusterIT::dataRegionReplicas, regions -> regions.size() == 1);
+ final int regionId = initialRegions.keySet().iterator().next();
+ final Set regionReplicas = initialRegions.get(regionId);
+ Assert.assertEquals(REPLICATION_FACTOR, regionReplicas.size());
+ final Map dataNodesById = new HashMap<>();
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ final int nodeId = dataNodeId(statement, node);
+ if (regionReplicas.contains(nodeId)) {
+ dataNodesById.put(nodeId, node);
+ }
+ }
+ }
+ final TRegionReplicaSet initialRoute =
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .until(
+ () -> dataRegionRoute(regionId),
+ replicaSet -> routeNodeIds(replicaSet).equals(regionReplicas));
+ Assert.assertEquals(regionReplicas, routeNodeIds(initialRoute));
+ final TDataNodeLocation slowLocation = initialRoute.getDataNodeLocations().get(1);
+ final DataNodeWrapper slowReplica = dataNodesById.get(slowLocation.getDataNodeId());
+ Assert.assertNotNull(slowReplica);
+
+ slowReplica.stopForcibly();
+ Assert.assertFalse("the slow replica process must be stopped", slowReplica.isAlive());
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(slowReplica), Collections.singletonList(NodeStatus.Unknown));
+ // IoTConsensus LOAD selects the first peer in the current route, not SHOW REGIONS' row order
+ // or its Ratis-style Role column. Re-read after marking the offline peer Unknown.
+ final TRegionReplicaSet route = dataRegionRoute(regionId);
+ Assert.assertEquals(regionReplicas, routeNodeIds(route));
+ final DataNodeWrapper coordinator =
+ dataNodesById.get(route.getDataNodeLocations().get(0).getDataNodeId());
+ Assert.assertNotNull(coordinator);
+ Assert.assertNotEquals(coordinator, slowReplica);
+ final long searchIndexBeforeLoad = consensusIndex(coordinator, regionId, "searchIndex");
+ final long safeIndexBeforeLoad = consensusIndex(coordinator, regionId, "safeIndex");
+ final Map> taskNamesBeforeLoad = new HashMap<>();
+ for (final DataNodeWrapper node : dataNodesById.values()) {
+ taskNamesBeforeLoad.put(node, taskDirectoryNames(node, regionId));
+ }
+
+ final File source = new File(tmpDir, "retained.tsfile");
+ final List schemas =
+ Arrays.asList(
+ new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN),
+ new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN));
+ try (final TsFileWriter writer = new TsFileWriter(source)) {
+ writer.registerTimeseries(new org.apache.tsfile.read.common.Path(DEVICES.get(0)), schemas);
+ for (int chunk = 0; chunk < 4; chunk++) {
+ final Tablet tablet = new Tablet(DEVICES.get(0), schemas);
+ for (int row = 0; row < 8; row++) {
+ final int ordinal = 8 * chunk + row;
+ final long time = chunk * PARTITION_INTERVAL + row + 1;
+ final long value = 1000L + 17L * ordinal;
+ tablet.addTimestamp(row, time);
+ tablet.addValue("s1", row, value);
+ tablet.addValue("s2", row, -value);
+ expectedS1.put(time, value);
+ expectedS2.put(time, -value);
+ }
+ writer.writeTree(tablet);
+ writer.flush();
+ }
+ }
+ try (final ModificationFile mods =
+ new ModificationFile(new File(source + ModificationFile.FILE_SUFFIX), false)) {
+ mods.write(
+ new TreeDeletionEntry(
+ new MeasurementPath(DEVICES.get(0), "s2"),
+ PARTITION_INTERVAL + 2,
+ 2 * PARTITION_INTERVAL + 2));
+ }
+ try (final Connection connection =
+ EnvFactory.getEnv().getWriteOnlyConnectionWithSpecifiedDataNode(coordinator);
+ final Statement statement = connection.createStatement()) {
+ statement.execute(
+ "load \""
+ + source.getAbsolutePath()
+ + "\" with ('on-success'='none', 'tablet-conversion-threshold'='0')");
+ }
+
+ Assert.assertTrue("on-success=none must keep the source TsFile", source.isFile());
+ Assert.assertTrue(
+ "on-success=none must keep the source mods",
+ new File(source + ModificationFile.FILE_SUFFIX).isFile());
+
+ final RetainedTask retainedTask =
+ awaitRetainedTask(
+ coordinator,
+ regionId,
+ taskNamesBeforeLoad.get(coordinator),
+ searchIndexBeforeLoad,
+ safeIndexBeforeLoad);
+ final File task = retainedTask.taskDir();
+ final LoadTsFileProgress.TerminalRecord terminal = retainedTask.terminal();
+ assertTsFilesData(listTsFiles(task), expectedS1, expectedS2, "retained task " + task);
+ final Map retained = new HashMap<>();
+ try (final Stream paths = Files.walk(task.toPath())) {
+ for (final java.nio.file.Path path :
+ (Iterable) paths.filter(Files::isRegularFile)::iterator) {
+ retained.put(path, Files.readAllBytes(path));
+ }
+ }
+ Assert.assertTrue(
+ retained.keySet().stream().anyMatch(path -> path.toString().endsWith(".progress")));
+ Assert.assertTrue(
+ retained.keySet().stream()
+ .anyMatch(path -> path.toString().endsWith(ModificationFile.FILE_SUFFIX)));
+ assertLocalReplicaData(coordinator, expectedS1, expectedS2);
+
+ // IoTConsensus periodically persists peer acknowledgements every three minutes. Check the
+ // healthy peer's durable checkpoint before the crash so it will not replay an already applied
+ // task after its terminal directory was cleaned; only the offline replica needs this replay.
+ Awaitility.await()
+ .atMost(4, TimeUnit.MINUTES)
+ .until(
+ () -> {
+ assertRetainedTask(task, terminal, retained);
+ Assert.assertTrue(
+ consensusIndex(coordinator, regionId, "safeIndex") < terminal.searchIndex());
+ boolean healthyPeersFlushed = true;
+ for (final TDataNodeLocation peer : route.getDataNodeLocations()) {
+ if (dataNodesById.get(peer.getDataNodeId()) != coordinator
+ && peer.getDataNodeId() != slowLocation.getDataNodeId()) {
+ healthyPeersFlushed &=
+ persistedPeerSyncIndex(coordinator, regionId, peer) >= terminal.searchIndex();
+ }
+ }
+ return healthyPeersFlushed;
+ });
+
+ // ASYNC WAL may acknowledge before its buffer reaches disk. This scenario needs durable WAL
+ // references before the crash; it does not test the separate in-flight durability window.
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .untilAsserted(() -> assertTaskWALPayloads(coordinator, task, terminal));
+ coordinator.stopForcibly();
+ Assert.assertFalse(
+ "inspect WAL only after the holder process has stopped", coordinator.isAlive());
+ final Set taskIndexes = assertTaskWALPayloads(coordinator, task, terminal);
+ final long firstPieceIndex = Collections.min(taskIndexes);
+ Assert.assertTrue(firstPieceIndex > searchIndexBeforeLoad);
+ coordinator.start();
+ awaitNode(coordinator);
+ // Only the periodic disk scan can reclaim this empty, unmanaged directory. It carries no
+ // LOAD command or data: its removal witnesses a real scan after the five-second idle timeout,
+ // while the original committed task must remain byte-for-byte intact in that same region.
+ final File scanProbe =
+ Files.createTempDirectory(task.getParentFile().toPath(), "cleaner-scan-probe-").toFile();
+ final long retentionStartNanos = System.nanoTime();
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ Assert.assertTrue(
+ consensusIndex(coordinator, regionId, "safeIndex") < terminal.searchIndex());
+ Assert.assertTrue(
+ peerSyncIndex(coordinator, regionId, slowLocation) < firstPieceIndex);
+ assertRetainedTask(task, terminal, retained);
+ // A Callable condition propagates assertion failures immediately instead of
+ // restarting a successful observation period after a retention violation.
+ return System.nanoTime() - retentionStartNanos >= TimeUnit.SECONDS.toNanos(6)
+ && !scanProbe.exists();
+ });
+ slowReplica.start();
+ Awaitility.await()
+ // safeIndex is the minimum in-memory acknowledgement, not the WAL deletion watermark.
+ // With no subscriptions, the latter is the minimum durable peer checkpoint. Keep checking
+ // retention while the follower starts and catches up, until the three-minute periodic
+ // flush persists the original COMMIT. Waiting for readiness or scanning the follower first
+ // would leave an unchecked window where in-memory acknowledgements could release payloads
+ // before the durable checkpoint permits it.
+ .atMost(4, TimeUnit.MINUTES)
+ .until(
+ () -> {
+ if (persistedSafeIndex(coordinator, regionId, route) >= terminal.searchIndex()) {
+ return true;
+ }
+ try {
+ assertRetainedTask(task, terminal, retained);
+ } catch (final AssertionError | IOException e) {
+ // The checkpoint may advance and legitimately delete files during this read.
+ if (persistedSafeIndex(coordinator, regionId, route) < terminal.searchIndex()) {
+ throw e;
+ }
+ return true;
+ }
+ return false;
+ });
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Assert.assertTrue(
+ consensusIndex(coordinator, regionId, "safeIndex") >= terminal.searchIndex());
+ Assert.assertTrue(
+ peerSyncIndex(coordinator, regionId, slowLocation) >= terminal.searchIndex());
+ Assert.assertTrue(
+ persistedSafeIndex(coordinator, regionId, route) >= terminal.searchIndex());
+ Assert.assertEquals(
+ "no extra writes may drive replication or cleanup",
+ terminal.searchIndex(),
+ consensusIndex(coordinator, regionId, "searchIndex"));
+ // The restarted writer has no pre-crash dispatch queue. The original task's WAL
+ // records reference the retained bytes above; the target peer now acknowledges
+ // COMMIT.
+ Assert.assertTrue(
+ consensusIndex(coordinator, regionId, "LogEntriesFromWAL") >= taskIndexes.size());
+ assertNoNewTaskDirectories(
+ dataNodesById.values(), regionId, taskNamesBeforeLoad, task.getName());
+ });
+ // A durable acknowledgement above already proves application of the original COMMIT. Check
+ // readiness and physical data only after observing retention through that durable boundary.
+ awaitNode(slowReplica);
+ // Local payloads remain readable after staging is gone; SQL also checks the mods boundaries.
+ for (final DataNodeWrapper node : dataNodesById.values()) {
+ assertLocalReplicaData(node, expectedS1, expectedS2);
+ }
+ final List expectedRows = new ArrayList<>();
+ for (final Map.Entry row : expectedS1.entrySet()) {
+ final boolean deleted =
+ row.getKey() >= PARTITION_INTERVAL + 2 && row.getKey() <= 2 * PARTITION_INTERVAL + 2;
+ expectedRows.add(
+ row.getKey()
+ + ","
+ + row.getValue()
+ + ","
+ + (deleted ? "null" : Long.toString(-row.getValue()))
+ + ",");
+ }
+ TestUtils.treeResultSetEqualTestUnordered(
+ "select s1,s2 from " + DEVICES.get(0),
+ new String[] {"Time", DEVICES.get(0) + ".s1", DEVICES.get(0) + ".s2"},
+ expectedRows.toArray(new String[0]));
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement();
+ final ResultSet result =
+ statement.executeQuery("select s1,s2 from " + DEVICES.get(0) + " limit 1")) {
+ for (final ResultSet nodeResult : ClusterTestResultSet.getNodeResultSets(result)) {
+ Assert.assertEquals(TSDataType.INT64.name(), nodeResult.getMetaData().getColumnTypeName(2));
+ Assert.assertEquals(TSDataType.INT64.name(), nodeResult.getMetaData().getColumnTypeName(3));
+ }
+ }
+ }
+
+ private static RetainedTask awaitRetainedTask(
+ final DataNodeWrapper writer,
+ final int regionId,
+ final Set taskNamesBeforeLoad,
+ final long searchIndexBeforeLoad,
+ final long safeIndexBeforeLoad) {
+ final java.util.concurrent.atomic.AtomicReference taskReference =
+ new java.util.concurrent.atomic.AtomicReference<>();
+ final java.util.concurrent.atomic.AtomicReference
+ terminalReference = new java.util.concurrent.atomic.AtomicReference<>();
+ Awaitility.await()
+ .atMost(2, TimeUnit.MINUTES)
+ .ignoreExceptions()
+ .untilAsserted(
+ () -> {
+ final List newTasks =
+ findNewTaskDirectories(writer, regionId, taskNamesBeforeLoad);
+ Assert.assertEquals(
+ "the IoTConsensus write peer must retain exactly this LOAD task",
+ 1,
+ newTasks.size());
+ final File task = newTasks.get(0);
+ final LoadTsFileProgress.TerminalRecord terminal =
+ LoadTsFileProgress.readTerminalOfTaskDir(task);
+ Assert.assertNotNull("the LOAD task must have a terminal record", terminal);
+ Assert.assertEquals(LoadTsFileConsensusOp.COMMIT, terminal.op());
+ Assert.assertTrue(
+ "the task must be newer than the pre-LOAD consensus position",
+ terminal.searchIndex() > searchIndexBeforeLoad);
+ Assert.assertTrue(
+ "the writer must have applied the task",
+ consensusIndex(writer, regionId, "searchIndex") >= terminal.searchIndex());
+ Assert.assertTrue(
+ "the slow replica must keep the task above the safe-deletion watermark",
+ safeIndexBeforeLoad < terminal.searchIndex()
+ && consensusIndex(writer, regionId, "safeIndex") < terminal.searchIndex());
+ taskReference.set(task);
+ terminalReference.set(terminal);
+ });
+ return new RetainedTask(taskReference.get(), terminalReference.get());
+ }
+
+ private static Set taskDirectoryNames(final DataNodeWrapper node, final int regionId) {
+ final Set names = new HashSet<>();
+ final File regionDir =
+ new File(node.getDataNodeDir(), "data/load/" + DATABASE + "-" + regionId);
+ final File[] taskDirs = regionDir.listFiles(File::isDirectory);
+ if (taskDirs != null) {
+ for (final File taskDir : taskDirs) {
+ names.add(taskDir.getName());
+ }
+ }
+ return names;
+ }
+
+ private static List findNewTaskDirectories(
+ final DataNodeWrapper node, final int regionId, final Set taskNamesBeforeLoad) {
+ final File regionDir =
+ new File(node.getDataNodeDir(), "data/load/" + DATABASE + "-" + regionId);
+ final File[] taskDirs = regionDir.listFiles(File::isDirectory);
+ if (taskDirs == null) {
+ return Collections.emptyList();
+ }
+ final List newTaskDirs = new ArrayList<>();
+ for (final File taskDir : taskDirs) {
+ if (!taskNamesBeforeLoad.contains(taskDir.getName())) {
+ newTaskDirs.add(taskDir);
+ }
+ }
+ return newTaskDirs;
+ }
+
+ private static void assertNoNewTaskDirectories(
+ final Iterable nodes,
+ final int regionId,
+ final Map> taskNamesBeforeLoad,
+ final String expectedTaskName) {
+ for (final DataNodeWrapper node : nodes) {
+ final List remaining =
+ findNewTaskDirectories(node, regionId, taskNamesBeforeLoad.get(node));
+ Assert.assertTrue(
+ "the retained task must be deleted on DataNode "
+ + node.getId()
+ + " after the safe watermark; remaining="
+ + remaining,
+ remaining.stream().noneMatch(task -> task.getName().equals(expectedTaskName)));
+ Assert.assertTrue(
+ "an unrelated new task must not be left behind on DataNode " + node.getId(),
+ remaining.isEmpty());
+ }
+ }
+
+ private static TRegionReplicaSet dataRegionRoute(final int regionId) throws Exception {
+ try (final SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final TRegionReplicaSet route =
+ client
+ .getLatestRegionRouteMap()
+ .getRegionRouteMap()
+ .get(new DataRegionId(regionId).convertToTConsensusGroupId());
+ Assert.assertNotNull("missing DataRegion route " + regionId, route);
+ return route;
+ }
+ }
+
+ private static Set routeNodeIds(final TRegionReplicaSet route) {
+ final Set ids = new HashSet<>();
+ for (final TDataNodeLocation location : route.getDataNodeLocations()) {
+ ids.add(location.getDataNodeId());
+ }
+ return ids;
+ }
+
+ private static long consensusIndex(
+ final DataNodeWrapper node, final int regionId, final String kind) throws Exception {
+ return consensusMetric(node, regionId, kind, "");
+ }
+
+ private static long peerSyncIndex(
+ final DataNodeWrapper writer, final int regionId, final TDataNodeLocation peer)
+ throws Exception {
+ return consensusMetric(
+ writer,
+ regionId,
+ "currentSyncIndex",
+ ",tag.name="
+ + ObjectName.quote(
+ "logDispatcher-"
+ + peer.getDataRegionConsensusEndPoint().getIp()
+ + ":"
+ + peer.getDataRegionConsensusEndPoint().getPort()));
+ }
+
+ private static long persistedPeerSyncIndex(
+ final DataNodeWrapper writer, final int regionId, final TDataNodeLocation peer) {
+ // IndexController restores the largest - checkpoint in this region's directory.
+ final File directory =
+ new File(
+ writer.getDataNodeDir(),
+ "consensus/data_region/"
+ + new DataRegionId(regionId).getType().getValue()
+ + "_"
+ + regionId);
+ final String prefix = peer.getDataNodeId() + "-";
+ final File[] checkpoints = directory.listFiles((dir, name) -> name.matches(prefix + "\\d+"));
+ long index = -1;
+ if (checkpoints != null) {
+ for (final File checkpoint : checkpoints) {
+ index = Math.max(index, Long.parseLong(checkpoint.getName().substring(prefix.length())));
+ }
+ }
+ return index;
+ }
+
+ private static long persistedSafeIndex(
+ final DataNodeWrapper writer, final int regionId, final TRegionReplicaSet route) {
+ long index = Long.MAX_VALUE;
+ for (final TDataNodeLocation peer : route.getDataNodeLocations()) {
+ if (peer.getDataRegionConsensusEndPoint().getPort() != writer.getDataRegionConsensusPort()
+ || !peer.getDataRegionConsensusEndPoint().getIp().equals(writer.getIp())) {
+ index = Math.min(index, persistedPeerSyncIndex(writer, regionId, peer));
+ }
+ }
+ Assert.assertNotEquals("the writer must have remote peers", Long.MAX_VALUE, index);
+ return index;
+ }
+
+ private static long consensusMetric(
+ final DataNodeWrapper node, final int regionId, final String kind, final String peerTag)
+ throws Exception {
+ final int[] ports = node.getPortList();
+ final JMXServiceURL url =
+ new JMXServiceURL(
+ "service:jmx:rmi:///jndi/rmi://"
+ + node.getIp()
+ + ":"
+ + ports[ports.length - 1]
+ + "/jmxrmi");
+ try (final JMXConnector connector = JMXConnectorFactory.connect(url)) {
+ final ObjectName pattern =
+ new ObjectName(
+ "org.apache.iotdb.metrics:name=iot_consensus,tag.region="
+ + ObjectName.quote(new DataRegionId(regionId).toString())
+ + ",tag.type="
+ + ObjectName.quote(kind)
+ + peerTag
+ + ",*");
+ final java.util.Set names =
+ connector.getMBeanServerConnection().queryNames(pattern, null);
+ Assert.assertEquals("missing consensus metric " + pattern, 1, names.size());
+ return ((Number)
+ connector.getMBeanServerConnection().getAttribute(names.iterator().next(), "Value"))
+ .longValue();
+ }
+ }
+
+ private static void assertLocalReplicaData(
+ final DataNodeWrapper node,
+ final Map expectedS1,
+ final Map expectedS2)
+ throws Exception {
+ final List files = new ArrayList<>();
+ for (final String space : Arrays.asList("sequence", "unsequence")) {
+ files.addAll(listTsFiles(new File(node.getDataNodeDir(), "data/" + space + "/" + DATABASE)));
+ }
+ assertTsFilesData(files, expectedS1, expectedS2, "local replica on " + node.getId());
+ }
+
+ private static List listTsFiles(final File directory) throws Exception {
+ final List files = new ArrayList<>();
+ if (directory.exists()) {
+ try (final Stream paths = Files.walk(directory.toPath())) {
+ paths
+ .filter(path -> path.toString().endsWith(".tsfile"))
+ .map(java.nio.file.Path::toFile)
+ .forEach(files::add);
+ }
+ }
+ return files;
+ }
+
+ private static void assertTsFilesData(
+ final List files,
+ final Map expectedS1,
+ final Map expectedS2,
+ final String description)
+ throws Exception {
+ final Map> raw = new HashMap<>();
+ final Map> visible = new HashMap<>();
+ for (final String measurement : Arrays.asList("s1", "s2")) {
+ raw.put(measurement, new TreeMap<>());
+ visible.put(measurement, new TreeMap<>());
+ }
+ for (final File file : files) {
+ final List modifications = ModificationFile.readAllModifications(file, false);
+ try (final TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) {
+ for (final Map.Entry> device :
+ reader.getAllTimeseriesMetadata(true).entrySet()) {
+ Assert.assertEquals(description, DEVICES.get(0), device.getKey().toString());
+ for (final TimeseriesMetadata series : device.getValue()) {
+ final String measurement = series.getMeasurementId();
+ Assert.assertTrue(
+ description + ": unexpected measurement " + measurement,
+ raw.containsKey(measurement));
+ Assert.assertEquals(description, TSDataType.INT64, series.getTsDataType());
+ final MeasurementPath seriesPath = new MeasurementPath(DEVICES.get(0), measurement);
+ // Read each physical chunk/page directly: TsFileReader.query may merge equal timestamps
+ // within one file and hide a duplicate import even when its SQL count is unchanged.
+ for (final IChunkMetadata metadata : series.getChunkMetadataList()) {
+ try (final ChunkReader chunk =
+ new ChunkReader(reader.readMemChunk((ChunkMetadata) metadata))) {
+ while (chunk.hasNextSatisfiedPage()) {
+ final BatchData page = chunk.nextPageData();
+ while (page.hasCurrent()) {
+ final long time = page.currentTime();
+ final long value = page.getLong();
+ Assert.assertNull(
+ description + ": duplicate physical point " + measurement + "@" + time,
+ raw.get(measurement).put(time, value));
+ boolean deleted = false;
+ for (final ModEntry modification : modifications) {
+ Assert.assertTrue(description, modification instanceof TreeDeletionEntry);
+ final TreeDeletionEntry deletion = (TreeDeletionEntry) modification;
+ deleted |=
+ deletion.matches(seriesPath)
+ && deletion.getStartTime() <= time
+ && time <= deletion.getEndTime();
+ }
+ if (!deleted) {
+ visible.get(measurement).put(time, value);
+ }
+ page.next();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ Assert.assertEquals(description + ": raw s1", expectedS1, raw.get("s1"));
+ Assert.assertEquals(description + ": raw s2", expectedS2, raw.get("s2"));
+ final Map visibleS2 = new TreeMap<>(expectedS2);
+ visibleS2
+ .keySet()
+ .removeIf(time -> time >= PARTITION_INTERVAL + 2 && time <= 2 * PARTITION_INTERVAL + 2);
+ // Apply each file's own mods. One correct mods file cannot hide a missing/wrong deletion on
+ // another partition, and extra deletions of s1 or of boundary points are detected as well.
+ Assert.assertEquals(description + ": visible s1", expectedS1, visible.get("s1"));
+ Assert.assertEquals(description + ": visible s2", visibleS2, visible.get("s2"));
+ }
+
+ private static Set assertTaskWALPayloads(
+ final DataNodeWrapper writer,
+ final File task,
+ final LoadTsFileProgress.TerminalRecord terminal)
+ throws Exception {
+ // The offline peer pins these indexes. Before the stop this read waits for durable ASYNC WAL;
+ // after the stop it confirms the same records survived, without an active writer racing it.
+ final Map commands = new TreeMap<>();
+ final List walFiles = new ArrayList<>();
+ try (final Stream paths =
+ Files.walk(new File(writer.getWalDir()).toPath())) {
+ for (final java.nio.file.Path wal :
+ (Iterable)
+ paths.filter(path -> path.toString().endsWith(".wal"))::iterator) {
+ walFiles.add(wal.getFileName() + ":" + Files.size(wal));
+ try (final WALReader reader = new WALReader(wal.toFile(), true)) {
+ while (reader.hasNext()) {
+ final Object value = reader.next().getValue();
+ if (value instanceof LoadTsFileConsensusNode command
+ && task.getName().equals(command.getLoadId())) {
+ Assert.assertNull(
+ "duplicate writer WAL index", commands.put(command.getSearchIndex(), command));
+ }
+ }
+ }
+ }
+ }
+ Assert.assertFalse(
+ "the retained task must have original writer WAL records; files=" + walFiles,
+ commands.isEmpty());
+ final LoadTsFileConsensusNode commit = commands.get(terminal.searchIndex());
+ Assert.assertNotNull("COMMIT must be in this writer's WAL", commit);
+ Assert.assertEquals(LoadTsFileConsensusOp.COMMIT, commit.getOp());
+ Assert.assertTrue(
+ commands.values().stream()
+ .anyMatch(command -> command.getOp() == LoadTsFileConsensusOp.PREPARE));
+ final Set chunkIndexes = new HashSet<>();
+ for (final LoadTsFileConsensusNode command : commands.values()) {
+ if (command.getOp() != LoadTsFileConsensusOp.PIECE) {
+ continue;
+ }
+ for (final TsFileData data : command.getTsFileDataList()) {
+ if (!(data instanceof ChunkData chunk)) {
+ continue;
+ }
+ Assert.assertFalse(
+ "WAL must reference staged payload, not inline bytes",
+ chunk.getChunkPayloadRefs().isEmpty());
+ for (final ChunkPayloadRef ref : chunk.getChunkPayloadRefs()) {
+ final File file = stagedReferenceFile(writer, task, ref.getFilePath());
+ Assert.assertTrue(ref.getSize() > 0 && ref.getOffset() + ref.getSize() <= file.length());
+ Assert.assertTrue(
+ "payload reference must match this task's indexed progress",
+ new LoadTsFileProgress(file)
+ .readAllRecords().stream()
+ .anyMatch(
+ record ->
+ record.searchIndex() == command.getSearchIndex()
+ && record.physicalEnd() - record.dataLength() == ref.getOffset()
+ && record.dataLength() == ref.getSize()));
+ chunkIndexes.add(command.getSearchIndex());
+ }
+ }
+ }
+ Assert.assertFalse("the LOAD must need chunk payload replay", chunkIndexes.isEmpty());
+ final Set progressIndexes = new HashSet<>();
+ for (final File staged : listTsFiles(task)) {
+ for (final LoadTsFileProgress.ChunkRangeRecord record :
+ new LoadTsFileProgress(staged).readAllRecords()) {
+ progressIndexes.add(record.searchIndex());
+ }
+ }
+ Assert.assertEquals(
+ "all staged chunks must belong to these WAL PIECEs", progressIndexes, chunkIndexes);
+ return commands.keySet();
+ }
+
+ private static File stagedReferenceFile(
+ final DataNodeWrapper writer, final File task, final String recordedPath) throws Exception {
+ final File recorded = new File(recordedPath);
+ final File file =
+ recorded.isAbsolute()
+ ? recorded
+ : new File(new File(writer.getDataNodeDir(), "data/load"), recordedPath);
+ Assert.assertTrue(
+ "WAL must refer to this retained task: " + file,
+ file.getCanonicalFile().toPath().startsWith(task.getCanonicalFile().toPath()));
+ Assert.assertTrue("missing staged WAL payload " + file, file.isFile());
+ return file;
+ }
+
+ private static void assertRetainedTask(
+ final File task,
+ final LoadTsFileProgress.TerminalRecord terminal,
+ final Map retained)
+ throws IOException {
+ Assert.assertEquals(terminal, LoadTsFileProgress.readTerminalOfTaskDir(task));
+ final Set currentFiles = new HashSet<>();
+ try (final Stream paths = Files.walk(task.toPath())) {
+ paths.filter(Files::isRegularFile).forEach(currentFiles::add);
+ }
+ Assert.assertEquals(retained.keySet(), currentFiles);
+ for (final Map.Entry file : retained.entrySet()) {
+ Assert.assertArrayEquals(
+ "retained payload changed: " + file.getKey(),
+ file.getValue(),
+ Files.readAllBytes(file.getKey()));
+ }
+ }
+
+ private record RetainedTask(File taskDir, LoadTsFileProgress.TerminalRecord terminal) {}
+
/**
* Waits until every data region replica on the given DataNode holds as many points as the other
* replicas of that region.
@@ -612,8 +1369,8 @@ public void testLoadWithTwoDataNodesDownFailsWithAMeaningfulMessage() throws Exc
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+ EnvFactory.getEnv().getDataNodeWrapper(0).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
statement.setQueryTimeout(DEGRADED_LOAD_TIMEOUT_SECONDS);
@@ -693,11 +1450,23 @@ public void testDirectoryQueryFallbackRetainsOnlyFailedSources() throws Exceptio
public void testConcurrentQueryFallbackFailureRetainsSourceForRetry() throws Exception {
final File good = new File(tmpDir, "301-0-0-0.tsfile");
final File bad = new File(tmpDir, "302-0-0-0.tsfile");
- final String device = DATABASE + ".concurrent";
- writeFallbackSource(good, device + ".good", false);
- writeFallbackSource(bad, device + ".bad", false);
+ final String device = DATABASE + ".concurrent.d";
+ writeFallbackSource(good, device, false, 1);
+ writeFallbackSource(bad, device, false, 101);
final long compressionOffset = firstChunkCompressionOffset(bad);
final int originalCompression = replaceByte(bad, compressionOffset, 0x7f);
+ // One device and partition guarantee identical routing without changing the class's Region
+ // allocation policy. Disjoint timestamps expose a failed load borrowing the healthy data.
+ final int sharedRegion;
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("create database " + DATABASE);
+ // Match the aligned source and avoid racing automatic schema creation in the two LOADs.
+ statement.execute(
+ "create aligned timeseries " + device + "(region_marker INT64,s1 INT64,s2 INT64)");
+ statement.execute("insert into " + device + "(time,region_marker) aligned values(0,0)");
+ sharedRegion = assertSharedDataRegion(device);
+ }
final ExecutorService loaders = Executors.newFixedThreadPool(2);
final CountDownLatch ready = new CountDownLatch(2);
final CountDownLatch start = new CountDownLatch(1);
@@ -707,7 +1476,10 @@ public void testConcurrentQueryFallbackFailureRetainsSourceForRetry() throws Exc
loads.add(
loaders.submit(
() -> {
- try (final Connection connection = EnvFactory.getEnv().getConnection();
+ try (final Connection connection =
+ EnvFactory.getEnv()
+ .getConnectionWithSpecifiedDataNode(
+ EnvFactory.getEnv().getDataNodeWrapper(0));
final Statement statement = connection.createStatement()) {
ready.countDown();
Assert.assertTrue(start.await(30, TimeUnit.SECONDS));
@@ -731,15 +1503,25 @@ public void testConcurrentQueryFallbackFailureRetainsSourceForRetry() throws Exc
}
assertFallbackSourceExists(good, false);
assertFallbackSourceExists(bad, true);
+ assertLoadsOverlapped(good, bad);
+ // Check the failed attempt before a new LOAD can hide or release leaked state.
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertLoadMemoryReleased());
try (final Connection connection = EnvFactory.getEnv().getConnection();
final Statement statement = connection.createStatement()) {
- assertFallbackRows(statement, device + ".good");
+ assertFallbackRows(statement, device);
replaceByte(bad, compressionOffset, originalCompression);
+ // This is an explicitly new submission after source repair, not restoration of the
+ // original failed task.
statement.execute(fallbackLoadSql(bad, true));
- assertFallbackRows(statement, device + ".bad");
- assertFallbackRows(statement, device + ".good");
+ assertFallbackRows(statement, device, 1, 101);
+ Assert.assertEquals(sharedRegion, assertSharedDataRegion(device));
}
assertFallbackSourceExists(bad, false);
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertLoadMemoryReleased());
} finally {
start.countDown();
loaders.shutdownNow();
@@ -747,6 +1529,134 @@ public void testConcurrentQueryFallbackFailureRetainsSourceForRetry() throws Exc
}
}
+ private static int assertSharedDataRegion(final String device) throws Exception {
+ try (final SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final ByteArrayOutputStream serializedDevice = new ByteArrayOutputStream();
+ IDeviceID.Factory.DEFAULT_FACTORY.create(device).serialize(serializedDevice);
+ final TGetRegionIdResp response =
+ client.getRegionId(
+ new TGetRegionIdReq(TConsensusGroupType.DataRegion)
+ .setDatabase(DATABASE)
+ .setDevice(serializedDevice.toByteArray())
+ .setStartTimeSlot(new TTimePartitionSlot(0))
+ .setEndTimeSlot(new TTimePartitionSlot(0)));
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode());
+ Assert.assertEquals(1, response.getDataRegionIdListSize());
+ final TConsensusGroupId region = response.getDataRegionIdList().get(0);
+ final TRegionRouteMapResp routes = client.getLatestRegionRouteMap();
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), routes.getStatus().getCode());
+ final List replicas = new ArrayList<>();
+ routes
+ .getRegionRouteMap()
+ .get(region)
+ .getDataNodeLocations()
+ .forEach(node -> replicas.add(node.getClientRpcEndPoint().getPort()));
+ final List expected = new ArrayList<>();
+ for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ expected.add(node.getPort());
+ }
+ Collections.sort(replicas);
+ Collections.sort(expected);
+ Assert.assertEquals(REPLICATION_FACTOR, replicas.size());
+ Assert.assertEquals(expected, replicas);
+ return region.getId();
+ }
+ }
+
+ private static void assertLoadsOverlapped(final File good, final File bad) throws IOException {
+ final DataNodeWrapper coordinator = EnvFactory.getEnv().getDataNodeWrapper(0);
+ final List lines =
+ Files.readAllLines(
+ new File(coordinator.getLogDirPath(), coordinator.getId() + ".log").toPath(),
+ StandardCharsets.UTF_8);
+ final int goodStart =
+ findLogLine(
+ lines,
+ DataNodeQueryMessages.TSFILE_ARG_IS_A_ARG_MODEL_FILE,
+ good.getAbsolutePath(),
+ "tree");
+ final int badStart =
+ findLogLine(
+ lines,
+ DataNodeQueryMessages.TSFILE_ARG_IS_A_ARG_MODEL_FILE,
+ bad.getAbsolutePath(),
+ "tree");
+ final int goodEnd =
+ findLogLine(
+ lines,
+ DataNodeQueryMessages.LOAD_TSFILE_ARG_SUCCESSFULLY_LOAD_PROCESS_ARG_ARG,
+ good.getAbsolutePath(),
+ 1,
+ 1);
+ final int badSplitEnd =
+ findLogLine(
+ lines,
+ DataNodeQueryMessages
+ .LOAD_TSFILE_S_FAILED_WILL_TRY_TO_CONVERT_TO_TABLETS_AND_INSERT_FAILED_TSFILES_ARG,
+ bad.getAbsolutePath());
+ Assert.assertTrue(
+ "Both LOADs must reach the server before either initial load finishes",
+ Math.max(goodStart, badStart) < Math.min(goodEnd, badSplitEnd));
+ }
+
+ private static int findLogLine(
+ final List lines, final String template, final Object... arguments) {
+ final String message = String.format(template.replace("{}", "%s"), arguments);
+ for (int line = 0; line < lines.size(); line++) {
+ if (lines.get(line).contains(message)) {
+ return line;
+ }
+ }
+ throw new AssertionError("Missing LOAD log: " + message);
+ }
+
+ private static void assertLoadMemoryReleased() throws Exception {
+ // Only DataNodes own LOAD memory. Bound each HTTP request and the outer eventual check.
+ for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ final HttpURLConnection metrics =
+ (HttpURLConnection)
+ new URL(
+ "http://"
+ + UrlUtils.formatTEndPointIpv4AndIpv6Url(
+ node.getIp(), node.getMetricPort())
+ + "/metrics")
+ .openConnection();
+ metrics.setConnectTimeout(5000);
+ metrics.setReadTimeout(5000);
+ try {
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, metrics.getResponseCode());
+ final Set gauges = new HashSet<>();
+ try (final BufferedReader lines =
+ new BufferedReader(
+ new InputStreamReader(metrics.getInputStream(), StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = lines.readLine()) != null) {
+ for (final String name :
+ Arrays.asList(
+ "LoadTsFileUsedMemory", "LoadTsFileDataCacheMemory", "LoadTsFileOtherMemory")) {
+ if (line.startsWith("load_mem{") && line.contains("name=\"" + name + "\"")) {
+ Assert.assertEquals(
+ line, 0.0, Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)), 0.0);
+ Assert.assertTrue("Duplicate gauge: " + line, gauges.add(name));
+ }
+ }
+ }
+ }
+ Assert.assertEquals(
+ node.getIpAndPortString(),
+ new HashSet<>(
+ Arrays.asList(
+ "LoadTsFileUsedMemory", "LoadTsFileDataCacheMemory", "LoadTsFileOtherMemory")),
+ gauges);
+ } finally {
+ metrics.disconnect();
+ }
+ }
+ }
+
private static String fallbackLoadSql(final File source, final boolean delete) {
return String.format(
"load '%s' with ('database-level'='1','on-success'='%s',"
@@ -761,6 +1671,12 @@ private static void assertFallbackLoadFailure(final SQLException failure, final
private static long writeFallbackSource(
final File file, final String device, final boolean includeAnchor) throws Exception {
+ return writeFallbackSource(file, device, includeAnchor, 1);
+ }
+
+ private static long writeFallbackSource(
+ final File file, final String device, final boolean includeAnchor, final long startTimestamp)
+ throws Exception {
final List schemas =
Arrays.asList(
new MeasurementSchema(
@@ -785,7 +1701,7 @@ private static long writeFallbackSource(
writer.registerAlignedTimeseries(device, schemas);
final Tablet tablet = new Tablet(device, schemas);
for (int i = 0; i < 4; i++) {
- tablet.addTimestamp(i, i + 1);
+ tablet.addTimestamp(i, i + startTimestamp);
tablet.addValue(i, 0, (long) i);
tablet.addValue(i, 1, (long) i * 10);
}
@@ -793,8 +1709,8 @@ private static long writeFallbackSource(
}
final TsFileResource resource = new TsFileResource(file);
final IDeviceID deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(device);
- resource.updateStartTime(deviceId, 1);
- resource.updateEndTime(deviceId, 4);
+ resource.updateStartTime(deviceId, startTimestamp);
+ resource.updateEndTime(deviceId, startTimestamp + 3);
if (includeAnchor) {
final IDeviceID anchorId = IDeviceID.Factory.DEFAULT_FACTORY.create(anchor);
resource.updateStartTime(anchorId, 0);
@@ -802,7 +1718,9 @@ private static long writeFallbackSource(
}
resource.serialize();
try (final ModificationFile mods = resource.getExclusiveModFile()) {
- mods.write(new TreeDeletionEntry(new MeasurementPath(device + ".s1"), 2, 2));
+ mods.write(
+ new TreeDeletionEntry(
+ new MeasurementPath(device + ".s1"), startTimestamp + 1, startTimestamp + 1));
}
return groupMarkerOffset;
}
@@ -839,16 +1757,24 @@ private static void assertFallbackSourceExists(final File file, final boolean ex
private static void assertFallbackRows(final Statement statement, final String device)
throws SQLException {
+ assertFallbackRows(statement, device, 1);
+ }
+
+ private static void assertFallbackRows(
+ final Statement statement, final String device, final long... startTimestamps)
+ throws SQLException {
try (final ResultSet rows = statement.executeQuery("select s1,s2 from " + device)) {
- for (int i = 0; i < 4; i++) {
- Assert.assertTrue(device, rows.next());
- Assert.assertEquals(i + 1, rows.getLong(1));
- if (i == 1) {
- Assert.assertNull(rows.getObject(2));
- } else {
- Assert.assertEquals(i, rows.getLong(2));
+ for (final long startTimestamp : startTimestamps) {
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue(device, rows.next());
+ Assert.assertEquals(i + startTimestamp, rows.getLong(1));
+ if (i == 1) {
+ Assert.assertNull(rows.getObject(2));
+ } else {
+ Assert.assertEquals((long) i, rows.getObject(2));
+ }
+ Assert.assertEquals(i * 10L, rows.getObject(3));
}
- Assert.assertEquals(i * 10, rows.getLong(3));
}
Assert.assertFalse(device, rows.next());
}
@@ -868,6 +1794,22 @@ private void createDatabaseAndSchema(final Statement statement) throws SQLExcept
}
}
+ /** Waits until the cluster answers again, so the next case does not inherit a broken one. */
+ private static void awaitNode(final DataNodeWrapper node) {
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .until(
+ () -> {
+ try (final Connection connection = EnvFactory.getEnv().getConnection(node);
+ final Statement statement = connection.createStatement();
+ final ResultSet resultSet = statement.executeQuery("show databases")) {
+ return resultSet.next();
+ }
+ });
+ }
+
/** Waits until the cluster answers again, so the next case does not inherit a broken one. */
private static void waitUntilClusterAnswers() {
Awaitility.await()
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
index 3d07749825daa..deb065dff8679 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.it;
+import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
@@ -38,8 +39,6 @@
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -50,18 +49,20 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBPartialInsertionIT {
- private final Logger logger = LoggerFactory.getLogger(IoTDBPartialInsertionIT.class);
-
@BeforeClass
public static void setUp() throws Exception {
- EnvFactory.getEnv().getConfig().getCommonConfig().setAutoCreateSchemaEnabled(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setAutoCreateSchemaEnabled(false)
+ .setWalMode("SYNC");
EnvFactory.getEnv().initClusterEnvironment();
}
@@ -116,46 +117,40 @@ public void testPartialInsertionRestart() throws SQLException {
statement.execute("CREATE TIMESERIES root.sg.d1.s1 datatype=text");
statement.execute("CREATE TIMESERIES root.sg.d1.s2 datatype=double");
- try {
- statement.execute("INSERT INTO root.sg.d1(time,s1,s2) VALUES(100,'test','test')");
- } catch (SQLException e) {
- // ignore
- }
+ SQLException partial =
+ assertThrows(
+ SQLException.class,
+ () ->
+ statement.execute(
+ "INSERT INTO root.sg.d1(time,s1,s2) VALUES(100,'test','test')"));
+ assertEquals(507, partial.getErrorCode());
+ assertTrue(partial.getMessage(), partial.getMessage().contains("s2"));
+ SQLException rejected =
+ assertThrows(
+ SQLException.class,
+ () -> statement.execute("INSERT INTO root.sg.d1(time,s2) VALUES(101,'rejected')"));
+ assertEquals(507, rejected.getErrorCode());
}
- // TODO: replace restartDaemon() with new methods in Env.
- /*
- long time = 0;
- try {
- EnvironmentUtils.restartDaemon();
- StorageEngine.getInstance().recover();
- // wait for recover
- while (!StorageEngine.getInstance().isReadyForReadAndWrite()) {
- Thread.sleep(500);
- time += 500;
- if (time > 10000) {
- logger.warn("wait too long in restart, wait for: " + time / 1000 + "s");
- }
- }
- } catch (Exception e) {
- fail(e.getMessage());
- }
- */
+ assertPartialInsertionData();
+ // No final FLUSH: recover the accepted subset from WAL after a real process crash.
+ TestUtils.stopForciblyAndRestartDataNodes();
+ assertPartialInsertionData();
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.sg.d1")) {
- assertNotNull(resultSet);
- int cnt = 0;
- while (resultSet.next()) {
- cnt++;
- assertEquals("test", resultSet.getString("root.sg.d1.s1"));
- }
- assertEquals(1, cnt);
- }
- try (ResultSet resultSet = statement.executeQuery("SELECT s2 FROM root.sg.d1")) {
- assertNotNull(resultSet);
+ statement.execute("INSERT INTO root.sg.d1(time,s2) VALUES(100,10),(101,11)");
+ statement.execute("FLUSH");
+ try (ResultSet resultSet = statement.executeQuery("SELECT s1,s2 FROM root.sg.d1")) {
+ assertTrue(resultSet.next());
+ assertEquals(100, resultSet.getLong(1));
+ assertEquals("test", resultSet.getString(2));
+ assertEquals(10, resultSet.getDouble(3), 0);
+ assertTrue(resultSet.next());
+ assertEquals(101, resultSet.getLong(1));
+ assertEquals(null, resultSet.getString(2));
+ assertEquals(11, resultSet.getDouble(3), 0);
assertFalse(resultSet.next());
}
}
@@ -164,6 +159,18 @@ public void testPartialInsertionRestart() throws SQLException {
}
}
+ private static void assertPartialInsertionData() throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("SELECT s1,s2 FROM root.sg.d1")) {
+ assertTrue(result.next());
+ assertEquals(100, result.getLong(1));
+ assertEquals("test", result.getString(2));
+ assertEquals(null, result.getString(3));
+ assertFalse("An entirely rejected row must not survive", result.next());
+ }
+ }
+
@Test
public void testPartialInsertTablet() {
try {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
index 8d3609ee87289..9ec4c5356404e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
@@ -20,13 +20,23 @@
package org.apache.iotdb.db.it;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.it.utils.DeletionTestUtils;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import org.awaitility.Awaitility;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -39,6 +49,9 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@@ -55,6 +68,10 @@ public static void setUp() throws Exception {
EnvFactory.getEnv()
.getConfig()
.getCommonConfig()
+ .setDataRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus")
+ .setDataReplicationFactor(1)
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1)
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false)
@@ -68,6 +85,16 @@ public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Before
+ public void resetCompactionConfiguration() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("SET CONFIGURATION 'enable_seq_space_compaction'='false'");
+ statement.execute("SET CONFIGURATION 'enable_unseq_space_compaction'='false'");
+ statement.execute("SET CONFIGURATION 'enable_cross_space_compaction'='false'");
+ }
+ }
+
@Test
public void testDeletionAfterRecreatingTwoLevelTimeseriesSurvivesCompaction() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -96,6 +123,228 @@ public void testDeletionAfterRecreatingTwoLevelTimeseriesSurvivesCompaction() th
}
}
+ @Test
+ public void testDeletionAndRewriteSurviveSeqUnseqCompaction() throws Exception {
+ final String database = "root.delete_compaction";
+ List sourceFiles = new ArrayList<>();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s1 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s2 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d2.s1 WITH DATATYPE=INT64");
+
+ // Keep both the unrewritten deleted point and a surviving measurement in the same files.
+ statement.execute(
+ "INSERT INTO " + database + ".d1(time,s1,s2) VALUES(1,1,101),(2,2,102),(3,3,103)");
+ statement.execute("INSERT INTO " + database + ".d2(time,s1) VALUES(1,201),(2,202),(3,203)");
+ statement.execute("FLUSH");
+ List sequenceFiles = DeletionTestUtils.getClosedTsFiles(database, true);
+ Assert.assertEquals(1, sequenceFiles.size());
+ Assert.assertEquals(
+ Map.of(1L, 1L, 2L, 2L, 3L, 3L),
+ DeletionTestUtils.readTreePoints(sequenceFiles.get(0), database + ".d1", "s1"));
+
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(1,10),(2,20)");
+ statement.execute("INSERT INTO " + database + ".d2(time,s1) VALUES(2,220)");
+ statement.execute("FLUSH");
+ List unsequenceFiles = DeletionTestUtils.getClosedTsFiles(database, false);
+ Assert.assertEquals(1, unsequenceFiles.size());
+ Assert.assertEquals(
+ Map.of(1L, 10L, 2L, 20L),
+ DeletionTestUtils.readTreePoints(unsequenceFiles.get(0), database + ".d1", "s1"));
+ sourceFiles.addAll(sequenceFiles);
+ sourceFiles.addAll(unsequenceFiles);
+
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time <= 2");
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(1,100)");
+ statement.execute("FLUSH");
+ assertCompactionData(statement);
+ sourceFiles.addAll(DeletionTestUtils.getClosedTsFiles(database, false));
+
+ // Only cross-space compaction is enabled. Track the actual input paths and verify the
+ // output's cross-compaction generation, rather than inferring a task from a file count.
+ // These tiny level-0 unseq files must be eligible without an inner-space compaction first.
+ statement.execute("SET CONFIGURATION 'min_cross_compaction_unseq_file_level'='0'");
+ statement.execute("SET CONFIGURATION 'enable_cross_space_compaction'='true'");
+ }
+
+ Awaitility.await()
+ .pollInterval(200, TimeUnit.MILLISECONDS)
+ .atMost(60, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ if (sourceFiles.stream().anyMatch(Files::exists)
+ || !DeletionTestUtils.getClosedTsFiles(database, false).isEmpty()) {
+ return false;
+ }
+ for (Path file : DeletionTestUtils.getClosedTsFiles(database, true)) {
+ if (TsFileNameGenerator.getTsFileName(file.getFileName().toString())
+ .getCrossCompactionCnt()
+ > 0
+ && Long.valueOf(100)
+ .equals(
+ DeletionTestUtils.readTreePoints(file, database + ".d1", "s1")
+ .get(1L))) {
+ return true;
+ }
+ }
+ return false;
+ });
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ assertCompactionData(statement);
+ }
+ TestUtils.stopForciblyAndRestartDataNodes();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ assertCompactionData(statement);
+ statement.execute("INSERT INTO " + database + ".d1(time,s1,s2) VALUES(4,4,104)");
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time = 3");
+ statement.execute("FLUSH");
+ assertCompactionRows(
+ statement, new String[] {"1,100,101,", "2,null,102,", "3,null,103,", "4,4,104,"});
+ }
+ }
+
+ private static void assertCompactionData(Statement statement) throws Exception {
+ assertCompactionRows(statement, new String[] {"1,100,101,", "2,null,102,", "3,3,103,"});
+ }
+
+ @Test
+ public void testDeletionQueryCancellationAllowsCompactionAndContinuedWrites() throws Exception {
+ final String database = "root.delete_cancel";
+ final String device = database + ".d1";
+ final String query = "SELECT s1,s2 FROM " + device + " ORDER BY time";
+ // As in IoTDBShowQueriesIT, leave a full TsBlock after the initial Session fetch so the
+ // query stays registered until it is explicitly killed. Do not assert a concurrent snapshot.
+ final int rowCount = SessionConfig.DEFAULT_FETCH_SIZE + 1000;
+ try (ISession session = EnvFactory.getEnv().getSessionConnection();
+ Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + device + ".s1 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + device + ".s2 WITH DATATYPE=INT64");
+ Tablet tablet =
+ new Tablet(
+ device,
+ List.of(
+ new MeasurementSchema("s1", TSDataType.INT64),
+ new MeasurementSchema("s2", TSDataType.INT64)),
+ rowCount);
+ for (int time = 0; time < rowCount; time++) {
+ tablet.addTimestamp(time, time);
+ tablet.addValue("s1", time, 100000L + time);
+ tablet.addValue("s2", time, 200000L + time);
+ }
+ session.insertTablet(tablet);
+ statement.execute("FLUSH");
+ statement.execute(
+ "INSERT INTO "
+ + device
+ + "(time,s1,s2) VALUES("
+ + rowCount
+ + ","
+ + (100000L + rowCount)
+ + ","
+ + (200000L + rowCount)
+ + ")");
+ statement.execute("FLUSH");
+ final List inputs = DeletionTestUtils.getClosedTsFiles(database, true);
+ Assert.assertEquals(2, inputs.size());
+
+ try (SessionDataSet pending = session.executeQueryStatement(query)) {
+ Assert.assertTrue(pending.hasNext());
+ final String queryId = findDeletionQueryId(statement, query);
+ Assert.assertNotNull("The unconsumed query must actually be registered", queryId);
+ for (int time = 0; time < 5; time++) {
+ statement.execute("DELETE FROM " + device + ".s1 WHERE time=" + time);
+ }
+ statement.execute("KILL QUERY '" + queryId + "'");
+ Awaitility.await()
+ .pollInterval(200, TimeUnit.MILLISECONDS)
+ .atMost(30, TimeUnit.SECONDS)
+ .until(() -> findDeletionQueryId(statement, query) == null);
+ }
+ assertDeletionCancellationRows(statement, device, rowCount, false);
+ statement.execute("SET CONFIGURATION 'enable_seq_space_compaction'='true'");
+ try {
+ Awaitility.await()
+ .pollInterval(200, TimeUnit.MILLISECONDS)
+ .atMost(60, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ if (inputs.stream().anyMatch(Files::exists)) {
+ return false;
+ }
+ for (Path output : DeletionTestUtils.getClosedTsFiles(database, true)) {
+ if (TsFileNameGenerator.getTsFileName(output.getFileName().toString())
+ .getInnerCompactionCnt()
+ > 0) {
+ return true;
+ }
+ }
+ return false;
+ });
+ assertDeletionCancellationRows(statement, device, rowCount, false);
+ // Keep the original Session after cancellation: it must still complete writes/deletes.
+ session.executeNonQueryStatement("INSERT INTO " + device + "(time,s1) VALUES(2,902)");
+ session.deleteData(List.of(device + ".s1"), 7, 7);
+ session.executeNonQueryStatement("FLUSH");
+ assertDeletionCancellationRows(statement, device, rowCount, true);
+ Assert.assertNull(findDeletionQueryId(statement, query));
+ } finally {
+ statement.execute("SET CONFIGURATION 'enable_seq_space_compaction'='false'");
+ }
+ }
+ }
+
+ private static String findDeletionQueryId(Statement statement, String sql) throws Exception {
+ try (ResultSet queries = statement.executeQuery("SHOW QUERIES")) {
+ while (queries.next()) {
+ if (sql.equals(queries.getString("Statement"))) {
+ return queries.getString("QueryId");
+ }
+ }
+ return null;
+ }
+ }
+
+ private static void assertDeletionCancellationRows(
+ Statement statement, String device, int lastTime, boolean continued) throws Exception {
+ try (ResultSet rows =
+ statement.executeQuery("SELECT s1,s2 FROM " + device + " ORDER BY time")) {
+ for (int time = 0; time <= lastTime; time++) {
+ Assert.assertTrue("Missing timestamp " + time, rows.next());
+ Assert.assertEquals(time, rows.getLong(1));
+ if (continued && time == 2) {
+ Assert.assertEquals(902, rows.getLong(2));
+ } else if (time < 5 || (continued && time == 7)) {
+ Assert.assertNull(rows.getObject(2));
+ } else {
+ Assert.assertEquals(100000L + time, rows.getLong(2));
+ }
+ Assert.assertEquals(200000L + time, rows.getLong(3));
+ }
+ Assert.assertFalse(rows.next());
+ }
+ }
+
+ private static void assertCompactionRows(Statement statement, String[] expected)
+ throws Exception {
+ try (ResultSet rows =
+ statement.executeQuery("SELECT s1,s2 FROM root.delete_compaction.d1 ORDER BY time")) {
+ TestUtils.assertResultSetEqual(
+ rows, "Time,root.delete_compaction.d1.s1,root.delete_compaction.d1.s2,", expected);
+ }
+ try (ResultSet rows =
+ statement.executeQuery("SELECT s1 FROM root.delete_compaction.d2 ORDER BY time")) {
+ TestUtils.assertResultSetEqual(
+ rows, "Time,root.delete_compaction.d2.s1,", new String[] {"1,201,", "2,220,", "3,203,"});
+ }
+ }
+
private static void assertOnlyRecreatedTimeseriesDataExists(Statement statement)
throws Exception {
try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.repro")) {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
index 302d059959a5a..68c9df74cfcbe 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
@@ -183,6 +183,75 @@ public void testRestartDelete() throws SQLException {
}
}
+ @Test
+ public void testDeleteIsolationWithinSharedTsFile() throws Exception {
+ final String database = "root.shared_delete";
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s1 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d1.s2 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d2.s1 WITH DATATYPE=INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d2.s2 WITH DATATYPE=INT64");
+ statement.execute("INSERT INTO " + database + ".d1(time,s1,s2) VALUES(1,11,101),(2,12,102)");
+ statement.execute("INSERT INTO " + database + ".d2(time,s1,s2) VALUES(1,21,201),(2,22,202)");
+ statement.execute("FLUSH");
+
+ // One flushed TsFile contains both devices, so the following checks exercise object-level
+ // deletion instead of relying on a separate-file fixture.
+ DeletionTestUtils.assertSharedTsFile(
+ database,
+ IDeviceID.Factory.DEFAULT_FACTORY.create(database + ".d1"),
+ IDeviceID.Factory.DEFAULT_FACTORY.create(database + ".d2"));
+ statement.execute("DELETE FROM " + database + ".d1.s1 WHERE time <= 1");
+ try (ResultSet resultSet =
+ statement.executeQuery("SELECT s1,s2 FROM " + database + ".d1 ORDER BY time")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertNull(resultSet.getObject(2));
+ assertEquals(101, resultSet.getLong(3));
+ assertTrue(resultSet.next());
+ assertEquals(2, resultSet.getLong(1));
+ assertEquals(12, resultSet.getLong(2));
+ assertEquals(102, resultSet.getLong(3));
+ assertFalse(resultSet.next());
+ }
+
+ statement.execute("DELETE FROM " + database + ".d1.**");
+ try (ResultSet rows = statement.executeQuery("SELECT s1,s2 FROM " + database + ".d1")) {
+ assertFalse(rows.next());
+ }
+ statement.execute("INSERT INTO " + database + ".d1(time,s1) VALUES(1,1101)");
+ statement.execute("FLUSH");
+ }
+
+ TestUtils.stopForciblyAndRestartDataNodes();
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet resultSet =
+ statement.executeQuery("SELECT s1,s2 FROM " + database + ".d1 ORDER BY time")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertEquals(1101, resultSet.getLong(2));
+ assertNull(resultSet.getObject(3));
+ assertFalse(resultSet.next());
+ }
+ try (ResultSet resultSet =
+ statement.executeQuery("SELECT s1,s2 FROM " + database + ".d2 ORDER BY time")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertEquals(21, resultSet.getLong(2));
+ assertEquals(201, resultSet.getLong(3));
+ assertTrue(resultSet.next());
+ assertEquals(2, resultSet.getLong(1));
+ assertEquals(22, resultSet.getLong(2));
+ assertEquals(202, resultSet.getLong(3));
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+
@Test
public void testDeleteAndRewriteSurviveForcedRestart() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteBackpressureIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteBackpressureIT.java
new file mode 100644
index 0000000000000..4bde94521758e
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteBackpressureIT.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.session.Session;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(IoTDBTestRunner.class)
+@Category(LocalStandaloneIT.class)
+public class IoTDBWriteBackpressureIT {
+ @BeforeClass
+ public static void setUp() {
+ // A distinct fixture is necessary: these budgets deliberately reject a bounded 2 MiB write.
+ EnvFactory.getEnv().getConfig().getDataNodeJVMConfig().setMaxHeapSize(512);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+ .setWalMode("SYNC")
+ .setEnableMemControl(true)
+ .setEnableLastCache(false)
+ .setDatanodeMemoryProportion("1:1:1:1:1:1")
+ .setWriteMemoryProportion("1:99")
+ .setMemtableSizeThreshold(64L * 1024 * 1024);
+ Properties properties = new Properties();
+ properties.setProperty("write_memory_variation_report_proportion", "0");
+ properties.setProperty("max_waiting_time_when_insert_blocked", "1000");
+ properties.setProperty("check_period_when_insert_blocked", "10");
+ properties.setProperty("wal_entry_size_limit_in_byte", "16777216");
+ properties.setProperty("prometheus_reporter_async_update", "false");
+ ((MppCommonConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig())
+ .updateProperties(properties);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getConfigNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+ EnvFactory.getEnv().initClusterEnvironment(1, 1);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test(timeout = 120_000)
+ public void testMemoryRejectionRollsBackAndOriginalWritersResume() throws Exception {
+ try (Session writer = newWriter();
+ Session other = newWriter()) {
+ writer.executeNonQueryStatement("CREATE DATABASE root.backpressure");
+ writer.executeNonQueryStatement("CREATE DATABASE root.backpressure_rejected");
+ writer.executeNonQueryStatement("CREATE TIMESERIES root.backpressure.plain.s TEXT");
+ writer.executeNonQueryStatement(
+ "CREATE ALIGNED TIMESERIES root.backpressure.aligned(s TEXT)");
+ writer.executeNonQueryStatement("CREATE TIMESERIES root.backpressure_rejected.plain.s TEXT");
+ writer.executeNonQueryStatement(
+ "CREATE ALIGNED TIMESERIES root.backpressure_rejected.aligned(s TEXT)");
+ writer.executeNonQueryStatement(
+ "INSERT INTO root.backpressure.plain(time,s) VALUES(0,'before')");
+ other.executeNonQueryStatement(
+ "INSERT INTO root.backpressure.aligned(time,s) ALIGNED VALUES(0,'before')");
+ long baseline = memoryUsage();
+ assertTrue(baseline > 0);
+ for (boolean aligned : new boolean[] {false, true}) {
+ // Keep the target processor empty: it cannot escape admission by selecting its own
+ // existing memtable for flush. Other Regions can flush, but not the pending reservation.
+ String device = "root.backpressure_rejected." + (aligned ? "aligned" : "plain");
+ Tablet tablet =
+ new Tablet(
+ device, Collections.singletonList(new MeasurementSchema("s", TSDataType.TEXT)), 64);
+ for (int row = 0; row < 64; row++) {
+ tablet.addTimestamp(row, row + 1);
+ tablet.addValue("s", row, "x".repeat(32 * 1024));
+ }
+ StatementExecutionException rejected =
+ assertThrows(
+ StatementExecutionException.class,
+ () -> {
+ if (aligned) {
+ other.insertAlignedTablet(tablet);
+ } else {
+ writer.insertTablet(tablet);
+ }
+ });
+ assertEquals(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode(), rejected.getStatusCode());
+ // A single tablet/processor times out during admission, before WAL or any row insertion.
+ await()
+ .atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertTrue("Rejected reservation leaked memory", memoryUsage() <= baseline);
+ assertEquals(0, memoryUsage("root.backpressure_rejected"));
+ });
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("SELECT s FROM " + device)) {
+ assertFalse("Rejected tablet must not insert any rows", result.next());
+ }
+ }
+ // Keep both original clients. Clearing admission pressure must release every writer.
+ writer.executeNonQueryStatement(
+ "INSERT INTO root.backpressure.plain(time,s) VALUES(1,'after')");
+ other.executeNonQueryStatement(
+ "INSERT INTO root.backpressure.aligned(time,s) ALIGNED VALUES(1,'after')");
+ writer.executeNonQueryStatement(
+ "INSERT INTO root.backpressure_rejected.plain(time,s) VALUES(1,'recovered')");
+ other.executeNonQueryStatement(
+ "INSERT INTO root.backpressure_rejected.aligned(time,s) ALIGNED VALUES(1,'recovered')");
+ assertTrue(
+ "Recovered writes must be charged to their database",
+ memoryUsage("root.backpressure_rejected") > 0);
+ writer.executeNonQueryStatement("FLUSH");
+ await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertEquals(0, memoryUsage());
+ assertEquals(0, memoryUsage("root.backpressure_rejected"));
+ });
+ assertRows("root.backpressure.plain", true);
+ assertRows("root.backpressure.aligned", true);
+ for (String device : new String[] {"plain", "aligned"}) {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result =
+ statement.executeQuery("SELECT s FROM root.backpressure_rejected." + device)) {
+ assertTrue(result.next());
+ assertEquals(1, result.getLong(1));
+ assertEquals("recovered", result.getString(2));
+ assertFalse(result.next());
+ }
+ }
+ // A second admission after flush also checks that rejection did not permanently latch.
+ other.executeNonQueryStatement(
+ "INSERT INTO root.backpressure.aligned(time,s) ALIGNED VALUES(1,'after')");
+ writer.executeNonQueryStatement("FLUSH");
+ await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertEquals(0, memoryUsage());
+ assertEquals(0, memoryUsage("root.backpressure_rejected"));
+ });
+ }
+ }
+
+ private static Session newWriter() throws Exception {
+ Session session =
+ new Session.Builder()
+ .host(EnvFactory.getEnv().getDataNodeWrapper(0).getIp())
+ .port(EnvFactory.getEnv().getDataNodeWrapper(0).getPort())
+ .enableAutoFetch(false)
+ .maxRetryCount(0)
+ .build();
+ // Expose the memory rejection rather than hiding it behind the SDK's default 60 retries.
+ session.open(false, 30_000);
+ return session;
+ }
+
+ private static void assertRows(String device, boolean after) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("SELECT s FROM " + device)) {
+ assertTrue(result.next());
+ assertEquals(0, result.getLong(1));
+ assertEquals("before", result.getString(2));
+ if (after) {
+ assertTrue(result.next());
+ assertEquals(1, result.getLong(1));
+ assertEquals("after", result.getString(2));
+ }
+ assertFalse("Rejected tablet must not insert any rows", result.next());
+ }
+ }
+
+ private static long memoryUsage() {
+ return memoryUsage("root.backpressure");
+ }
+
+ private static long memoryUsage(String database) {
+ return EnvFactory.getEnv().getMetricPrometheusReporterContents(null).stream()
+ .flatMap(String::lines)
+ .filter(
+ line -> line.startsWith("mem{") && line.contains("name=\"database_" + database + "\""))
+ .mapToLong(line -> (long) Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Missing database write memory gauge"));
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java
index 6f743ee307397..f0783e76d59d7 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBWriteMemoryLifecycleIT.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.it;
import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
@@ -38,15 +41,25 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
@@ -82,6 +95,10 @@ public void setUp() throws Exception {
properties.setProperty("enable_timed_flush_seq_memtable", "false");
properties.setProperty("enable_timed_flush_unseq_memtable", "false");
properties.setProperty("wal_entry_size_limit_in_byte", "16384");
+ properties.setProperty("wal_file_size_threshold_in_byte", "1024");
+ properties.setProperty("enable_wal_compression", "false");
+ // Retain the rolled segments until the crash instead of snapshotting/deleting them.
+ properties.setProperty("delete_wal_files_period_in_ms", "3600000");
// Read live accounting, not the reporter's 15-second cached snapshot.
properties.setProperty("prometheus_reporter_async_update", "false");
((MppCommonConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig())
@@ -213,25 +230,47 @@ public void testWalRejectionRollsBackAndAllowsFurtherWrites() throws Exception {
}
@Test
- public void testCrashRecoveryThenWriteAndRelease() throws Exception {
+ public void testMultiWalSegmentMixedFlushCrashRecovery() throws Exception {
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
insert(session, 0, 129);
+ flushAndAssertReleased(session);
+ writeRecoveryTable(0);
+ session.executeNonQueryStatement("FLUSH");
+ assertRecoveryTable(0);
+ assertTrue("The baseline must have a sealed TsFile", sealedFileCount() > 0);
+ long sealedBaseline = sealedFileCount();
+
+ // Two time partitions, both aligned/non-aligned processors, and updates to disk keys.
+ insert(session, 935, 129);
+ insert(session, 1064, 129);
+ overwrite(session, 0, "before-crash");
+ writeRecoveryTable(1);
assertTrue(memoryUsage() > 0);
+ assertTrue(memoryUsage("wal_recovery_table") > 0);
+ assertEquals(
+ "The acknowledged tail must remain unflushed", sealedBaseline, sealedFileCount());
+ assertRolledRecoveryWal();
+ assertData();
+ assertRecoveryTable(1);
}
// No FLUSH and no graceful shutdown: acknowledged data must come back through WAL recovery.
TestUtils.stopForciblyAndRestartDataNodes();
assertData();
+ assertRecoveryTable(1);
// Recovery flushes the replayed memtable and seals its TsFile before accepting requests.
assertEquals("Recovery must leave no live write memory", 0, memoryUsage());
+ assertEquals(0, memoryUsage("wal_recovery_table"));
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
insert(session, 129, 129);
overwrite(session, 0, "after-recovery");
+ writeRecoveryTable(2);
assertTrue("New writes after recovery must be charged", memoryUsage() > 0);
assertData();
flushAndAssertReleased(session);
}
TestUtils.stopForciblyAndRestartDataNodes();
assertData();
+ assertRecoveryTable(2);
assertEquals("Closed files must not retain write memory after restart", 0, memoryUsage());
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
insert(session, 258, 65);
@@ -241,6 +280,125 @@ public void testCrashRecoveryThenWriteAndRelease() throws Exception {
}
}
+ private static long sealedFileCount() throws Exception {
+ try (Stream files =
+ Files.walk(Paths.get(EnvFactory.getEnv().getDataNodeWrapper(0).getDataNodeDir()))) {
+ return files.filter(path -> path.toString().endsWith(".tsfile.resource")).count();
+ }
+ }
+
+ private static void assertRolledRecoveryWal() throws Exception {
+ Map> walNodes;
+ try (Stream files =
+ Files.walk(Paths.get(EnvFactory.getEnv().getDataNodeWrapper(0).getWalDir()))) {
+ walNodes =
+ files
+ .filter(path -> path.toString().endsWith(".wal"))
+ .collect(Collectors.groupingBy(Path::getParent));
+ }
+ Set rolledDevices = new HashSet<>();
+ for (List segments : walNodes.values()) {
+ segments.sort(
+ Comparator.comparingLong(
+ path -> WALFileUtils.parseVersionId(path.getFileName().toString())));
+ Map tailSegments = new HashMap<>();
+ // The newest segment may still be active. Read only sealed predecessors, and count
+ // segments containing the unflushed business tail, excluding the flushed baseline/audit.
+ for (int index = 0; index + 1 < segments.size(); index++) {
+ Set devicesInSegment = new HashSet<>();
+ try (WALReader reader = new WALReader(segments.get(index).toFile())) {
+ while (reader.hasNext()) {
+ Object value = reader.next().getValue();
+ if (value instanceof InsertTabletNode) {
+ InsertTabletNode tablet = (InsertTabletNode) value;
+ String device = tablet.getTargetPath().getFullPath();
+ if (Arrays.asList(DEVICES).contains(device)
+ && tablet.getRowCount() > 0
+ && tablet.getTimes()[0] >= 935) {
+ devicesInSegment.add(device);
+ }
+ }
+ }
+ assertFalse(
+ "Malformed sealed recovery WAL: " + segments.get(index), reader.isFileCorrupted());
+ }
+ devicesInSegment.forEach(device -> tailSegments.merge(device, 1, Integer::sum));
+ }
+ tailSegments.forEach(
+ (device, count) -> {
+ if (count >= 2) {
+ rolledDevices.add(device);
+ }
+ });
+ }
+ assertEquals(
+ "Each device's unflushed tail must span sealed segments within one WAL node",
+ new HashSet<>(Arrays.asList(DEVICES)),
+ rolledDevices);
+ }
+
+ private static void writeRecoveryTable(int phase) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ if (phase == 0) {
+ statement.execute("CREATE DATABASE wal_recovery_table");
+ statement.execute("USE wal_recovery_table");
+ statement.execute(
+ "CREATE TABLE readings(site STRING TAG, device STRING TAG, n INT64 FIELD, t STRING FIELD)");
+ } else {
+ statement.execute("USE wal_recovery_table");
+ }
+ // Same device tag at different sites must stay distinct through WAL replay.
+ for (int site = 0; site < 2; site++) {
+ for (int row = 0; row < 8; row++) {
+ long time = row < 4 ? row : 1000 + row;
+ if (phase != 0 && row == 0) {
+ time = 2000 + phase;
+ }
+ statement.execute(
+ "INSERT INTO readings(time,site,device,n,t) VALUES("
+ + time
+ + ",'s"
+ + site
+ + "','d',"
+ + (phase * 1000 + site * 100 + row)
+ + ","
+ + (row % 3 == 0 ? "null" : "'p" + phase + "-s" + site + "-r" + row + "'")
+ + ")");
+ }
+ }
+ }
+ }
+
+ private static void assertRecoveryTable(int phase) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE wal_recovery_table");
+ try (ResultSet result =
+ statement.executeQuery(
+ "SELECT time,site,device,n,t FROM readings ORDER BY site,time,device")) {
+ for (int site = 0; site < 2; site++) {
+ for (int row = 0; row < 8 + phase; row++) {
+ assertTrue(result.next());
+ boolean appended = row >= 8;
+ int writtenPhase = row == 0 ? 0 : appended ? row - 7 : phase;
+ int originalRow = appended ? 0 : row;
+ long time = appended ? 2000 + writtenPhase : row < 4 ? row : 1000 + row;
+ assertEquals(time, result.getLong(1));
+ assertEquals("s" + site, result.getString(2));
+ assertEquals("d", result.getString(3));
+ assertEquals(writtenPhase * 1000 + site * 100 + originalRow, result.getLong(4));
+ assertFalse(result.wasNull());
+ assertEquals(
+ originalRow % 3 == 0 ? null : "p" + writtenPhase + "-s" + site + "-r" + originalRow,
+ result.getString(5));
+ }
+ }
+ assertFalse("Unexpected TAG combination or extra recovery row", result.next());
+ }
+ }
+ }
+
@Test
public void testTableTabletSplitAcrossDevicesAndTimePartitions() throws Exception {
String database = "write_memory_table";
@@ -317,8 +475,9 @@ private static Tablet tablet(String device, int start, int count, String overrid
for (int row = 0; row < count; row++) {
int time = start + row;
tablet.addTimestamp(row, time);
- tablet.addValue("n", row, (long) time);
- tablet.addValue("t", row, override == null ? text(time) : override);
+ tablet.addValue("n", row, time + (DEVICES[1].equals(device) ? 100_000L : 0));
+ String value = override == null ? text(time) : override;
+ tablet.addValue("t", row, value == null ? null : device + ":" + value);
}
return tablet;
}
@@ -333,13 +492,13 @@ private void overwrite(ISession session, long time, String value) throws Excepti
time,
Collections.singletonList("t"),
Collections.singletonList(TSDataType.TEXT),
- Collections.singletonList(value));
+ Collections.singletonList(DEVICES[0] + ":" + value));
session.insertAlignedRecord(
DEVICES[1],
time,
Collections.singletonList("t"),
Collections.singletonList(TSDataType.TEXT),
- Collections.singletonList(value));
+ Collections.singletonList(DEVICES[1] + ":" + value));
expected.put(time, value);
}
@@ -351,9 +510,11 @@ private void assertData() throws Exception {
for (Map.Entry row : expected.entrySet()) {
assertTrue("Missing timestamp " + row.getKey() + " in " + device, result.next());
assertEquals((long) row.getKey(), result.getLong(1));
- assertEquals((long) row.getKey(), result.getLong(2));
+ assertEquals(
+ row.getKey() + (DEVICES[1].equals(device) ? 100_000L : 0), result.getLong(2));
assertFalse(result.wasNull());
- assertEquals(row.getValue(), result.getString(3));
+ assertEquals(
+ row.getValue() == null ? null : device + ":" + row.getValue(), result.getString(3));
}
assertFalse("Unexpected extra row in " + device, result.next());
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaLogRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaLogRecoveryIT.java
new file mode 100644
index 0000000000000..f233863acaebc
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaLogRecoveryIT.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.schema;
+
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.util.AbstractSchemaIT;
+
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Parameterized;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.*;
+import static org.junit.Assert.*;
+
+/** P1-3: schema records, tag indexes and aliases after a forced restart. */
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBSchemaLogRecoveryIT extends AbstractSchemaIT {
+ @Parameterized.Parameters(name = "SchemaEngineMode={0}")
+ public static Iterable data() {
+ return Collections.singletonList(SchemaTestMode.Memory);
+ }
+
+ public IoTDBSchemaLogRecoveryIT(SchemaTestMode mode) {
+ super(mode);
+ }
+
+ @Parameterized.BeforeParam
+ public static void before() throws Exception {
+ setUpEnvironment();
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+ .setSchemaReplicationFactor(1)
+ .setDataReplicationFactor(1);
+ EnvFactory.getEnv().initClusterEnvironment(1, 1);
+ }
+
+ @Parameterized.AfterParam
+ public static void after() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ tearDownEnvironment();
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ if (!node().isAlive()) {
+ node().start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ }
+ try (Connection cleanup = connection(node(), BaseEnv.TREE_SQL_DIALECT)) {
+ List databases = new ArrayList<>();
+ try (Statement statement = cleanup.createStatement();
+ ResultSet result = statement.executeQuery("SHOW DATABASES")) {
+ while (result.next()) {
+ if (result.getString(1).startsWith("root.p1_")) {
+ databases.add(result.getString(1));
+ }
+ }
+ }
+ for (String database : databases) {
+ execute(cleanup, "DELETE DATABASE " + database);
+ }
+ List templates = new ArrayList<>();
+ try (Statement statement = cleanup.createStatement();
+ ResultSet result = statement.executeQuery("SHOW DEVICE TEMPLATES")) {
+ while (result.next()) {
+ templates.add(result.getString(1));
+ }
+ }
+ for (String template : templates) {
+ execute(cleanup, "DROP DEVICE TEMPLATE " + template);
+ }
+ }
+ }
+
+ @Test
+ public void testTagAndAliasIndexesAfterSchemaLogRecovery() throws Exception {
+ String database = "root.p1_tags";
+ String device = database + ".d";
+ try (Connection control = connection(node(), BaseEnv.TREE_SQL_DIALECT)) {
+ execute(
+ control,
+ "CREATE DATABASE " + database,
+ "CREATE TIMESERIES "
+ + device
+ + ".s(old_alias) INT64 ENCODING=PLAIN COMPRESSION=SNAPPY TAGS(old_key='old',removed='gone') ATTRIBUTES(owner='alice',temporary='gone')",
+ "CREATE TIMESERIES "
+ + device
+ + ".reused(dead_alias) TEXT ENCODING=PLAIN COMPRESSION=SNAPPY TAGS(site='deleted')",
+ "CREATE TIMESERIES "
+ + database
+ + ".control.s INT64 ENCODING=PLAIN COMPRESSION=SNAPPY TAGS(site='control') ATTRIBUTES(owner='keep')",
+ "INSERT INTO " + device + "(time,s,reused) VALUES(1,10,'old_data')",
+ "INSERT INTO " + database + ".control(time,s) VALUES(1,99)",
+ "ALTER TIMESERIES " + device + ".s RENAME old_key TO site",
+ "ALTER TIMESERIES " + device + ".s SET site='middle',owner='bob'",
+ "ALTER TIMESERIES " + device + ".s DROP removed,temporary",
+ "ALTER TIMESERIES "
+ + device
+ + ".s UPSERT ALIAS=new_alias TAGS(site='new') ATTRIBUTES(owner='carol')",
+ "DELETE TIMESERIES " + device + ".reused",
+ "CREATE TIMESERIES "
+ + device
+ + ".reused(fresh_alias) DOUBLE ENCODING=PLAIN COMPRESSION=SNAPPY TAGS(site='rebuilt') ATTRIBUTES(owner='dave')",
+ "INSERT INTO " + device + "(time,reused) VALUES(3,3.5)",
+ "FLUSH");
+ assertTagState(control, database);
+ }
+ // Memory mode with SimpleConsensus rebuilds its schema from mlog on startup. Check the
+ // durable log before killing the only replica, then verify every mutation after recovery.
+ try (Stream files = Files.walk(Paths.get(node().getNodePath()))) {
+ assertTrue(
+ files.anyMatch(
+ path ->
+ path.getFileName().toString().equals("mlog.bin")
+ && path.toString().contains(database)
+ && path.toFile().length() > 0));
+ }
+ node().stopForcibly();
+ node().start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ try (Connection recovered = connection(node(), BaseEnv.TREE_SQL_DIALECT)) {
+ assertTagState(recovered, database);
+ execute(
+ recovered,
+ "ALTER TIMESERIES " + device + ".s SET site='after_recovery'",
+ "INSERT INTO " + device + "(time,new_alias) VALUES(4,40)");
+ assertRows(
+ recovered,
+ "SELECT new_alias FROM " + device,
+ "Time," + device + ".new_alias,",
+ "1,10,",
+ "4,40,");
+ assertRows(
+ recovered,
+ "SHOW TIMESERIES " + database + ".** WHERE TAGS(site)='new'",
+ TIMESERIES_HEADER);
+ assertRows(
+ recovered,
+ "SHOW TIMESERIES " + database + ".** WHERE TAGS(site)='after_recovery'",
+ TIMESERIES_HEADER,
+ device
+ + ".s,new_alias,"
+ + database
+ + ",INT64,PLAIN,SNAPPY,{\"site\":\"after_recovery\"},{\"owner\":\"carol\"},null,null,BASE,");
+ execute(recovered, "DELETE TIMESERIES " + device + ".s");
+ assertRows(
+ recovered,
+ "SHOW TIMESERIES " + database + ".** WHERE TAGS(site)='after_recovery'",
+ TIMESERIES_HEADER);
+ assertRows(
+ recovered,
+ "SELECT s FROM " + database + ".control",
+ "Time," + database + ".control.s,",
+ "1,99,");
+ }
+ }
+
+ private void assertTagState(Connection read, String database) throws Exception {
+ String device = database + ".d";
+ assertRows(
+ read,
+ "SHOW TIMESERIES " + database + ".**",
+ TIMESERIES_HEADER,
+ device
+ + ".s,new_alias,"
+ + database
+ + ",INT64,PLAIN,SNAPPY,{\"site\":\"new\"},{\"owner\":\"carol\"},null,null,BASE,",
+ device
+ + ".reused,fresh_alias,"
+ + database
+ + ",DOUBLE,PLAIN,SNAPPY,{\"site\":\"rebuilt\"},{\"owner\":\"dave\"},null,null,BASE,",
+ database
+ + ".control.s,null,"
+ + database
+ + ",INT64,PLAIN,SNAPPY,{\"site\":\"control\"},{\"owner\":\"keep\"},null,null,BASE,");
+ for (String filter :
+ new String[] {"old_key='old'", "site='middle'", "site='deleted'", "removed='gone'"}) {
+ String[] pair = filter.split("=", 2);
+ assertRows(
+ read,
+ "SHOW TIMESERIES " + database + ".** WHERE TAGS(" + pair[0] + ")=" + pair[1],
+ TIMESERIES_HEADER);
+ }
+ assertRows(
+ read,
+ "SHOW TIMESERIES " + database + ".** WHERE TAGS(site)='new'",
+ TIMESERIES_HEADER,
+ device
+ + ".s,new_alias,"
+ + database
+ + ",INT64,PLAIN,SNAPPY,{\"site\":\"new\"},{\"owner\":\"carol\"},null,null,BASE,");
+ assertRows(read, "COUNT TIMESERIES " + database + ".**", "count(timeseries),", "3,");
+ assertRows(
+ read,
+ "SELECT new_alias,fresh_alias FROM " + device,
+ "Time," + device + ".new_alias," + device + ".fresh_alias,",
+ "1,10,null,",
+ "3,null,3.5,");
+ assertRows(read, "SELECT old_alias,dead_alias FROM " + device, "Time,");
+ assertRows(
+ read,
+ "SELECT s FROM " + database + ".control",
+ "Time," + database + ".control.s,",
+ "1,99,");
+ }
+
+ private static DataNodeWrapper node() {
+ return EnvFactory.getEnv().getDataNodeWrapper(0);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaMetadataRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaMetadataRecoveryIT.java
new file mode 100644
index 0000000000000..6f31f6f757660
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBSchemaMetadataRecoveryIT.java
@@ -0,0 +1,504 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.schema;
+
+import org.apache.iotdb.db.it.utils.MetadataFaultTestUtils;
+import org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.Fault;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.fault.metadata.MetadataTestHooks;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.util.AbstractSchemaIT;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Parameterized;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.assertAllRows;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.assertFenced;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.awaitAllRows;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.connect;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.execute;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.interruptProcedure;
+import static org.apache.iotdb.itbase.env.BaseEnv.TREE_SQL_DIALECT;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tree-model metadata fault windows. AbstractSchemaIT runs this class once for Memory and once for
+ * PBTree, while the shared cluster topology provides three ConfigNodes and replicated
+ * SchemaRegions.
+ */
+@Category(ClusterIT.class)
+public class IoTDBSchemaMetadataRecoveryIT extends AbstractSchemaIT {
+
+ private static final String DB_LEASE = "root.p0_tree_lease";
+ private static final String DB_DELETE = "root.p0_tree_delete";
+ private static final String DB_LIFECYCLE = "root.p0_tree_lifecycle";
+ private static final String DB_SNAPSHOT = "root.p0_tree_snapshot";
+ private static final String TIMESERIES_HEADER =
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,";
+
+ public IoTDBSchemaMetadataRecoveryIT(final SchemaTestMode schemaTestMode) {
+ super(schemaTestMode);
+ }
+
+ @Parameterized.BeforeParam
+ public static void before() throws Exception {
+ if (setUpEnvironment() == SchemaTestMode.PBTree) {
+ // Replicated recovery needs enough memory for multiple pinned Region roots.
+ // Snapshot installation is forced explicitly, independently of cache eviction.
+ allocateMemoryForSchemaRegion(1024 * 1024);
+ }
+ MetadataFaultTestUtils.configureCluster();
+ EnvFactory.getEnv().initClusterEnvironment(3, 3);
+ }
+
+ @Parameterized.AfterParam
+ public static void after() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ tearDownEnvironment();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // Delete databases before dropping their bound template. Propagate cleanup failures instead
+ // of hiding an unfinished procedure behind AbstractSchemaIT.clearSchema's best-effort delete.
+ try (Connection connection = connect(0, TREE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ final List databases = new ArrayList<>();
+ try (ResultSet result = statement.executeQuery("show databases root.p0_tree_*")) {
+ while (result.next()) {
+ databases.add(result.getString("Database"));
+ }
+ }
+ for (String database : databases) {
+ statement.execute("delete database " + database);
+ }
+ boolean hasTemplate = false;
+ try (ResultSet result = statement.executeQuery("show device templates")) {
+ while (result.next()) {
+ hasTemplate |= "p0_tree_template".equals(result.getString(1));
+ }
+ }
+ if (hasTemplate) {
+ statement.execute("drop device template p0_tree_template");
+ }
+ }
+ }
+
+ /** P0-1: a live tree DataNode is fenced while a delete invalidates its warm schema cache. */
+ @Test
+ public void testTreeLiveNodeFencesOnDeleteAndReloadsBeforeWrites() throws Exception {
+ final var victim = EnvFactory.getEnv().getDataNodeWrapper(2);
+ try (Connection connection = connect(2, TREE_SQL_DIALECT);
+ Statement victimStatement = connection.createStatement()) {
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "create database " + DB_LEASE,
+ "create timeseries "
+ + DB_LEASE
+ + ".d.s with datatype=INT32, encoding=PLAIN "
+ + "tags('kind'='protected') attributes('owner'='old')",
+ "insert into " + DB_LEASE + ".d(time,s) values(1,10)");
+ // Wait for asynchronous IoTConsensus replication and warm the victim's schema cache before
+ // injecting metadata faults.
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_LEASE + ".d",
+ "Time," + DB_LEASE + ".d.s,",
+ "1,10,");
+ try (Fault isolation = new Fault(victim, MetadataTestHooks.CONFIG_NODE_RPC);
+ Fault heartbeat = new Fault(victim, "heartbeat-rejected").arm("observe");
+ Fault invalidation = new Fault(victim, "cache-broadcast-rejected").arm("observe");
+ Fault recovery = new Fault(victim, MetadataTestHooks.LEASE_RECOVERY).arm("fail")) {
+ isolation.arm("fail");
+ heartbeat.awaitHit();
+ execute(0, TREE_SQL_DIALECT, "delete timeseries " + DB_LEASE + ".d.s");
+ invalidation.awaitHit();
+ await()
+ .atMost(MetadataFaultTestUtils.TIMEOUT_SECONDS, java.util.concurrent.TimeUnit.SECONDS)
+ .untilAsserted(() -> assertFenced(victimStatement, "select s from " + DB_LEASE + ".d"));
+ assertTrue(victim.isAlive());
+ isolation.release();
+ recovery.awaitHit();
+ assertFenced(victimStatement, "insert into " + DB_LEASE + ".d(time,s) values(2,20)");
+ recovery.release();
+ awaitAllRows(
+ TREE_SQL_DIALECT, "count timeseries " + DB_LEASE + ".**", "count(timeseries),", "0,");
+ execute(
+ 2,
+ TREE_SQL_DIALECT,
+ "create timeseries " + DB_LEASE + ".d.s with datatype=DOUBLE, encoding=PLAIN",
+ "insert into " + DB_LEASE + ".d(time,s) values(3,2.5)");
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_LEASE + ".d",
+ "Time," + DB_LEASE + ".d.s,",
+ "3,2.5,");
+ }
+ }
+ }
+
+ /**
+ * P0-3: a real cross-SchemaRegion blacklist rolls back, then a later state completes deletion.
+ */
+ @Test
+ public void testCrossRegionDeleteTimeseriesRollsBackAndContinues() throws Exception {
+ final List devices = new ArrayList<>();
+ final List setup = new ArrayList<>();
+ final List metadata = new ArrayList<>();
+ final List data = new ArrayList<>();
+ setup.add("create database " + DB_DELETE);
+ setup.add(
+ "create aligned timeseries "
+ + DB_DELETE
+ + ".protected(s1 INT32 encoding=PLAIN compressor=SNAPPY, "
+ + "s2 DOUBLE encoding=PLAIN compressor=SNAPPY)");
+ setup.add("insert into " + DB_DELETE + ".protected(time,s1,s2) aligned values(1,77,7.7)");
+ for (int i = 0; i < 24; i++) {
+ final String device = DB_DELETE + ".d" + i;
+ final String properties = " tags('group'='delete') attributes('old'='yes')";
+ setup.add(
+ i % 2 == 0
+ ? "create timeseries "
+ + device
+ + ".s(alias"
+ + i
+ + ") with datatype=INT32, encoding=PLAIN, compressor=SNAPPY"
+ + properties
+ : "create aligned timeseries "
+ + device
+ + "(s(alias"
+ + i
+ + ") INT32 encoding=PLAIN compressor=SNAPPY"
+ + properties
+ + ")");
+ setup.add(
+ "insert into "
+ + device
+ + "(time,s)"
+ + (i % 2 == 0 ? "" : " aligned")
+ + " values(1,"
+ + i
+ + ")");
+ devices.add(IDeviceID.Factory.DEFAULT_FACTORY.create(device));
+ metadata.add(
+ device
+ + ".s,alias"
+ + i
+ + ","
+ + DB_DELETE
+ + ",INT32,PLAIN,SNAPPY,{\"group\":\"delete\"},{\"old\":\"yes\"},null,null,BASE,");
+ data.add("1," + device + "," + i + ",");
+ }
+ execute(0, TREE_SQL_DIALECT, setup.toArray(new String[0]));
+ MetadataFaultTestUtils.assertSpansRegions(DB_DELETE, devices);
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_DELETE + ".d*.s",
+ TIMESERIES_HEADER,
+ metadata.toArray(new String[0]));
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_DELETE + ".d* align by device",
+ "Time,Device,s,",
+ data.toArray(new String[0]));
+ assertAllRows(
+ TREE_SQL_DIALECT, "count timeseries " + DB_DELETE + ".d*.s", "count(timeseries),", "24,");
+ final String delete = "delete timeseries " + DB_DELETE + ".d*.s";
+ interruptProcedure(
+ "DeleteTimeSeriesProcedure",
+ "CLEAN_DATANODE_SCHEMA_CACHE",
+ true,
+ () -> execute(0, TREE_SQL_DIALECT, delete));
+ awaitAllRows(
+ TREE_SQL_DIALECT, "count timeseries " + DB_DELETE + ".d*.s", "count(timeseries),", "24,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_DELETE + ".d*.s",
+ TIMESERIES_HEADER,
+ metadata.toArray(new String[0]));
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_DELETE + ".d* align by device",
+ "Time,Device,s,",
+ data.toArray(new String[0]));
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s1,s2 from " + DB_DELETE + ".protected",
+ "Time," + DB_DELETE + ".protected.s1," + DB_DELETE + ".protected.s2,",
+ "1,77,7.7,");
+ interruptProcedure(
+ "DeleteTimeSeriesProcedure",
+ "DELETE_TIMESERIES_SCHEMA",
+ false,
+ () -> execute(0, TREE_SQL_DIALECT, delete));
+ awaitAllRows(
+ TREE_SQL_DIALECT, "count timeseries " + DB_DELETE + ".d*.s", "count(timeseries),", "0,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show devices " + DB_DELETE + ".protected",
+ "Device,IsAligned,Template,TTL(ms),",
+ DB_DELETE + ".protected,true,null,INF,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s1,s2 from " + DB_DELETE + ".protected",
+ "Time," + DB_DELETE + ".protected.s1," + DB_DELETE + ".protected.s2,",
+ "1,77,7.7,");
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "create timeseries "
+ + DB_DELETE
+ + ".d0.s with datatype=DOUBLE, encoding=PLAIN tags('group'='new')",
+ "insert into " + DB_DELETE + ".d0(time,s) values(2,100.5)");
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_DELETE + ".d0",
+ "Time," + DB_DELETE + ".d0.s,",
+ "2,100.5,");
+ }
+
+ /**
+ * P0-4: an old stopped node returns after a tree sequence is deleted and recreated with a new
+ * type.
+ */
+ @Test
+ public void testTreeDeleteRecreateDoesNotResurrectOldSchema() throws Exception {
+ final var victim = EnvFactory.getEnv().getDataNodeWrapper(1);
+ try {
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "create database " + DB_LIFECYCLE,
+ "create timeseries "
+ + DB_LIFECYCLE
+ + ".d.s with datatype=INT32, encoding=PLAIN, compressor=SNAPPY "
+ + "tags('generation'='old') attributes('owner'='old')",
+ "insert into " + DB_LIFECYCLE + ".d(time,s) values(1,10)");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_LIFECYCLE + ".d",
+ "Time," + DB_LIFECYCLE + ".d.s,",
+ "1,10,");
+ victim.stopForcibly();
+ assertFalse(victim.isAlive());
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "delete timeseries " + DB_LIFECYCLE + ".d.s",
+ "create timeseries "
+ + DB_LIFECYCLE
+ + ".d.s with datatype=DOUBLE, encoding=PLAIN, compressor=SNAPPY "
+ + "tags('generation'='new') attributes('owner'='new')",
+ "insert into " + DB_LIFECYCLE + ".d(time,s) values(2,2.5)");
+ victim.start();
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_LIFECYCLE + ".d.s",
+ TIMESERIES_HEADER,
+ DB_LIFECYCLE
+ + ".d.s,null,"
+ + DB_LIFECYCLE
+ + ",DOUBLE,PLAIN,SNAPPY,{\"generation\":\"new\"},{\"owner\":\"new\"},null,null,BASE,");
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_LIFECYCLE + ".d",
+ "Time," + DB_LIFECYCLE + ".d.s,",
+ "2,2.5,");
+ } finally {
+ if (!victim.isAlive()) {
+ victim.start();
+ }
+ }
+ }
+
+ /** P0-6: a stopped replica installs a tree snapshot and then replays a metadata log tail. */
+ @Test
+ public void testTreeSchemaSnapshotRestoresRichMetadataAndTail() throws Exception {
+ final String template = "p0_tree_template";
+ MetadataFaultTestUtils.createSingleSchemaRegionDatabase(DB_SNAPSHOT, TREE_SQL_DIALECT);
+ // Extend the ConfigNode template catalog with all peers online. Tail writes use that
+ // definition.
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "create timeseries "
+ + DB_SNAPSHOT
+ + ".n.s1(alias1) with datatype=INT32, encoding=PLAIN, compressor=SNAPPY "
+ + "tags('kind'='base') attributes('owner'='base')",
+ "create aligned timeseries "
+ + DB_SNAPSHOT
+ + ".a(s1 INT32 encoding=PLAIN compressor=SNAPPY, s2 DOUBLE encoding=PLAIN compressor=SNAPPY)",
+ "insert into " + DB_SNAPSHOT + ".n(time,s1) values(1,10)",
+ "insert into " + DB_SNAPSHOT + ".a(time,s1,s2) aligned values(1,20,2.0)",
+ "create timeseries " + DB_SNAPSHOT + ".gone.s with datatype=INT32, encoding=PLAIN",
+ "delete timeseries " + DB_SNAPSHOT + ".gone.s",
+ "create device template " + template + " (ts INT32 ENCODING=PLAIN COMPRESSOR=SNAPPY)",
+ "set device template " + template + " to " + DB_SNAPSHOT + ".templ",
+ "create timeseries using device template on " + DB_SNAPSHOT + ".templ",
+ "insert into " + DB_SNAPSHOT + ".templ(time,ts) values(1,31)",
+ "alter device template " + template + " add (ext DOUBLE ENCODING=RLE COMPRESSOR=SNAPPY)",
+ "insert into " + DB_SNAPSHOT + ".templ(time,ext) values(1,3.1)",
+ "alter device template "
+ + template
+ + " add (tail BOOLEAN ENCODING=PLAIN COMPRESSOR=SNAPPY)");
+ final var mapping =
+ MetadataFaultTestUtils.partitions(
+ DB_SNAPSHOT,
+ Arrays.asList(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(DB_SNAPSHOT + ".n"),
+ IDeviceID.Factory.DEFAULT_FACTORY.create(DB_SNAPSHOT + ".a"),
+ IDeviceID.Factory.DEFAULT_FACTORY.create(DB_SNAPSHOT + ".templ")));
+ assertEquals(1L, mapping.values().stream().distinct().count());
+ final int regionId = mapping.values().iterator().next().getId();
+ assertTrue(
+ MetadataFaultTestUtils.schemaRegions(DB_SNAPSHOT).stream()
+ .allMatch(info -> info.getConsensusGroupId().getId() == regionId));
+ MetadataFaultTestUtils.installSchemaSnapshot(
+ DB_SNAPSHOT,
+ regionId,
+ () -> {
+ for (int i = 0; i < 32; i++) {
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "create timeseries "
+ + DB_SNAPSHOT
+ + ".n.extra"
+ + i
+ + " with datatype=INT32, encoding=PLAIN, compressor=SNAPPY tags('batch'='"
+ + i
+ + "')");
+ }
+ },
+ () ->
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "alter timeseries "
+ + DB_SNAPSHOT
+ + ".n.s1 upsert alias=tail_alias tags('kind'='tail')",
+ "insert into " + DB_SNAPSHOT + ".templ(time,tail) values(2,true)",
+ "delete timeseries " + DB_SNAPSHOT + ".n.extra0",
+ "create timeseries " + DB_SNAPSHOT + ".tail.s INT64"),
+ () -> assertTreeSnapshotState(template));
+ execute(
+ 0,
+ TREE_SQL_DIALECT,
+ "alter timeseries " + DB_SNAPSHOT + ".n.s1 upsert attributes('owner'='after')",
+ "insert into " + DB_SNAPSHOT + ".tail(time,s) values(2,99)",
+ "insert into " + DB_SNAPSHOT + ".templ(time,tail) values(3,false)");
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "select s from " + DB_SNAPSHOT + ".tail",
+ "Time," + DB_SNAPSHOT + ".tail.s,",
+ "2,99,");
+ }
+
+ private static void assertTreeSnapshotState(final String template) throws Exception {
+ awaitAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_SNAPSHOT + ".n.s1",
+ TIMESERIES_HEADER,
+ DB_SNAPSHOT
+ + ".n.s1,tail_alias,"
+ + DB_SNAPSHOT
+ + ",INT32,PLAIN,SNAPPY,{\"kind\":\"tail\"},{\"owner\":\"base\"},null,null,BASE,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select tail_alias from " + DB_SNAPSHOT + ".n",
+ "Time," + DB_SNAPSHOT + ".n.tail_alias,",
+ "1,10,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_SNAPSHOT + ".** where TAGS(kind)='base'",
+ TIMESERIES_HEADER);
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show timeseries " + DB_SNAPSHOT + ".** where TAGS(kind)='tail'",
+ TIMESERIES_HEADER,
+ DB_SNAPSHOT
+ + ".n.s1,tail_alias,"
+ + DB_SNAPSHOT
+ + ",INT32,PLAIN,SNAPPY,{\"kind\":\"tail\"},{\"owner\":\"base\"},null,null,BASE,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "count timeseries " + DB_SNAPSHOT + ".gone.**",
+ "count(timeseries),",
+ "0,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "count timeseries " + DB_SNAPSHOT + ".n.extra0",
+ "count(timeseries),",
+ "0,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "count timeseries " + DB_SNAPSHOT + ".n.extra1",
+ "count(timeseries),",
+ "1,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show paths set device template " + template,
+ "Paths,",
+ DB_SNAPSHOT + ".templ,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show paths using device template " + template,
+ "Paths,",
+ DB_SNAPSHOT + ".templ,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select s1,s2 from " + DB_SNAPSHOT + ".a",
+ "Time," + DB_SNAPSHOT + ".a.s1," + DB_SNAPSHOT + ".a.s2,",
+ "1,20,2.0,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "show devices " + DB_SNAPSHOT + ".a",
+ "Device,IsAligned,Template,TTL(ms),",
+ DB_SNAPSHOT + ".a,true,null,INF,");
+ assertAllRows(
+ TREE_SQL_DIALECT,
+ "select ts,ext,tail from " + DB_SNAPSHOT + ".templ",
+ "Time,"
+ + DB_SNAPSHOT
+ + ".templ.ts,"
+ + DB_SNAPSHOT
+ + ".templ.ext,"
+ + DB_SNAPSHOT
+ + ".templ.tail,",
+ "1,31,3.1,null,",
+ "2,null,null,true,");
+ assertAllRows(
+ TREE_SQL_DIALECT, "count timeseries " + DB_SNAPSHOT + ".**", "count(timeseries),", "38,");
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/MetadataFaultTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/MetadataFaultTestUtils.java
new file mode 100644
index 0000000000000..6e12572a231ac
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/MetadataFaultTestUtils.java
@@ -0,0 +1,775 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.utils;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
+import org.apache.iotdb.commons.client.ClientPoolFactory;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor;
+import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters;
+import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp;
+import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.consensus.iot.thrift.IoTConsensusIService;
+import org.apache.iotdb.consensus.iot.thrift.TWaitSyncLogCompleteReq;
+import org.apache.iotdb.consensus.iot.thrift.TWaitSyncLogCompleteRes;
+import org.apache.iotdb.consensus.ratis.utils.Utils;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppBaseConfig;
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
+import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper;
+import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.fault.metadata.MetadataTestAgent;
+import org.apache.iotdb.it.fault.metadata.MetadataTestHooks;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq;
+import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory;
+import org.apache.iotdb.rpc.TConfigurationConst;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.tsfile.file.metadata.IDeviceID;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+/** Shared, node-specific fault controls and observations for the metadata P0 recovery ITs. */
+public final class MetadataFaultTestUtils {
+ public static final long FENCE_MS = 8_000;
+ public static final long TIMEOUT_SECONDS = 120;
+
+ private MetadataFaultTestUtils() {}
+
+ public static void configureCluster() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaReplicationFactor(3)
+ .setDataReplicationFactor(2)
+ .setSchemaRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultSchemaRegionGroupNumPerDatabase(2)
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1)
+ .setAutoCreateSchemaEnabled(false);
+ EnvFactory.getEnv().getConfig().getConfigNodeConfig().setMetadataLeaseFenceMs(FENCE_MS);
+ final Properties properties = new Properties();
+ properties.setProperty("schema_region_ratis_snapshot_trigger_threshold", "12");
+ properties.setProperty("schema_region_ratis_log_segment_size_max_in_byte", "1024");
+ properties.setProperty("schema_region_ratis_preserve_logs_num_when_purge", "0");
+ properties.setProperty("schema_region_ratis_periodic_snapshot_interval", "0");
+ ((MppBaseConfig) EnvFactory.getEnv().getConfig().getConfigNodeConfig())
+ .updateProperties(properties);
+ final Path agent = MetadataTestAgent.createJar(Paths.get("target", "metadata-test-agent.jar"));
+ final String option = "-javaagent:" + agent.toAbsolutePath();
+ ((MppJVMConfig) EnvFactory.getEnv().getConfig().getConfigNodeJVMConfig())
+ .setJvmOptions(Collections.singletonList(option));
+ ((MppJVMConfig) EnvFactory.getEnv().getConfig().getDataNodeJVMConfig())
+ .setJvmOptions(Collections.singletonList(option));
+ }
+
+ public static void createSingleSchemaRegionDatabase(final String database, final String dialect)
+ throws SQLException {
+ final boolean treeModel = BaseEnv.TREE_SQL_DIALECT.equals(dialect);
+ final String schemaRegionConfig =
+ treeModel
+ ? "set configuration \"default_schema_region_group_num_per_database\"=\"%s\""
+ : "set configuration default_schema_region_group_num_per_database='%s'";
+ execute(0, dialect, String.format(schemaRegionConfig, "1"));
+ try {
+ execute(
+ 0,
+ dialect,
+ treeModel
+ ? "create database " + database + " with MAX_SCHEMA_REGION_GROUP_NUM=1"
+ : "create database " + database + " with(max_schema_region_group_num=1)");
+ } finally {
+ execute(0, dialect, String.format(schemaRegionConfig, "2"));
+ }
+ }
+
+ /** A physical JDBC connection has exactly one endpoint and cannot silently redirect to a peer. */
+ public static Connection connect(final int index, final String dialect) throws SQLException {
+ final DataNodeWrapper node = EnvFactory.getEnv().getDataNodeWrapper(index);
+ final Properties properties =
+ BaseEnv.constructProperties(
+ SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, dialect);
+ properties.setProperty(Config.NETWORK_TIMEOUT, "180000");
+ return DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + node.getIpAndPortString(), properties);
+ }
+
+ public static void execute(final int index, final String dialect, final String... sqls)
+ throws SQLException {
+ try (Connection connection = connect(index, dialect);
+ Statement statement = connection.createStatement()) {
+ for (String sql : sqls) {
+ statement.execute(sql);
+ }
+ }
+ }
+
+ public static void assertRows(
+ final int index,
+ final String dialect,
+ final String sql,
+ final String header,
+ final String... rows)
+ throws SQLException {
+ try (Connection connection = connect(index, dialect);
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery(sql)) {
+ TestUtils.assertResultSetEqualUnordered(result, header, Arrays.asList(rows));
+ }
+ }
+
+ public static void assertAllRows(
+ final String dialect, final String sql, final String header, final String... rows)
+ throws SQLException {
+ // Check physical connections individually; do not read a cross-node wrapper into a Set.
+ for (int index = 0; index < EnvFactory.getEnv().getDataNodeWrapperList().size(); index++) {
+ assertRows(index, dialect, sql, header, rows);
+ }
+ }
+
+ public static void awaitAllRows(
+ final String dialect, final String sql, final String header, final String... rows) {
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreException(SQLException.class)
+ .untilAsserted(() -> assertAllRows(dialect, sql, header, rows));
+ }
+
+ public static void assertFenced(final Statement statement, final String sql) throws SQLException {
+ try {
+ statement.execute(sql);
+ org.junit.Assert.fail("A fenced endpoint accepted: " + sql);
+ } catch (SQLException e) {
+ assertTrue(
+ e.toString(),
+ e.getErrorCode() == TSStatusCode.METADATA_LEASE_FENCED.getStatusCode()
+ || e.getErrorCode()
+ == TSStatusCode.METADATA_LEASE_FENCED_RETRY_REQUIRED.getStatusCode());
+ }
+ }
+
+ public static List schemaRegions(final String database) throws Exception {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ return client
+ .showRegion(new TShowRegionReq().setIsTableModel(!database.startsWith("root.")))
+ .getRegionInfoList()
+ .stream()
+ .filter(
+ info ->
+ info.getConsensusGroupId().getType() == TConsensusGroupType.SchemaRegion
+ && info.getDatabase().equals(database))
+ .collect(Collectors.toList());
+ }
+ }
+
+ /** Wait for acknowledged fixture writes to reach every actual IoTConsensus data replica. */
+ public static void awaitDataReplication(final String database) throws Exception {
+ final List replicas;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ replicas =
+ client.showRegion(new TShowRegionReq().setIsTableModel(true)).getRegionInfoList().stream()
+ .filter(
+ info ->
+ info.getConsensusGroupId().getType() == TConsensusGroupType.DataRegion
+ && info.getDatabase().equals(database))
+ .collect(Collectors.toList());
+ }
+ assertFalse("No actual data replicas for " + database, replicas.isEmpty());
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreException(TException.class)
+ .untilAsserted(
+ () -> {
+ long localWrites = 0;
+ for (TRegionInfo replica : replicas) {
+ final DataNodeWrapper node =
+ EnvFactory.getEnv()
+ .dataNodeIdToWrapper(replica.getDataNodeId())
+ .orElseThrow(AssertionError::new);
+ try (TTransport transport =
+ DeepCopyRpcTransportFactory.INSTANCE.getTransport(
+ new TSocket(
+ TConfigurationConst.defaultTConfiguration,
+ node.getIp(),
+ node.getDataRegionConsensusPort(),
+ 2_000))) {
+ transport.open();
+ final TWaitSyncLogCompleteRes progress =
+ new IoTConsensusIService.Client(new TBinaryProtocol(transport))
+ .waitSyncLogComplete(
+ new TWaitSyncLogCompleteReq(replica.getConsensusGroupId()));
+ assertTrue(progress.toString(), progress.isComplete());
+ localWrites += progress.getSearchIndex();
+ }
+ }
+ assertTrue("No fixture writes observed by data consensus", localWrites > 0);
+ });
+ }
+
+ public static Map partitions(
+ final String database, final List devices) throws Exception {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final TClusterParameters parameters = client.showVariables().getClusterParameters();
+ final SeriesPartitionExecutor executor =
+ SeriesPartitionExecutor.getSeriesPartitionExecutor(
+ parameters.getSeriesPartitionExecutorClass(), parameters.getSeriesPartitionSlotNum());
+ final List slots =
+ devices.stream().map(executor::getSeriesPartitionSlot).collect(Collectors.toList());
+ final TSchemaPartitionTableResp response =
+ client.getSchemaPartitionTableWithSlots(Collections.singletonMap(database, slots));
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode());
+ assertTrue(
+ "No actual schema partition for " + database,
+ response.getSchemaPartitionTable().containsKey(database));
+ return response.getSchemaPartitionTable().get(database);
+ }
+ }
+
+ public static void assertSpansRegions(final String database, final List devices)
+ throws Exception {
+ assertTrue(
+ schemaRegions(database).stream().map(TRegionInfo::getConsensusGroupId).distinct().count()
+ >= 2);
+ assertTrue(
+ "The affected devices did not span SchemaRegions",
+ partitions(database, devices).values().stream().distinct().count() >= 2);
+ }
+
+ public static DataNodeWrapper schemaLeader(final String database) throws Exception {
+ final TRegionInfo leader =
+ schemaRegions(database).stream()
+ .filter(info -> "Leader".equals(info.getRoleType()))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ return EnvFactory.getEnv()
+ .dataNodeIdToWrapper(leader.getDataNodeId())
+ .orElseThrow(AssertionError::new);
+ }
+
+ public static DataNodeWrapper schemaLeader(final String database, final int regionId)
+ throws Exception {
+ final TRegionInfo leader =
+ schemaRegions(database).stream()
+ .filter(
+ info ->
+ info.getConsensusGroupId().getId() == regionId
+ && "Leader".equals(info.getRoleType()))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ return EnvFactory.getEnv()
+ .dataNodeIdToWrapper(leader.getDataNodeId())
+ .orElseThrow(AssertionError::new);
+ }
+
+ public static void transferSchemaLeader(
+ final String database, final int regionId, final DataNodeWrapper target) throws Exception {
+ final TDataNodeLocation location;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ location =
+ client.getDataNodeConfiguration(-1).getDataNodeConfigurationMap().values().stream()
+ .map(configuration -> configuration.getLocation())
+ .filter(candidate -> candidate.getClientRpcEndPoint().getPort() == target.getPort())
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ }
+ final IClientManager manager =
+ new IClientManager.Factory()
+ .createClientManager(
+ new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+ try {
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreExceptionsMatching(e -> e instanceof IOException || e instanceof TException)
+ .untilAsserted(
+ () -> {
+ final DataNodeWrapper current = schemaLeader(database, regionId);
+ if (current != target) {
+ // The RPC is handled by the intended new leader, as in RouteBalancer.
+ try (SyncDataNodeInternalServiceClient client =
+ manager.borrowClient(location.getInternalEndPoint())) {
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ client
+ .changeRegionLeader(
+ new TRegionLeaderChangeReq(
+ new TConsensusGroupId(
+ TConsensusGroupType.SchemaRegion, regionId),
+ location))
+ .getStatus()
+ .getCode());
+ }
+ }
+ assertEquals(target, schemaLeader(database, regionId));
+ });
+ } finally {
+ manager.close();
+ }
+ }
+
+ /**
+ * P0-6: prove installation of a newer snapshot on a previously stopped follower, then lead on it.
+ */
+ public static void installSchemaSnapshot(
+ final String database,
+ final int regionId,
+ final SqlOperation triggerSnapshot,
+ final SqlOperation tailChanges,
+ final SqlOperation assertRecoveredState)
+ throws Exception {
+ final DataNodeWrapper leader = schemaLeader(database, regionId);
+ // Both manifests use physical JDBC connections to node 0, so keep that coordinator running.
+ final DataNodeWrapper coordinator = EnvFactory.getEnv().getDataNodeWrapper(0);
+ final DataNodeWrapper follower =
+ schemaRegions(database).stream()
+ .filter(info -> info.getConsensusGroupId().getId() == regionId)
+ .map(
+ info ->
+ EnvFactory.getEnv()
+ .dataNodeIdToWrapper(info.getDataNodeId())
+ .orElseThrow(AssertionError::new))
+ .filter(node -> node != leader && node != coordinator)
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ final String group =
+ Utils.fromConsensusGroupIdToRaftGroupId(new SchemaRegionId(regionId)).getUuid().toString();
+ final List preventNewSnapshots = new ArrayList<>();
+ try (Fault created = new Fault(leader, MetadataTestHooks.SNAPSHOT_CREATED + "." + regionId);
+ Fault installed =
+ new Fault(follower, MetadataTestHooks.SNAPSHOT_LOADED + "." + regionId)
+ .arm("observe")) {
+ follower.stopForcibly();
+ assertFalse(follower.isAlive());
+ final long previousIndex = latestSnapshotIndex(follower, group);
+ created.arm("observe");
+ triggerSnapshot.execute();
+ // More than one snapshot may finish during the writes. An older observed directory may
+ // already have been purged, so select a completed snapshot that still exists.
+ final Path completedSnapshot =
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .until(() -> latestCompletedSnapshot(created, previousIndex), Objects::nonNull);
+ final long snapshotIndex = snapshotIndex(completedSnapshot.getFileName().toString());
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertTrue(
+ "SchemaRegion logs were not purged past the old snapshot",
+ firstRetainedLogIndex(leader, group) > previousIndex));
+ // Keep post-snapshot changes in the Raft log. A later snapshot must not absorb this manifest.
+ for (DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ preventNewSnapshots.add(new Fault(node, "snapshot-create." + regionId).arm("fail"));
+ }
+ tailChanges.execute();
+ follower.start();
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ try (Stream paths = Files.list(installed.directory)) {
+ assertTrue(
+ "No newer SchemaRegion snapshot was installed on the lagging replica",
+ paths.anyMatch(
+ path -> {
+ final String fileName = path.getFileName().toString();
+ final String prefix =
+ "hit." + MetadataTestHooks.SNAPSHOT_LOADED + "." + regionId + ".";
+ return fileName.startsWith(prefix)
+ && snapshotIndex(fileName.substring(prefix.length()))
+ >= snapshotIndex;
+ }));
+ }
+ });
+ assertRecoveredState.execute();
+ // Client connections can otherwise route schema reads to healthy replicas and hide a bad
+ // load.
+ transferSchemaLeader(database, regionId, follower);
+ assertRecoveredState.execute();
+ } finally {
+ try {
+ for (Fault fault : preventNewSnapshots) {
+ fault.close();
+ }
+ } finally {
+ if (!follower.isAlive()) {
+ follower.start();
+ }
+ }
+ }
+ }
+
+ private static Path latestCompletedSnapshot(final Fault created, final long previousIndex)
+ throws IOException {
+ Path latest = null;
+ long latestIndex = previousIndex;
+ try (Stream paths = Files.list(created.directory)) {
+ for (Path evidence : paths.collect(Collectors.toList())) {
+ if (!evidence.getFileName().toString().startsWith("hit." + created.point + ".")
+ || Files.size(evidence) == 0) {
+ continue;
+ }
+ final Path temporary =
+ Paths.get(new String(Files.readAllBytes(evidence), StandardCharsets.UTF_8));
+ final String name = temporary.getFileName().toString().replace(".tmp.", "");
+ final Path completed = temporary.resolveSibling(name);
+ if (Files.isDirectory(completed) && snapshotIndex(name) > latestIndex) {
+ latest = completed;
+ latestIndex = snapshotIndex(name);
+ }
+ }
+ }
+ return latest;
+ }
+
+ private static long latestSnapshotIndex(final DataNodeWrapper node, final String group)
+ throws IOException {
+ try (Stream paths = Files.walk(Paths.get(node.getNodePath()))) {
+ return paths
+ .filter(path -> path.toString().contains(group))
+ .mapToLong(path -> snapshotIndex(path.getFileName().toString()))
+ .max()
+ .orElse(0);
+ }
+ }
+
+ private static long firstRetainedLogIndex(final DataNodeWrapper node, final String group)
+ throws IOException {
+ try (Stream paths = Files.walk(Paths.get(node.getNodePath()))) {
+ return paths
+ .filter(path -> path.toString().contains(group))
+ .map(path -> path.getFileName().toString())
+ .filter(name -> name.matches("log_([0-9]+-[0-9]+|inprogress_[0-9]+)"))
+ .mapToLong(
+ name ->
+ Long.parseLong(
+ name.startsWith("log_inprogress_")
+ ? name.substring("log_inprogress_".length())
+ : name.substring("log_".length(), name.indexOf('-'))))
+ .min()
+ .orElse(0);
+ }
+ }
+
+ private static long snapshotIndex(final String name) {
+ return name.matches("[0-9]+_[0-9]+")
+ ? Long.parseLong(name.substring(name.indexOf('_') + 1))
+ : 0;
+ }
+
+ public static void awaitDifferentSchemaLeader(final String database, final DataNodeWrapper old)
+ throws Exception {
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreExceptionsMatching(e -> e instanceof IOException || e instanceof TException)
+ .untilAsserted(() -> assertNotEquals(old, schemaLeader(database)));
+ }
+
+ public static void awaitDifferentSchemaLeader(
+ final String database, final int regionId, final DataNodeWrapper old) {
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreExceptionsMatching(e -> e instanceof IOException || e instanceof TException)
+ .untilAsserted(() -> assertNotEquals(old, schemaLeader(database, regionId)));
+ }
+
+ public static int tableDeviceRegion(final String database, final String table, final String id)
+ throws Exception {
+ return partitions(
+ database,
+ Collections.singletonList(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {table, id})))
+ .values()
+ .iterator()
+ .next()
+ .getId();
+ }
+
+ public static void awaitDropSuccess(final String database, final String table) {
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertRows(
+ 0,
+ BaseEnv.TABLE_SQL_DIALECT,
+ "select state, written_mod_file_count, remaining_mod_file_count from information_schema.drop_table_procedures "
+ + "where database='"
+ + database
+ + "' and table_name='"
+ + table
+ + "'",
+ "state,written_mod_file_count,remaining_mod_file_count,",
+ "SUCCESS,1,0,"));
+ }
+
+ /** A single-shot process pause; its hit file records the actual procedure ID or captured RPC. */
+ public static final class Fault implements AutoCloseable {
+ private final Path directory;
+ private final String point;
+ private final List controls = new ArrayList<>();
+
+ public Fault(final AbstractNodeWrapper node, final String point) throws IOException {
+ this.directory = Paths.get(node.getNodePath(), MetadataTestHooks.DIRECTORY);
+ this.point = point;
+ assertTrue(
+ "Metadata test agent is missing from " + node.getId(),
+ Files.exists(directory.resolve("hit.agent-ready")));
+ clearEvidence();
+ }
+
+ public Fault arm(final String mode) throws IOException {
+ final Path control = directory.resolve(mode + "." + point);
+ Files.createFile(control);
+ controls.add(control);
+ return this;
+ }
+
+ public void disarm(final String mode) throws IOException {
+ Files.deleteIfExists(directory.resolve(mode + "." + point));
+ }
+
+ public void release() throws IOException {
+ Files.deleteIfExists(directory.resolve("pause." + point));
+ Files.deleteIfExists(directory.resolve("blocked." + point));
+ Files.deleteIfExists(directory.resolve("fail." + point));
+ }
+
+ public String awaitHit() throws IOException {
+ return awaitEvidence("hit." + point);
+ }
+
+ public void awaitFailure() throws IOException {
+ awaitEvidence("hit.failure." + point);
+ }
+
+ private String awaitEvidence(final String name) throws IOException {
+ final Path hit = directory.resolve(name);
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ final Path error = directory.resolve("hit.instrumentation-error");
+ if (Files.exists(error)) {
+ throw new AssertionError(
+ new String(Files.readAllBytes(error), StandardCharsets.UTF_8));
+ }
+ return Files.exists(hit) && Files.size(hit) > 0;
+ });
+ return new String(Files.readAllBytes(hit), StandardCharsets.UTF_8);
+ }
+
+ private void clearEvidence() throws IOException {
+ try (Stream paths = Files.list(directory)) {
+ for (Path path : paths.collect(Collectors.toList())) {
+ final String name = path.getFileName().toString();
+ if (name.equals("hit." + point)
+ || name.startsWith("hit." + point + ".")
+ || name.equals("hit.failure." + point)) {
+ Files.deleteIfExists(path);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ release();
+ for (Path control : controls) {
+ Files.deleteIfExists(control);
+ }
+ clearEvidence();
+ }
+ }
+
+ public interface SqlOperation {
+ void execute() throws Exception;
+ }
+
+ /**
+ * Stop the old leader only after the durable next state is reached, and require another leader to
+ * enter the same state with the same procedure ID before releasing it. The client outcome is
+ * deliberately separate from the catalog/data assertions made by the caller.
+ */
+ public static void interruptProcedure(
+ final String procedure,
+ final String state,
+ final boolean rollback,
+ final SqlOperation operation)
+ throws Exception {
+ interruptProcedure(procedure, state, rollback, operation, () -> {});
+ }
+
+ public static void interruptProcedure(
+ final String procedure,
+ final String state,
+ final boolean rollback,
+ final SqlOperation operation,
+ final SqlOperation whilePaused)
+ throws Exception {
+ final String point = "procedure." + procedure + "." + state;
+ final List faults = new ArrayList<>();
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ final int oldIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ final ConfigNodeWrapper old = EnvFactory.getEnv().getConfigNodeWrapper(oldIndex);
+ Future result = null;
+ try {
+ for (ConfigNodeWrapper node : EnvFactory.getEnv().getConfigNodeWrapperList()) {
+ faults.add(new Fault(node, point).arm("pause"));
+ }
+ result =
+ executor.submit(
+ () -> {
+ try {
+ operation.execute();
+ return null;
+ } catch (SQLException e) {
+ return e;
+ }
+ });
+ final String id = faults.get(oldIndex).awaitHit();
+ assertFalse("DDL finished before its pause", result.isDone());
+ whilePaused.execute();
+ old.stopForcibly();
+ assertFalse(old.isAlive());
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreException(IOException.class)
+ .untilAsserted(
+ () -> assertNotEquals(oldIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()));
+ final int newIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ assertEquals(
+ "The new leader must recover the original procedure",
+ id,
+ faults.get(newIndex).awaitHit());
+ if (rollback) {
+ faults.get(newIndex).arm("fail");
+ }
+ // Release pauses but keep the requested failure armed until its invocation is observed.
+ for (Fault fault : faults) {
+ Files.deleteIfExists(fault.directory.resolve("pause." + point));
+ Files.deleteIfExists(fault.directory.resolve("blocked." + point));
+ }
+ if (rollback) {
+ faults.get(newIndex).awaitFailure();
+ }
+ final Exception outcome = result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (outcome != null) {
+ final int code = ((SQLException) outcome).getErrorCode();
+ assertTrue(
+ outcome.toString(),
+ code == TSStatusCode.INTERNAL_REQUEST_TIME_OUT.getStatusCode()
+ || code == TSStatusCode.CAN_NOT_CONNECT_CONFIGNODE.getStatusCode()
+ || code == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()
+ || code == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()
+ || (!rollback
+ && "DropTableColumnProcedure".equals(procedure)
+ && "INVALIDATE_CACHE".equals(state)
+ && code == TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ || (!rollback
+ && "CreateTableProcedure".equals(procedure)
+ && "COMMIT_RELEASE".equals(state)
+ && code == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode()));
+ }
+ } finally {
+ try {
+ for (Fault fault : faults) {
+ fault.close();
+ }
+ } finally {
+ if (!old.isAlive()) {
+ old.start();
+ }
+ if (result != null && !result.isDone()) {
+ result.cancel(true);
+ }
+ executor.shutdownNow();
+ assertTrue(
+ "DDL executor did not stop",
+ executor.awaitTermination(TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ // A shared-cluster successor may immediately broadcast configuration to every ConfigNode.
+ // Starting the old process alone does not establish that its Thrift endpoint is ready.
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .ignoreException(IOException.class)
+ .ignoreException(TException.class)
+ .untilAsserted(
+ () -> {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient)
+ EnvFactory.getEnv().getConfigNodeConnection(oldIndex)) {
+ client.showCluster();
+ }
+ });
+ }
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestAgent.java b/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestAgent.java
new file mode 100644
index 0000000000000..806dbac4d80bd
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestAgent.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.it.fault.metadata;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.instrument.ClassFileTransformer;
+import java.lang.instrument.Instrumentation;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.security.ProtectionDomain;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtMethod;
+import javassist.LoaderClassPath;
+import javassist.expr.ExprEditor;
+import javassist.expr.MethodCall;
+
+/**
+ * Adds deterministic metadata fault boundaries only to child JVMs started by the recovery ITs.
+ * Server sources and artifacts contain no test hooks. The agent is built from test classes at IT
+ * setup, so the same setup works under Failsafe and in an IDE, without an installed agent artifact.
+ */
+public final class MetadataTestAgent {
+ private static final String HOOKS = MetadataTestHooks.class.getName();
+ private static final String STATE_MACHINE =
+ "org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure";
+ private static final String SCHEMA_STATE_MACHINE =
+ "org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine";
+ private static final String RATIS_CONSENSUS = "org.apache.iotdb.consensus.ratis.RatisConsensus";
+ private static final String CONFIG_CLIENT =
+ "org.apache.iotdb.db.protocol.client.ConfigNodeClient";
+ private static final String ABSTRACT_CONFIG_CLIENT =
+ "org.apache.iotdb.commons.client.AbstractConfigNodeClient";
+ private static final String INTERNAL_RPC =
+ "org.apache.iotdb.db.protocol.thrift.impl.DataNodeInternalRPCServiceImpl";
+ private static final String STATUS_CODES = "org.apache.iotdb.rpc.TSStatusCode.";
+ private static final Set TARGETS =
+ new HashSet<>(
+ Arrays.asList(
+ STATE_MACHINE,
+ SCHEMA_STATE_MACHINE,
+ RATIS_CONSENSUS,
+ CONFIG_CLIENT,
+ ABSTRACT_CONFIG_CLIENT,
+ INTERNAL_RPC));
+
+ private MetadataTestAgent() {}
+
+ public static void premain(final String arguments, final Instrumentation instrumentation)
+ throws IOException {
+ MetadataTestHooks.agentStarted();
+ instrumentation.addTransformer(new Transformer());
+ }
+
+ /** The dependency is test-scoped and bundled only into this temporary agent JAR. */
+ public static synchronized Path createJar(final Path destination) throws Exception {
+ final Path target = destination.toAbsolutePath();
+ Files.createDirectories(target.getParent());
+ final Path temporary = Files.createTempFile(target.getParent(), "metadata-agent-", ".jar");
+ final Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ manifest.getMainAttributes().putValue("Premain-Class", MetadataTestAgent.class.getName());
+ try {
+ try (JarOutputStream output =
+ new JarOutputStream(Files.newOutputStream(temporary), manifest);
+ JarFile dependency =
+ new JarFile(
+ Paths.get(
+ ClassPool.class
+ .getProtectionDomain()
+ .getCodeSource()
+ .getLocation()
+ .toURI())
+ .toFile())) {
+ addClass(output, MetadataTestAgent.class);
+ addClass(output, MetadataTestHooks.class);
+ for (JarEntry entry : java.util.Collections.list(dependency.entries())) {
+ if (!entry.isDirectory() && entry.getName().startsWith("javassist/")) {
+ output.putNextEntry(new JarEntry(entry.getName()));
+ try (InputStream input = dependency.getInputStream(entry)) {
+ input.transferTo(output);
+ }
+ output.closeEntry();
+ }
+ }
+ }
+ Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);
+ return target;
+ } finally {
+ Files.deleteIfExists(temporary);
+ }
+ }
+
+ private static void addClass(final JarOutputStream output, final Class type)
+ throws IOException {
+ final String resource = type.getName().replace('.', '/') + ".class";
+ output.putNextEntry(new JarEntry(resource));
+ try (InputStream input = type.getResourceAsStream("/" + resource)) {
+ if (input == null) {
+ throw new IOException("Missing test agent class: " + resource);
+ }
+ input.transferTo(output);
+ }
+ output.closeEntry();
+ for (Class nested : type.getDeclaredClasses()) {
+ addClass(output, nested);
+ }
+ }
+
+ private static String errorStatus(final String code) {
+ return "new org.apache.iotdb.common.rpc.thrift.TSStatus("
+ + STATUS_CODES
+ + code
+ + ".getStatusCode())";
+ }
+
+ private static final class Transformer implements ClassFileTransformer {
+ @Override
+ public byte[] transform(
+ final ClassLoader loader,
+ final String name,
+ final Class redefined,
+ final ProtectionDomain domain,
+ final byte[] original) {
+ if (name == null || !TARGETS.contains(name.replace('/', '.'))) {
+ return null;
+ }
+ CtClass type = null;
+ try {
+ final ClassPool pool = new ClassPool(false);
+ pool.appendSystemPath();
+ pool.insertClassPath(new LoaderClassPath(loader));
+ type = pool.makeClass(new ByteArrayInputStream(original));
+ switch (type.getName()) {
+ case STATE_MACHINE:
+ final ProcedureBarrier barrier = new ProcedureBarrier();
+ type.getDeclaredMethod("execute").instrument(barrier);
+ if (barrier.matches != 1) {
+ throw new CannotCompileException("Expected exactly one procedure state transition");
+ }
+ break;
+ case SCHEMA_STATE_MACHINE:
+ snapshotBoundaries(type);
+ break;
+ case RATIS_CONSENSUS:
+ // The production default needs 1024 entries before purging. Keep the real purge and
+ // snapshot-install paths, with a small threshold only in these child test JVMs.
+ type.getDeclaredConstructors()[0].insertAfter(
+ "{ if ($1.getConsensusGroupType() == "
+ + "org.apache.iotdb.common.rpc.thrift.TConsensusGroupType.SchemaRegion) "
+ + "org.apache.ratis.server.RaftServerConfigKeys$Log.setPurgeGap(properties, 1); }");
+ break;
+ case CONFIG_CLIENT:
+ failRpc(type, "reloadCacheAfterLeaseRecovery", MetadataTestHooks.LEASE_RECOVERY);
+ break;
+ case ABSTRACT_CONFIG_CLIENT:
+ failRpc(type, "executeRemoteCallWithRetry", MetadataTestHooks.CONFIG_NODE_RPC);
+ break;
+ case INTERNAL_RPC:
+ internalRpcBoundaries(type, pool);
+ break;
+ default:
+ throw new AssertionError(type.getName());
+ }
+ final byte[] transformed = type.toBytecode();
+ MetadataTestHooks.transformed(type.getName());
+ return transformed;
+ } catch (Exception e) {
+ MetadataTestHooks.transformationFailed(name, e);
+ // Instrumentation normally ignores transformer exceptions and loads the original class.
+ // Reject this class instead: an IT must never silently run without its fault boundaries.
+ return new byte[] {0};
+ } finally {
+ if (type != null) {
+ type.detach();
+ }
+ }
+ }
+
+ private static void failRpc(final CtClass type, final String method, final String point)
+ throws Exception {
+ type.getDeclaredMethod(method)
+ .insertBefore(
+ "{ if ("
+ + HOOKS
+ + ".fail(\""
+ + point
+ + "\"))"
+ + " throw new org.apache.thrift.TException(); }");
+ }
+
+ private static void internalRpcBoundaries(final CtClass type, final ClassPool pool)
+ throws Exception {
+ for (String method :
+ Arrays.asList(
+ "invalidatePartitionCache",
+ "invalidateSchemaCache",
+ "invalidateMatchedSchemaCache",
+ "updateTable",
+ "invalidateTableCache",
+ "invalidateColumnCache")) {
+ type.getDeclaredMethod(method)
+ .insertBefore(
+ "{ if ("
+ + HOOKS
+ + ".fail(\""
+ + MetadataTestHooks.CONFIG_NODE_RPC
+ + "\")) {"
+ + HOOKS
+ + ".observe(\"cache-broadcast-rejected\", \"rejected\"); return "
+ + errorStatus("CAN_NOT_CONNECT_DATANODE")
+ + "; } }");
+ }
+ type.getDeclaredMethod("getDataNodeHeartBeat")
+ .insertBefore(
+ "{ if ("
+ + HOOKS
+ + ".fail(\""
+ + MetadataTestHooks.CONFIG_NODE_RPC
+ + "\")) {"
+ + HOOKS
+ + ".observe(\"heartbeat-rejected\", \"rejected\");"
+ + " throw new org.apache.thrift.TException(); } }");
+ final CtMethod update = type.getDeclaredMethod("updateAttribute");
+ update.addLocalVariable("metadataTestInvocation", pool.get("java.lang.String"));
+ update.insertBefore(
+ "{ metadataTestInvocation = null; if ("
+ + HOOKS
+ + ".fail(\""
+ + MetadataTestHooks.ATTRIBUTE_UPDATE
+ + "\")) return "
+ + errorStatus("EXECUTE_STATEMENT_ERROR")
+ + ";"
+ + " metadataTestInvocation = "
+ + HOOKS
+ + ".captureAttributeUpdate();"
+ + " if (metadataTestInvocation == null) return "
+ + errorStatus("EXECUTE_STATEMENT_ERROR")
+ + "; }");
+ update.insertAfter(
+ "{ if (metadataTestInvocation != null && $_.getCode() == "
+ + STATUS_CODES
+ + "SUCCESS_STATUS.getStatusCode()) "
+ + HOOKS
+ + ".observe(\"attribute-update-returned.\" + metadataTestInvocation,"
+ + " \"delivered\"); }");
+ }
+
+ private static void snapshotBoundaries(final CtClass type) throws Exception {
+ final String region = "schemaRegion.getSchemaRegionId().getId()";
+ final String reject = HOOKS + ".fail(\"snapshot-create.\" + " + region + ")";
+ final CtMethod snapshot = type.getDeclaredMethod("takeSnapshot");
+ snapshot.insertBefore("{ if (" + reject + ") return false; }");
+ snapshot.insertAfter(
+ "{ if ($_ && "
+ + reject
+ + ") $_ = false; if ($_) "
+ + HOOKS
+ + ".observe(\""
+ + MetadataTestHooks.SNAPSHOT_CREATED
+ + ".\" + "
+ + region
+ + ", $1.getName(), $1.getAbsolutePath()); }");
+ type.getDeclaredMethod("loadSnapshot")
+ .insertAfter(
+ "{ if ($_) "
+ + HOOKS
+ + ".observe(\""
+ + MetadataTestHooks.SNAPSHOT_LOADED
+ + ".\" + "
+ + region
+ + ", $1.getName(), $1.getAbsolutePath()); }");
+ }
+ }
+
+ private static final class ProcedureBarrier extends ExprEditor {
+ private int matches;
+
+ @Override
+ public void edit(final MethodCall call) throws CannotCompileException {
+ if ("executeFromState".equals(call.getMethodName())) {
+ matches++;
+ // The executor persisted this scheduled state before invoking execute(). Keep the pause
+ // immediately before the real transition, not at an estimated time after the DDL starts.
+ call.replace(
+ "{ if ("
+ + HOOKS
+ + ".procedure($0, $2)) {"
+ + " setFailure(new org.apache.iotdb.confignode.procedure.exception.ProcedureException("
+ + "new java.io.IOException())); $_ = "
+ + STATE_MACHINE
+ + "$Flow.HAS_MORE_STATE;"
+ + " } else { $_ = $proceed($$); } }");
+ }
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestHooks.java b/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestHooks.java
new file mode 100644
index 0000000000000..1e1c9c2403423
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/it/fault/metadata/MetadataTestHooks.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.it.fault.metadata;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Controls called only by the metadata IT agent in child node JVMs. This class is packaged into a
+ * temporary test agent, never a server artifact. Controls live beneath the child node's IOTDB_HOME;
+ * removing a control releases the captured call without restarting it.
+ */
+public final class MetadataTestHooks {
+ public static final String DIRECTORY = "metadata-test-hooks";
+ public static final String CONFIG_NODE_RPC = "config-node-rpc";
+ public static final String LEASE_RECOVERY = "lease-recovery";
+ public static final String ATTRIBUTE_UPDATE = "attribute-update";
+ public static final String SNAPSHOT_CREATED = "snapshot-created";
+ public static final String SNAPSHOT_LOADED = "snapshot-loaded";
+
+ private MetadataTestHooks() {}
+
+ private static Path directory() {
+ return Paths.get(System.getProperty("IOTDB_HOME"), DIRECTORY);
+ }
+
+ public static boolean fail(final String point) {
+ if (Files.exists(directory().resolve("fail." + point))) {
+ record(point, "failed");
+ record("failure." + point, "failed");
+ return true;
+ }
+ return false;
+ }
+
+ public static void pause(final String point, final String context) throws InterruptedException {
+ final Path control = directory().resolve("pause." + point);
+ if (Files.exists(control)) {
+ final Path blocked = directory().resolve("blocked." + point);
+ try {
+ Files.move(control, blocked);
+ record(point, context);
+ while (Files.exists(blocked)) {
+ TimeUnit.MILLISECONDS.sleep(10);
+ }
+ } catch (NoSuchFileException | FileAlreadyExistsException ignored) {
+ // Only the captured invocation waits. Later retries can overtake this old request.
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ public static void observe(final String point, final String context) {
+ if (Files.exists(directory().resolve("observe." + point))) {
+ record(point, context);
+ }
+ }
+
+ public static void observe(final String point, final String invocation, final String context) {
+ if (Files.exists(directory().resolve("observe." + point))) {
+ record(point + "." + invocation, context);
+ }
+ }
+
+ private static void record(final String point, final String context) {
+ try {
+ // Keep the first invocation, including its procedure ID / snapshot path, as evidence.
+ Files.write(
+ directory().resolve("hit." + point),
+ context.getBytes(StandardCharsets.UTF_8),
+ StandardOpenOption.CREATE_NEW,
+ StandardOpenOption.WRITE);
+ } catch (FileAlreadyExistsException ignored) {
+ // A retry must not replace the evidence for the original invocation.
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static boolean procedure(final Object procedure, final Object state)
+ throws InterruptedException {
+ if (!procedure
+ .getClass()
+ .getName()
+ .startsWith("org.apache.iotdb.confignode.procedure.impl.schema.")) {
+ return false;
+ }
+ final String point = "procedure." + procedure.getClass().getSimpleName() + "." + state;
+ try {
+ pause(point, procedure.getClass().getMethod("getProcId").invoke(procedure).toString());
+ } catch (ReflectiveOperationException e) {
+ throw new AssertionError("Cannot observe the real procedure ID", e);
+ }
+ return fail(point);
+ }
+
+ public static String captureAttributeUpdate() {
+ final String invocation = UUID.randomUUID().toString();
+ try {
+ pause(ATTRIBUTE_UPDATE, invocation);
+ return invocation;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return null;
+ }
+ }
+
+ static void agentStarted() throws IOException {
+ Files.createDirectories(directory());
+ record("agent-ready", "ready");
+ }
+
+ static void transformed(final String name) {
+ record("instrumented." + name, "ready");
+ }
+
+ static void transformationFailed(final String name, final Exception failure) {
+ record("instrumentation-error", name + ": " + failure);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/PipeTransferGate.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/PipeTransferGate.java
new file mode 100644
index 0000000000000..721b42028de2e
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/PipeTransferGate.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.IClientRPCService;
+
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.transport.TMemoryInputTransport;
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.IntPredicate;
+
+/**
+ * A test-only, framed binary Thrift relay. Each downstream connection owns one upstream connection,
+ * so handshakes, receiver identity and cleanup RPCs retain their real server-side lifecycle. No
+ * responses are fabricated and no credentials or payloads are retained in diagnostics.
+ */
+public final class PipeTransferGate implements AutoCloseable {
+ private final DataNodeWrapper receiver;
+ private final int port;
+ private final ServerSocket listener;
+ private final Object lifecycleLock = new Object();
+ private final Set sockets = ConcurrentHashMap.newKeySet();
+ private final ExecutorService workers = Executors.newCachedThreadPool();
+ private final AtomicReference failure = new AtomicReference<>();
+ private final AtomicReference hold = new AtomicReference<>();
+ private final AtomicInteger completedDataRequests = new AtomicInteger();
+ private final AtomicInteger completedPieces = new AtomicInteger();
+ private final AtomicInteger completedSeals = new AtomicInteger();
+ private final AtomicInteger rejectedRequests = new AtomicInteger();
+ private final ConcurrentHashMap>
+ rejectedStatusCodes = new ConcurrentHashMap<>();
+ private volatile boolean closed;
+
+ public PipeTransferGate(final DataNodeWrapper receiver) throws IOException {
+ this.receiver = receiver;
+ port = EnvUtils.searchAvailablePorts()[0];
+ ServerSocket opened = null;
+ try {
+ opened = new ServerSocket();
+ opened.bind(new InetSocketAddress(receiver.getIp(), port));
+ listener = opened;
+ workers.execute(
+ () -> {
+ while (!closed) {
+ try {
+ final Socket downstream = listener.accept();
+ synchronized (lifecycleLock) {
+ if (closed) {
+ downstream.close();
+ return;
+ }
+ sockets.add(downstream);
+ try {
+ workers.execute(() -> relay(downstream));
+ } catch (final RuntimeException e) {
+ sockets.remove(downstream);
+ downstream.close();
+ throw e;
+ }
+ }
+ } catch (final IOException | RuntimeException e) {
+ if (!closed) {
+ failure.compareAndSet(null, e);
+ }
+ return;
+ }
+ }
+ });
+ } catch (final IOException | RuntimeException | Error e) {
+ workers.shutdownNow();
+ try {
+ if (opened != null) {
+ opened.close();
+ }
+ } catch (final IOException cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ try {
+ Files.deleteIfExists(Paths.get(EnvUtils.getLockFilePath(port)));
+ } catch (final IOException cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ throw e;
+ }
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public Hold holdNext(final IntPredicate requestType, final boolean afterResponse) {
+ final Hold next = new Hold(requestType, afterResponse);
+ synchronized (lifecycleLock) {
+ Assert.assertFalse("The relay is closed", closed);
+ Assert.assertTrue("An earlier gate is still armed", hold.compareAndSet(null, next));
+ }
+ return next;
+ }
+
+ public static boolean isTablet(final int type) {
+ return type == PipeRequestType.TRANSFER_TABLET_INSERT_NODE.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_RAW.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_BINARY.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_BATCH.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_RAW_V2.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_BINARY_V2.getType()
+ || type == PipeRequestType.TRANSFER_TABLET_BATCH_V2.getType();
+ }
+
+ public static boolean isSeal(final int type) {
+ return type == PipeRequestType.TRANSFER_TS_FILE_SEAL.getType()
+ || type == PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD.getType();
+ }
+
+ public int completedDataRequests() {
+ assertHealthy();
+ return completedDataRequests.get();
+ }
+
+ public int completedPieces() {
+ assertHealthy();
+ return completedPieces.get();
+ }
+
+ public int rejectedRequests() {
+ assertHealthy();
+ return rejectedRequests.get();
+ }
+
+ public int rejectedRequests(final int statusCode) {
+ assertHealthy();
+ final ConcurrentHashMap counts = rejectedStatusCodes.get(statusCode);
+ return counts == null ? 0 : counts.values().stream().mapToInt(AtomicInteger::get).sum();
+ }
+
+ public int rejectedRequests(final IntPredicate requestType, final int statusCode) {
+ assertHealthy();
+ final ConcurrentHashMap counts = rejectedStatusCodes.get(statusCode);
+ return counts == null
+ ? 0
+ : counts.entrySet().stream()
+ .filter(entry -> requestType.test(entry.getKey()))
+ .mapToInt(entry -> entry.getValue().get())
+ .sum();
+ }
+
+ public int completedSeals() {
+ assertHealthy();
+ return completedSeals.get();
+ }
+
+ public int openConnections() {
+ assertHealthy();
+ return sockets.size();
+ }
+
+ public void assertHealthy() {
+ final Throwable error = failure.get();
+ if (error != null) {
+ throw new AssertionError("Thrift relay failed", error);
+ }
+ }
+
+ private void relay(final Socket downstream) {
+ final Socket upstream = new Socket();
+ try (Socket client = downstream;
+ upstream) {
+ synchronized (lifecycleLock) {
+ if (closed) {
+ return;
+ }
+ sockets.add(upstream);
+ }
+ upstream.connect(new InetSocketAddress(receiver.getIp(), receiver.getPort()), 5000);
+ upstream.setSoTimeout(120000);
+ final DataInputStream requests = new DataInputStream(client.getInputStream());
+ final DataOutputStream forwards = new DataOutputStream(upstream.getOutputStream());
+ final DataInputStream responses = new DataInputStream(upstream.getInputStream());
+ final DataOutputStream replies = new DataOutputStream(client.getOutputStream());
+ while (!closed) {
+ final byte[] request = readFrame(requests);
+ final TBinaryProtocol requestProtocol =
+ new TBinaryProtocol(new TMemoryInputTransport(request));
+ Assert.assertEquals("pipeTransfer", requestProtocol.readMessageBegin().name);
+ final IClientRPCService.pipeTransfer_args args = new IClientRPCService.pipeTransfer_args();
+ args.read(requestProtocol);
+ final int type = args.getReq().getType();
+ final Hold candidate = hold.get();
+ final Hold selected =
+ candidate != null && candidate.requestType.test(type) ? candidate : null;
+ if (selected != null && !selected.afterResponse) {
+ if (!selected.block(client, upstream)) {
+ return;
+ }
+ }
+ writeFrame(forwards, request);
+ final byte[] response = readFrame(responses);
+ final TBinaryProtocol responseProtocol =
+ new TBinaryProtocol(new TMemoryInputTransport(response));
+ responseProtocol.readMessageBegin();
+ final IClientRPCService.pipeTransfer_result result =
+ new IClientRPCService.pipeTransfer_result();
+ result.read(responseProtocol);
+ final int code = result.getSuccess().getStatus().getCode();
+ if (code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ if (isTablet(type) || isSeal(type)) {
+ completedDataRequests.incrementAndGet();
+ }
+ if (isSeal(type)) {
+ completedSeals.incrementAndGet();
+ }
+ if (type == PipeRequestType.TRANSFER_TS_FILE_PIECE.getType()
+ || type == PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD.getType()) {
+ completedPieces.incrementAndGet();
+ }
+ } else {
+ rejectedRequests.incrementAndGet();
+ recordRejectedStatus(type, result.getSuccess().getStatus());
+ }
+ if (selected != null && selected.afterResponse) {
+ Assert.assertEquals(
+ "Only a successful receiver response establishes the lost-ACK boundary",
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ code);
+ if (!selected.block(client, upstream)) {
+ return;
+ }
+ }
+ writeFrame(replies, response);
+ }
+ } catch (final EOFException | SocketException e) {
+ // STOP/DROP, the held disconnect and node restarts deliberately close these connections.
+ } catch (final Throwable e) {
+ if (!closed) {
+ failure.compareAndSet(null, e);
+ }
+ } finally {
+ sockets.remove(upstream);
+ sockets.remove(downstream);
+ }
+ }
+
+ private void recordRejectedStatus(final int requestType, final TSStatus status) {
+ rejectedStatusCodes
+ .computeIfAbsent(status.getCode(), ignored -> new ConcurrentHashMap<>())
+ .computeIfAbsent(requestType, ignored -> new AtomicInteger())
+ .incrementAndGet();
+ if (status.isSetSubStatus()) {
+ status.getSubStatus().forEach(subStatus -> recordRejectedStatus(requestType, subStatus));
+ }
+ }
+
+ private static byte[] readFrame(final DataInputStream input) throws IOException {
+ final int size = input.readInt();
+ Assert.assertTrue("Invalid Thrift frame size: " + size, size > 0 && size <= 64 * 1024 * 1024);
+ final byte[] frame = new byte[size];
+ input.readFully(frame);
+ return frame;
+ }
+
+ private static void writeFrame(final DataOutputStream output, final byte[] frame)
+ throws IOException {
+ output.writeInt(frame.length);
+ output.write(frame);
+ output.flush();
+ }
+
+ @Override
+ public void close() throws Exception {
+ final ArrayList toClose;
+ synchronized (lifecycleLock) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ toClose = new ArrayList<>(sockets);
+ }
+ IOException closeFailure = null;
+ try {
+ listener.close();
+ } catch (final IOException e) {
+ closeFailure = e;
+ }
+ for (final Socket socket : toClose) {
+ try {
+ socket.close();
+ } catch (final IOException e) {
+ if (closeFailure == null) {
+ closeFailure = e;
+ } else {
+ closeFailure.addSuppressed(e);
+ }
+ }
+ }
+ try {
+ workers.shutdownNow();
+ Assert.assertTrue(
+ "Relay workers did not stop", workers.awaitTermination(10, TimeUnit.SECONDS));
+ } finally {
+ Files.deleteIfExists(Paths.get(EnvUtils.getLockFilePath(port)));
+ }
+ if (closeFailure != null) {
+ throw closeFailure;
+ }
+ assertHealthy();
+ }
+
+ public final class Hold implements AutoCloseable {
+ private final IntPredicate requestType;
+ private final boolean afterResponse;
+ private final CountDownLatch reached = new CountDownLatch(1);
+ private final CountDownLatch released = new CountDownLatch(1);
+ private final Set heldSockets = ConcurrentHashMap.newKeySet();
+ private boolean finished;
+ private boolean disconnected;
+
+ private Hold(final IntPredicate requestType, final boolean afterResponse) {
+ this.requestType = requestType;
+ this.afterResponse = afterResponse;
+ }
+
+ private boolean block(final Socket downstream, final Socket upstream) throws Exception {
+ synchronized (this) {
+ if (finished) {
+ return !disconnected;
+ }
+ heldSockets.add(downstream);
+ heldSockets.add(upstream);
+ reached.countDown();
+ }
+ Assert.assertTrue("Held request was never released", released.await(120, TimeUnit.SECONDS));
+ synchronized (this) {
+ return !disconnected;
+ }
+ }
+
+ public void awaitReached() {
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ assertHealthy();
+ Assert.assertEquals("Expected RPC boundary was not reached", 0, reached.getCount());
+ });
+ }
+
+ /** Discard the request/response at the selected boundary while keeping the relay available. */
+ public synchronized void disconnect() throws IOException {
+ Assert.assertEquals(0, reached.getCount());
+ if (disconnected) {
+ return;
+ }
+ Assert.assertFalse("The held request was already released", finished);
+ finished = true;
+ disconnected = true;
+ hold.compareAndSet(this, null);
+ IOException closeFailure = null;
+ try {
+ for (final Socket socket : heldSockets) {
+ try {
+ socket.close();
+ } catch (final IOException e) {
+ if (closeFailure == null) {
+ closeFailure = e;
+ } else {
+ closeFailure.addSuppressed(e);
+ }
+ }
+ }
+ } finally {
+ released.countDown();
+ }
+ if (closeFailure != null) {
+ throw closeFailure;
+ }
+ }
+
+ @Override
+ public synchronized void close() {
+ if (finished) {
+ return;
+ }
+ finished = true;
+ hold.compareAndSet(this, null);
+ released.countDown();
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRecoveryIT.java
new file mode 100644
index 0000000000000..bf6fd2e57b4aa
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRecoveryIT.java
@@ -0,0 +1,867 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.RegionRoleType;
+import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdReq;
+import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdResp;
+import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
+import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.MultiEnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.MultipleFailureException;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced.PipeTransferFaultInjector.Mode;
+import static org.awaitility.Awaitility.await;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({MultiClusterIT2DualTreeAutoEnhanced.class})
+public class IoTDBPipeRecoveryIT {
+
+ private static BaseEnv senderEnv;
+ private static BaseEnv receiverEnv;
+ private static final int RETRY_QUEUE_THRESHOLD = 2;
+ private static final List SINK_PENDING_METRICS =
+ Arrays.asList(
+ "untransferred_tablet_count",
+ "untransferred_tsfile_count",
+ "pipe_async_connector_retry_event_queue_size",
+ "pipe_pending_handlers_size");
+ private final List pipes = new ArrayList<>();
+ private final List dataSets = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ MultiEnvFactory.createEnv(2);
+ senderEnv = MultiEnvFactory.getEnv(0);
+ receiverEnv = MultiEnvFactory.getEnv(1);
+ senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
+ receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(false);
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ env.getConfig()
+ .getCommonConfig()
+ .setEnforceStrongPassword(false)
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaReplicationFactor(3)
+ .setDataReplicationFactor(3)
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(2)
+ .setPipeMemoryManagementEnabled(true)
+ .setIsPipeEnableMemoryCheck(true)
+ .setPipeAutoSplitFullEnabled(false)
+ .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1)
+ .setDatanodeMemoryProportion("3:3:1:1:2:0");
+ env.getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+ final Properties properties = new Properties();
+ // A small admission budget and forced-retry threshold expose pressure with few rows.
+ properties.setProperty("pipe_insert_node_queue_memory", "262144");
+ properties.setProperty(
+ "pipe_async_sink_forced_retry_tablet_event_queue_size",
+ Integer.toString(RETRY_QUEUE_THRESHOLD));
+ properties.setProperty(
+ "pipe_async_sink_forced_retry_total_event_queue_size",
+ Integer.toString(RETRY_QUEUE_THRESHOLD));
+ properties.setProperty("pipe_sink_rpc_thrift_compression_enabled", "false");
+ ((MppCommonConfig) env.getConfig().getDataNodeCommonConfig()).updateProperties(properties);
+ env.initClusterEnvironment(1, 3);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ final List failures = new ArrayList<>();
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ if (env != null) {
+ try {
+ env.cleanClusterEnvironment();
+ } catch (final Throwable e) {
+ failures.add(e);
+ }
+ }
+ }
+ MultipleFailureException.assertEmpty(failures);
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ // JUnit combines @After failures with the test failure; retain all cleanup failures as well.
+ final List failures = new ArrayList<>();
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ try {
+ startStoppedDataNodes(env);
+ } catch (final Throwable e) {
+ failures.add(e);
+ }
+ }
+ for (final String pipe : pipes) {
+ try {
+ execute(senderEnv, "DROP PIPE IF EXISTS " + pipe);
+ assertPipeReleased(pipe);
+ } catch (final Throwable e) {
+ failures.add(e);
+ }
+ }
+ for (final DataSet dataSet : dataSets) {
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ try {
+ execute(env, "DELETE DATABASE " + dataSet.database);
+ } catch (final Throwable e) {
+ failures.add(e);
+ }
+ }
+ }
+ MultipleFailureException.assertEmpty(failures);
+ }
+
+ @Test
+ public void testAllReceiverDataNodesDownRetainAndDrainBacklogWithoutNewWrites() throws Exception {
+ // All endpoints, rather than just one failover candidate, fail after an established transfer.
+ final DataSet data = dataSet("receiver_outage");
+ final String pipe = createPipe(data, "outage", receiverEndpoints(), "iotdb-thrift-sync-sink");
+ data.write(0, 0, 4, senderEnv.getDataNodeWrapper(0));
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+
+ stopAllDataNodes(receiverEnv);
+ assertEndpointsUnavailable(receiverEnv);
+ data.write(0, 4, 28, senderEnv.getDataNodeWrapper(0));
+ execute(senderEnv, "FLUSH");
+ assertBacklog(pipe);
+ // From this point onward, no writes, endpoint edits, or pipe recreation may help recovery.
+ startStoppedDataNodes(receiverEnv);
+ assertData(senderEnv, data);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ }
+
+ @Test
+ public void testHistoryRealtimeHandoffSurvivesReceiverRestart() throws Exception {
+ // Hold a real historical request before forwarding, proving history is still incomplete when
+ // realtime rows and the receiver restart cross the history/realtime boundary.
+ final DataSet data = dataSet("handoff");
+ for (int batch = 0; batch < 4; batch++) {
+ data.write(0, batch * 8, batch * 8 + 8, senderEnv.getDataNodeWrapper(0));
+ execute(senderEnv, "FLUSH");
+ }
+ try (final PipeTransferFaultInjector relay =
+ new PipeTransferFaultInjector(
+ receiverEnv.getDataNodeWrapper(0), Mode.PAUSE_BEFORE_FORWARD)) {
+ final String pipe =
+ createPipe(data, "handoff", relay.getEndpoint(), "iotdb-thrift-sync-sink");
+ await().atMost(60, TimeUnit.SECONDS).until(() -> relay.getPausedRequests() > 0);
+ assertBacklog(pipe);
+ Assert.assertEquals(0, visibleRows(receiverEnv, data.device(0)));
+ data.write(0, 32, 48, senderEnv.getDataNodeWrapper(1));
+ stopAllDataNodes(receiverEnv);
+ relay.passThrough();
+ await().atMost(60, TimeUnit.SECONDS).until(() -> relay.getForwardingFailures() > 0);
+ startStoppedDataNodes(receiverEnv);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ data.write(0, 48, 56, senderEnv.getDataNodeWrapper(2));
+ assertData(senderEnv, data);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ dropPipe(pipe);
+ }
+ }
+
+ @Test
+ public void testMultipleDataRegionsAndWritersDrainAfterLeaderTransfer() throws Exception {
+ // Hold an earlier event while writes cross two DataRegions and the owner of one region moves.
+ final DataSet data = dataSet("region_writers");
+ final Map deviceRegions = new LinkedHashMap<>();
+ final Set regions = new HashSet<>();
+ final Set writers = new HashSet<>();
+ for (int device = 0; device < 16 && regions.size() < 2; device++) {
+ final int writer = device % 3;
+ data.write(device, 0, 4, senderEnv.getDataNodeWrapper(writer));
+ writers.add(writer);
+ final int region = regionOf(data.device(device));
+ deviceRegions.put(device, region);
+ regions.add(region);
+ }
+ Assert.assertEquals("Devices must actually occupy two DataRegions", 2, regions.size());
+ Assert.assertTrue("Writes must use multiple DataNodes", writers.size() > 1);
+ execute(senderEnv, "FLUSH");
+ final int region = deviceRegions.values().iterator().next();
+ try (final PipeTransferFaultInjector relay =
+ new PipeTransferFaultInjector(
+ receiverEnv.getDataNodeWrapper(0), Mode.PAUSE_BEFORE_FORWARD)) {
+ final String pipe =
+ createPipe(data, "regions", relay.getEndpoint(), "iotdb-thrift-sync-sink");
+ await().atMost(60, TimeUnit.SECONDS).until(() -> relay.getPausedRequests() > 0);
+ assertBacklog(pipe);
+
+ for (final int device : deviceRegions.keySet()) {
+ data.write(device, 4, 12, senderEnv.getDataNodeWrapper(device % 3));
+ }
+ final TRegionInfo oldLeader = awaitLeader(data.database, region, -1);
+ final DataNodeWrapper oldLeaderNode = nodeAtPort(senderEnv, oldLeader.getClientRpcPort());
+ // SHOW REGIONS established the exact leader before the stop; require a different Running
+ // leader before writing the next range.
+ oldLeaderNode.stop();
+ Assert.assertFalse("Selected DataRegion leader is still alive", oldLeaderNode.isAlive());
+ final TRegionInfo newLeader = awaitLeader(data.database, region, oldLeader.getDataNodeId());
+ final DataNodeWrapper newLeaderNode = nodeAtPort(senderEnv, newLeader.getClientRpcPort());
+ for (final int device : deviceRegions.keySet()) {
+ data.setAlias(device, "s_after_transfer", newLeaderNode);
+ data.write(device, 12, 20, newLeaderNode);
+ }
+
+ startStoppedDataNodes(senderEnv);
+ for (final int device : deviceRegions.keySet()) {
+ data.write(device, 20, 24, oldLeaderNode);
+ }
+ relay.passThrough();
+ assertData(senderEnv, data);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ // Reconcile again after a quiet drain while no more source writes are being produced.
+ assertData(receiverEnv, data);
+ }
+ }
+
+ @Test
+ public void testRequestNotDeliveredRetriesWithoutStarvingAnotherPipe() throws Exception {
+ assertTransferBoundary(Mode.REJECT_BEFORE_FORWARD, "request_loss");
+ }
+
+ @Test
+ public void testCommittedWriteWithLostResponseRetriesIdempotently() throws Exception {
+ assertTransferBoundary(Mode.LOSE_RESPONSE, "response_loss");
+ }
+
+ private void assertTransferBoundary(final Mode mode, final String name) throws Exception {
+ // The relay distinguishes a request never forwarded from a successful receiver response never
+ // returned. Final-state equality checks IoTDB's idempotent writes, not exactly-once delivery.
+ final DataSet data = dataSet(name);
+ final DataSet healthy = dataSet(name + "_healthy");
+ try (final PipeTransferFaultInjector relay =
+ new PipeTransferFaultInjector(receiverEnv.getDataNodeWrapper(0), Mode.PASS)) {
+ final String pipe = createPipe(data, name, relay.getEndpoint(), "iotdb-thrift-sync-sink");
+ final String healthyPipe =
+ createPipe(healthy, name + "_healthy", receiverEndpoints(), "iotdb-thrift-sync-sink");
+ data.write(0, 0, 1, senderEnv.getDataNodeWrapper(0));
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ relay.setMode(mode);
+ data.write(0, 1, 2, senderEnv.getDataNodeWrapper(0));
+ if (mode == Mode.LOSE_RESPONSE) {
+ await().atMost(60, TimeUnit.SECONDS).until(() -> relay.getCommittedResponses() > 0);
+ assertData(receiverEnv, data);
+ execute(receiverEnv, "FLUSH");
+ Assert.assertEquals(
+ "Response must remain withheld until the flush", 0, relay.getLostResponses());
+ relay.allowResponseLoss();
+ }
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .until(
+ () ->
+ relay.hasRetriedFailedRequest()
+ && (mode == Mode.LOSE_RESPONSE
+ ? relay.getLostResponses() >= 2
+ : relay.getRejectedRequests() >= 2));
+ assertBacklog(pipe);
+ if (mode == Mode.REJECT_BEFORE_FORWARD) {
+ Assert.assertEquals(1, visibleRows(receiverEnv, data.device(0)));
+ }
+ healthy.write(0, 0, 8, senderEnv.getDataNodeWrapper(1));
+ assertData(receiverEnv, healthy);
+ assertQuiet(healthyPipe);
+ relay.passThrough();
+ assertData(senderEnv, data);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ assertData(receiverEnv, data);
+ final int deliveredRequests = relay.getSuccessfulDataRequests();
+ await()
+ .during(2, TimeUnit.SECONDS)
+ .atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> Assert.assertEquals(deliveredRequests, relay.getSuccessfulDataRequests()));
+ dropPipe(pipe);
+ }
+ }
+
+ @Test
+ public void testRuntimeQueueBackpressureDrainsAndReleasesMemory() throws Exception {
+ // Unlike creation-time admission rejection, a running async sink reaches its forced-retry
+ // threshold while another pipe and ordinary writes continue on the same sender and receiver.
+ final DataSet data = dataSet("backpressure");
+ final DataSet healthy = dataSet("backpressure_healthy");
+ try (final PipeTransferFaultInjector relay =
+ new PipeTransferFaultInjector(receiverEnv.getDataNodeWrapper(0), Mode.PASS)) {
+ final String pipe =
+ createPipe(data, "backpressure", relay.getEndpoint(), "iotdb-thrift-sink");
+ final String healthyPipe =
+ createPipe(healthy, "backpressure_healthy", receiverEndpoints(), "iotdb-thrift-sink");
+ data.write(0, 0, 1, senderEnv.getDataNodeWrapper(0));
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+ final double baselineFloatingMemory = floatingMemory(pipe);
+ relay.setMode(Mode.REJECT_BEFORE_FORWARD);
+ data.write(0, 1, 49, senderEnv.getDataNodeWrapper(0));
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Assert.assertTrue(relay.getRejectedRequests() > 0);
+ Assert.assertTrue(
+ "Async retry queue did not reach its configured forced-retry threshold",
+ metric(pipe, "pipe_async_connector_retry_event_queue_size", true)
+ >= RETRY_QUEUE_THRESHOLD);
+ Assert.assertTrue(
+ "No retained floating pipe memory",
+ floatingMemory(pipe) > baselineFloatingMemory);
+ });
+ assertBacklog(pipe);
+ healthy.write(0, 0, 12, senderEnv.getDataNodeWrapper(1));
+ assertData(senderEnv, healthy);
+ assertData(receiverEnv, healthy);
+ assertQuiet(healthyPipe);
+ relay.passThrough();
+ assertQuiet(pipe);
+ assertData(receiverEnv, data);
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assert.assertEquals(
+ 0, metric(pipe, "pipe_async_connector_retry_event_queue_size"), 0));
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ final double memory = floatingMemory(pipe);
+ Assert.assertTrue(
+ "Floating pipe memory did not return to its idle bound: "
+ + memory
+ + " > "
+ + baselineFloatingMemory,
+ memory <= baselineFloatingMemory);
+ });
+ // STOP must preserve this suffix for a later START, without another business write.
+ final int failuresBeforeStop = relay.getRejectedRequests();
+ relay.setMode(Mode.REJECT_BEFORE_FORWARD);
+ data.write(0, 49, 57, senderEnv.getDataNodeWrapper(0));
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .until(() -> relay.getRejectedRequests() > failuresBeforeStop);
+ assertBacklog(pipe);
+ execute(senderEnv, "STOP PIPE " + pipe);
+ Assert.assertEquals("STOPPED", pipeInfo(pipe).getState());
+ await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> assertNoReceiverRuntime(pipe));
+ relay.passThrough();
+ execute(senderEnv, "START PIPE " + pipe);
+ assertData(senderEnv, data);
+ assertData(receiverEnv, data);
+ assertQuiet(pipe);
+
+ // DROP also releases a running sink with retained events. These rows are deliberately
+ // abandoned only after verifying complete delivery of the earlier ranges above.
+ final double floatingMemoryBeforeDrop = floatingMemory(pipe);
+ final int failuresBeforeDrop = relay.getRejectedRequests();
+ relay.setMode(Mode.REJECT_BEFORE_FORWARD);
+ data.write(0, 57, 105, senderEnv.getDataNodeWrapper(0));
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Assert.assertTrue(relay.getRejectedRequests() > failuresBeforeDrop);
+ Assert.assertTrue(
+ metric(pipe, "pipe_async_connector_retry_event_queue_size", true)
+ >= RETRY_QUEUE_THRESHOLD);
+ Assert.assertTrue(floatingMemory(pipe) > floatingMemoryBeforeDrop);
+ });
+ assertBacklog(pipe);
+ Assert.assertTrue(
+ "No retained floating pipe memory before DROP",
+ floatingMemory(pipe) > floatingMemoryBeforeDrop);
+ dropPipe(pipe);
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ Assert.assertTrue(
+ "DROP did not release floating pipe memory",
+ floatingMemory(pipe) <= floatingMemoryBeforeDrop));
+ healthy.write(0, 12, 16, senderEnv.getDataNodeWrapper(2));
+ assertData(receiverEnv, healthy);
+ assertQuiet(healthyPipe);
+ }
+ }
+
+ private DataSet dataSet(final String name) throws Exception {
+ final DataSet data = new DataSet("root.pipe_recovery_" + name);
+ dataSets.add(data);
+ // Create the database at both ends so even an early test failure has deterministic cleanup.
+ execute(senderEnv, "CREATE DATABASE " + data.database);
+ execute(receiverEnv, "CREATE DATABASE " + data.database);
+ return data;
+ }
+
+ private String createPipe(
+ final DataSet data, final String name, final String endpoints, final String sink)
+ throws Exception {
+ final String pipe = "recovery_" + name;
+ pipes.add(pipe);
+ final Map source = new HashMap<>();
+ source.put("source.path", data.database + ".**");
+ source.put("source.inclusion", "data.insert,schema.timeseries");
+ source.put("source.realtime.mode", "log");
+ source.put("source.history.enable", "true");
+ source.put("source.realtime.enable", "true");
+ source.put("capture.tree", "true");
+ source.put("capture.table", "false");
+ source.put("user", SessionConfig.DEFAULT_USER);
+ final Map connector = new HashMap<>();
+ connector.put("sink", sink);
+ connector.put("sink.node-urls", endpoints);
+ connector.put("sink.batch.enable", "false");
+ connector.put("sink.parallel.tasks", "1");
+ // Keep all requests on the relay when it is used, including after receiver leader discovery.
+ connector.put("sink.leader-cache.enable", "false");
+ try (final SyncConfigNodeIServiceClient client = leaderClient()) {
+ assertSuccess(
+ client.createPipe(new TCreatePipeReq(pipe, connector).setExtractorAttributes(source)));
+ }
+ Assert.assertEquals("RUNNING", pipeInfo(pipe).getState());
+ return pipe;
+ }
+
+ private static String receiverEndpoints() {
+ return receiverEnv.getDataNodeWrapperList().stream()
+ .map(DataNodeWrapper::getIpAndPortString)
+ .collect(Collectors.joining(","));
+ }
+
+ private static void stopAllDataNodes(final BaseEnv env) {
+ env.shutdownAllDataNodes();
+ for (final DataNodeWrapper node : env.getDataNodeWrapperList()) {
+ Assert.assertFalse(
+ "Sink endpoint still running: " + node.getIpAndPortString(), node.isAlive());
+ }
+ }
+
+ private static void assertEndpointsUnavailable(final BaseEnv env) throws IOException {
+ for (final DataNodeWrapper node : env.getDataNodeWrapperList()) {
+ try (final Socket socket = new Socket()) {
+ try {
+ socket.connect(new InetSocketAddress(node.getIp(), node.getPort()), 1000);
+ Assert.fail("Sink endpoint is reachable while all receiver nodes are stopped: " + node);
+ } catch (final IOException expected) {
+ // Connection refusal or timeout confirms that this endpoint cannot receive Pipe data.
+ }
+ }
+ }
+ }
+
+ private static void startStoppedDataNodes(final BaseEnv env) {
+ for (final DataNodeWrapper node : env.getDataNodeWrapperList()) {
+ if (!node.isAlive()) {
+ node.start();
+ }
+ }
+ env.ensureNodeStatus(
+ new ArrayList<>(env.getDataNodeWrapperList()),
+ Collections.nCopies(env.getDataNodeWrapperList().size(), NodeStatus.Running));
+ }
+
+ private static SyncConfigNodeIServiceClient leaderClient() throws Exception {
+ return (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection();
+ }
+
+ private static TShowPipeInfo pipeInfo(final String pipe) throws Exception {
+ try (final SyncConfigNodeIServiceClient client = leaderClient()) {
+ final TShowPipeResp response =
+ client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER));
+ assertSuccess(response.getStatus());
+ return response.getPipeInfoList().stream()
+ .filter(info -> pipe.equals(info.getId()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Missing pipe " + pipe));
+ }
+ }
+
+ private static void assertBacklog(final String pipe) {
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ final TShowPipeInfo info = pipeInfo(pipe);
+ Assert.assertTrue("No pending events: " + info, info.getRemainingEventCount() > 0);
+ });
+ }
+
+ private static void assertQuiet(final String pipe) {
+ await()
+ .atMost(90, TimeUnit.SECONDS)
+ .during(2, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ final TShowPipeInfo info = pipeInfo(pipe);
+ Assert.assertEquals(info.toString(), "RUNNING", info.getState());
+ Assert.assertTrue(info.toString(), info.isSetRemainingEventCount());
+ Assert.assertEquals(info.toString(), 0, info.getRemainingEventCount());
+ final Map> metrics = sinkMetrics(pipe);
+ for (final String name : SINK_PENDING_METRICS) {
+ Assert.assertTrue(
+ "Missing sink metric " + name + " for " + pipe, metrics.containsKey(name));
+ Assert.assertEquals(
+ "Pending sink work for " + pipe + ": " + name,
+ 0,
+ metrics.get(name).stream().mapToDouble(Double::doubleValue).sum(),
+ 0);
+ }
+ });
+ }
+
+ private static void assertData(final BaseEnv env, final DataSet data) {
+ await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ try (final Connection connection = env.getConnection();
+ final Statement statement = connection.createStatement()) {
+ for (final Map.Entry> entry : data.rows.entrySet()) {
+ final String device = data.device(entry.getKey());
+ try (final ResultSet result =
+ statement.executeQuery("SELECT s, payload FROM " + device)) {
+ TestUtils.assertResultSetEqualUnordered(
+ result, "Time," + device + ".s," + device + ".payload,", entry.getValue());
+ }
+ }
+ final List schema = new ArrayList<>();
+ for (final int device : data.rows.keySet()) {
+ schema.add(
+ data.device(device)
+ + ".s,"
+ + data.aliases.getOrDefault(device, "null")
+ + ","
+ + data.database
+ + ",INT32,PLAIN,SNAPPY,null,null,null,null,BASE,");
+ schema.add(
+ data.device(device)
+ + ".payload,null,"
+ + data.database
+ + ",TEXT,PLAIN,SNAPPY,null,null,null,null,BASE,");
+ }
+ try (final ResultSet result =
+ statement.executeQuery("SHOW TIMESERIES " + data.database + ".**")) {
+ TestUtils.assertResultSetEqualUnordered(
+ result,
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,",
+ schema);
+ }
+ } catch (final SQLException e) {
+ throw new AssertionError("Cannot reconcile " + data.database, e);
+ }
+ });
+ }
+
+ private static int visibleRows(final BaseEnv env, final String device) throws Exception {
+ int rows = 0;
+ try (final Connection connection = env.getConnection();
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("SELECT s FROM " + device)) {
+ while (result.next()) {
+ rows++;
+ }
+ }
+ return rows;
+ }
+
+ private static int regionOf(final String device) throws Exception {
+ final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ IDeviceID.Factory.DEFAULT_FACTORY.create(device).serialize(buffer);
+ try (final SyncConfigNodeIServiceClient client = leaderClient()) {
+ final TGetRegionIdResp response =
+ client.getRegionId(
+ new TGetRegionIdReq(TConsensusGroupType.DataRegion).setDevice(buffer.toByteArray()));
+ assertSuccess(response.getStatus());
+ Assert.assertEquals(response.toString(), 1, response.getDataRegionIdListSize());
+ return response.getDataRegionIdList().get(0).getId();
+ }
+ }
+
+ private static TRegionInfo awaitLeader(
+ final String database, final int region, final int excludedNode) {
+ final TRegionInfo[] leader = new TRegionInfo[1];
+ await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ try (final SyncConfigNodeIServiceClient client = leaderClient()) {
+ final TShowRegionResp response =
+ client.showRegion(
+ new TShowRegionReq()
+ .setConsensusGroupType(TConsensusGroupType.DataRegion)
+ .setDatabases(Collections.singletonList(database)));
+ assertSuccess(response.getStatus());
+ leader[0] =
+ response.getRegionInfoList().stream()
+ .filter(info -> info.getConsensusGroupId().getId() == region)
+ .filter(
+ info -> RegionRoleType.Leader.getRoleType().equals(info.getRoleType()))
+ .filter(info -> NodeStatus.Running.getStatus().equals(info.getStatus()))
+ .filter(info -> info.getDataNodeId() != excludedNode)
+ .findFirst()
+ .orElse(null);
+ Assert.assertNotNull("No new leader: " + response, leader[0]);
+ }
+ });
+ return leader[0];
+ }
+
+ private static DataNodeWrapper nodeAtPort(final BaseEnv env, final int port) {
+ return env.getDataNodeWrapperList().stream()
+ .filter(node -> node.getPort() == port)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Unknown DataNode port " + port));
+ }
+
+ private static double metric(final String pipe, final String metricName) {
+ return metric(pipe, metricName, false);
+ }
+
+ private static double metric(final String pipe, final String metricName, final boolean maximum) {
+ final List values = sinkMetrics(pipe).get(metricName);
+ Assert.assertNotNull("No " + metricName + " metric for " + pipe, values);
+ return maximum
+ ? values.stream().mapToDouble(Double::doubleValue).max().orElse(0)
+ : values.stream().mapToDouble(Double::doubleValue).sum();
+ }
+
+ private static Map> sinkMetrics(final String pipe) {
+ final Map> metrics = new HashMap<>();
+ for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) {
+ final String content =
+ senderEnv.getUrlContent(
+ "http://" + node.getIp() + ":" + node.getMetricPort() + "/metrics", null);
+ Assert.assertNotNull("Cannot read metrics from " + node.getIpAndPortString(), content);
+ for (final String line : content.split("\\R")) {
+ final int tags = line.indexOf('{');
+ if (tags > 0 && line.contains("pipe=\"" + pipe + "\"")) {
+ final double value = Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1));
+ metrics.computeIfAbsent(line.substring(0, tags), ignored -> new ArrayList<>()).add(value);
+ }
+ }
+ }
+ return metrics;
+ }
+
+ private static long ownedMemory(final String pipe) throws Exception {
+ long usage = 0;
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ for (final DataNodeWrapper node : env.getDataNodeWrapperList()) {
+ try (final Connection connection = env.getConnection(node);
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("SHOW PIPE MEMORY")) {
+ while (result.next()) {
+ if (pipe.equals(result.getString("pipe_name"))) {
+ usage += result.getLong("memory_usage_in_bytes");
+ }
+ }
+ }
+ }
+ }
+ return usage;
+ }
+
+ private static double floatingMemory(final String pipe) {
+ double usage = 0;
+ for (final DataNodeWrapper node : senderEnv.getDataNodeWrapperList()) {
+ final String content =
+ senderEnv.getUrlContent(
+ "http://" + node.getIp() + ":" + node.getMetricPort() + "/metrics", null);
+ Assert.assertNotNull("Cannot read metrics from " + node.getIpAndPortString(), content);
+ for (final String line : content.split("\\R")) {
+ if (line.startsWith("pipe_floating_memory_usage{")
+ && line.contains("name=\"" + pipe + "\"")) {
+ usage += Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1));
+ }
+ }
+ }
+ return usage;
+ }
+
+ private static void assertPipeReleased(final String pipe) {
+ await()
+ .atMost(60, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Assert.assertEquals("Pipe-owned memory remains after DROP", 0, ownedMemory(pipe));
+ Assert.assertTrue(
+ "Sink task metrics remain after DROP: " + pipe, sinkMetrics(pipe).isEmpty());
+ try (final SyncConfigNodeIServiceClient client = leaderClient()) {
+ final TShowPipeResp response =
+ client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER));
+ assertSuccess(response.getStatus());
+ Assert.assertFalse(
+ "Pipe metadata remains after DROP: " + pipe,
+ response.getPipeInfoList().stream()
+ .anyMatch(info -> pipe.equals(info.getId())));
+ }
+ assertNoReceiverRuntime(pipe);
+ });
+ }
+
+ private static void assertNoReceiverRuntime(final String pipe) throws SQLException {
+ for (final DataNodeWrapper node : receiverEnv.getDataNodeWrapperList()) {
+ try (final Connection connection = receiverEnv.getConnection(node);
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("SHOW RECEIVERS")) {
+ while (result.next()) {
+ final String ids = result.getString("PipeIDs");
+ Assert.assertTrue(
+ "Receiver runtime remains for " + pipe, ids == null || !ids.contains(pipe + "@"));
+ }
+ }
+ }
+ }
+
+ private static void dropPipe(final String pipe) throws Exception {
+ execute(senderEnv, "DROP PIPE IF EXISTS " + pipe);
+ assertPipeReleased(pipe);
+ }
+
+ private static void execute(final BaseEnv env, final String sql) throws Exception {
+ try (final Connection connection = env.getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ }
+ }
+
+ private static void assertSuccess(final TSStatus status) {
+ Assert.assertEquals(
+ status.toString(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+ }
+
+ private static final class DataSet {
+ private final String database;
+ private final Map> rows = new LinkedHashMap<>();
+ private final Map aliases = new HashMap<>();
+
+ private DataSet(final String database) {
+ this.database = database;
+ }
+
+ private String device(final int device) {
+ return database + ".d" + device;
+ }
+
+ private void setAlias(final int device, final String alias, final DataNodeWrapper writer)
+ throws Exception {
+ try (final Connection connection = senderEnv.getConnection(writer);
+ final Statement statement = connection.createStatement()) {
+ statement.execute("ALTER TIMESERIES " + device(device) + ".s UPSERT ALIAS=" + alias);
+ aliases.put(device, alias);
+ }
+ }
+
+ private void write(
+ final int device, final int start, final int end, final DataNodeWrapper writer)
+ throws Exception {
+ try (final Connection connection = senderEnv.getConnection(writer);
+ final Statement statement = connection.createStatement()) {
+ if (!rows.containsKey(device)) {
+ statement.execute(
+ "CREATE TIMESERIES "
+ + device(device)
+ + ".s WITH DATATYPE=INT32, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES "
+ + device(device)
+ + ".payload WITH DATATYPE=TEXT, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+ rows.put(device, new ArrayList<>());
+ }
+ for (int time = start; time < end; time++) {
+ final int value = device * 1000 + time * 7;
+ final String payload = "device_" + device + "_key_" + time + "_value_" + value;
+ statement.execute(
+ "INSERT INTO "
+ + device(device)
+ + "(time,s,payload) VALUES ("
+ + time
+ + ","
+ + value
+ + ",'"
+ + payload
+ + "')");
+ // Record only successful business writes; expected rows never come from a receiver query.
+ rows.get(device).add(time + "," + value + "," + payload + ",");
+ }
+ }
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRuntimeRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRuntimeRecoveryIT.java
new file mode 100644
index 0000000000000..7a2b8f7dd3b85
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeRuntimeRecoveryIT.java
@@ -0,0 +1,1648 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.MultiEnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
+import org.apache.iotdb.it.env.cluster.env.AbstractEnv;
+import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.itbase.runtime.ClusterTestConnection;
+import org.apache.iotdb.itbase.runtime.NodeConnection;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.pipe.it.PipeTransferGate;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import com.google.gson.JsonParser;
+import org.apache.thrift.transport.TTransportException;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ThrowingRunnable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+/** Tree and Table cases deliberately share the same two clusters and deterministic RPC gate. */
+@RunWith(IoTDBTestRunner.class)
+@Category(MultiClusterIT2DualTreeAutoEnhanced.class)
+public class IoTDBPipeRuntimeRecoveryIT {
+ private static BaseEnv sender;
+ private static BaseEnv receiver;
+ private static final int SOCKET_TIMEOUT_MS = 20000;
+ private static final int QUERY_TIMEOUT_SECONDS = 15;
+
+ @BeforeClass
+ public static void setUp() {
+ MultiEnvFactory.createEnv(2);
+ sender = MultiEnvFactory.getEnv(0);
+ receiver = MultiEnvFactory.getEnv(1);
+ for (final BaseEnv env : Arrays.asList(sender, receiver)) {
+ env.getConfig()
+ .getCommonConfig()
+ .setEnforceStrongPassword(false)
+ .setAutoCreateSchemaEnabled(true)
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaReplicationFactor(1)
+ .setDataReplicationFactor(1)
+ .setPipeMemoryManagementEnabled(true)
+ .setIsPipeEnableMemoryCheck(false)
+ .setPipeAutoSplitFullEnabled(false)
+ .setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1)
+ .setDnConnectionTimeoutMs(10000);
+ if (env == sender) {
+ env.getConfig().getCommonConfig().setDefaultDataRegionGroupNumPerDatabase(2);
+ }
+ env.getConfig().getDataNodeConfig().setDataNodeMemoryProportion("3:3:1:1:9:0");
+ env.initClusterEnvironment(1, 1);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ try {
+ if (sender != null) {
+ sender.cleanClusterEnvironment();
+ }
+ } finally {
+ if (receiver != null) {
+ receiver.cleanClusterEnvironment();
+ }
+ }
+ }
+
+ @Test
+ public void testStopStartWithUnacknowledgedBatchKeepsBacklogAndDropReleasesReceiver()
+ throws Exception {
+ // Unlike a status-switch test, the receiver has applied a batch whose ACK is still in flight.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("stop_ack", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final long generation = pipe(f.name).getCreationTime();
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.assertRows(0, 2);
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ operateWhileHeld(
+ "stop pipe " + f.name, ack, () -> assertEquals("STOPPED", pipe(f.name).getState()));
+ assertEquals("STOPPED", pipe(f.name).getState());
+ // STOP suspends the existing sink; its idle clients remain reusable until DROP.
+ f.assertBacklog(0, 8, 0, 2);
+ sql(sender, "start pipe " + f.name);
+ }
+ f.assertRows(0, 8);
+ assertEquals(generation, pipe(f.name).getCreationTime());
+ awaitDrained(f.name);
+ f.assertSchema();
+ sql(sender, "drop pipe " + f.name);
+ awaitNoPipeResources(f.name);
+ }
+ }
+
+ @Test
+ public void testFailedAlterIsAtomicWithCommittedInflightAndPendingData() throws Exception {
+ // Validation failure must preserve all three plugins while a real receiver ACK is outstanding.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("failed_alter", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final TShowPipeInfo before = pipe(f.name);
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ final SQLException validation =
+ assertThrows(
+ SQLException.class,
+ () ->
+ sql(
+ sender,
+ "alter pipe "
+ + f.name
+ + " modify source ('source.path'='root.must_not_apply.**')"
+ + " replace processor ('processor'='p1_missing_processor')"
+ + " modify sink ('sink.batch.enable'='true')"));
+ assertTrue(
+ validation.getMessage(),
+ validation.getMessage().toLowerCase(Locale.ROOT).contains("p1_missing_processor"));
+ final TShowPipeInfo after = pipe(f.name);
+ assertEquals(before.getCreationTime(), after.getCreationTime());
+ assertEquals(before.getPipeExtractor(), after.getPipeExtractor());
+ assertEquals(before.getPipeProcessor(), after.getPipeProcessor());
+ assertEquals(before.getPipeConnector(), after.getPipeConnector());
+ }
+ f.assertRows(0, 8);
+ awaitDrained(f.name);
+ restartConfigNode();
+ assertEquals(before.getPipeExtractor(), pipe(f.name).getPipeExtractor());
+ assertEquals(before.getPipeProcessor(), pipe(f.name).getPipeProcessor());
+ assertEquals(before.getPipeConnector(), pipe(f.name).getPipeConnector());
+ f.assertRows(0, 8);
+ }
+ }
+
+ @Test
+ public void testSinkAlterAfterLostAckDrainsOriginalBacklogWithNewFormat() throws Exception {
+ // A sink replacement may retry an applied batch; assert final values, not exactly-once RPCs.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("sink_alter", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final TShowPipeInfo before = pipe(f.name);
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ operateWhileHeld(
+ "alter pipe "
+ + f.name
+ + " modify sink ('sink.batch.enable'='true', 'sink.batch.max-delay-seconds'='1',"
+ + " 'sink.format'='tsfile')",
+ ack,
+ () -> assertTrue(pipe(f.name).getPipeConnector().contains("sink.format=tsfile")));
+ }
+ assertTrue(pipe(f.name).getPipeConnector().contains("sink.format=tsfile"));
+ assertEquals(before.getPipeExtractor(), pipe(f.name).getPipeExtractor());
+ assertEquals(before.getPipeProcessor(), pipe(f.name).getPipeProcessor());
+ f.assertRows(0, 8);
+ await().until(() -> gate.completedSeals() > 0);
+ awaitDrained(f.name);
+ f.assertSchema();
+ }
+ }
+
+ @Test
+ public void testDropWithLostAckReleasesResourcesAndSameNameStartsNewGeneration()
+ throws Exception {
+ // DROP must release a batch waiting for ACK; same-name realtime-only recreation skips old WAL.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("drop_ack", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final long oldGeneration = pipe(f.name).getCreationTime();
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.assertRows(0, 2);
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ operateWhileHeld(
+ "drop pipe " + f.name, ack, () -> assertEquals("PRE_DELETE", pipe(f.name).getState()));
+ }
+ awaitNoPipeResources(f.name);
+ await().until(() -> gate.openConnections() == 0);
+ f.write(10, 12);
+ f.create(gate, ", 'source.history.enable'='false'", "");
+ assertNotEquals(oldGeneration, pipe(f.name).getCreationTime());
+ f.write(20, 24);
+ f.assertRows(0, 2, 20, 24);
+ awaitDrained(f.name);
+ restartConfigNode();
+ f.assertRows(0, 2, 20, 24);
+ }
+ }
+
+ @Test
+ public void testReceiverRoleRevocationRetriesPendingRowsAfterGrant() throws Exception {
+ // Static permission tests miss revocation on a connector that has already authenticated.
+ final String user = "p1_role_user";
+ final String password = "P1-test-pass1234";
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("role_recovery", gate)) {
+ sql(receiver, "create user " + user + " '" + password + "'");
+ try (Cleanup cleanup =
+ new Cleanup(
+ () -> sql(sender, "drop pipe if exists " + f.name),
+ () -> awaitNoPipeResources(f.name),
+ () -> sql(receiver, "drop user " + user))) {
+ sql(receiver, "grant WRITE on " + f.database + ".** to user " + user);
+ sql(receiver, "grant SYSTEM on root.** to user " + user);
+ f.create(gate, "", ", 'sink.user'='" + user + "', 'sink.password'='" + password + "'");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final long generation = pipe(f.name).getCreationTime();
+ sql(receiver, "revoke WRITE on " + f.database + ".** from user " + user);
+ final int noPermission = TSStatusCode.NO_PERMISSION.getStatusCode();
+ final int rejectedBefore = gate.rejectedRequests(noPermission);
+ f.write(1, 8);
+ await().until(() -> gate.rejectedRequests(noPermission) > rejectedBefore);
+ f.assertBacklog(0, 8, 0, 1);
+ f.assertRows(0, 1);
+ final TShowPipeInfo failed = pipe(f.name);
+ assertFalse(failed.toString().contains(password));
+ sql(receiver, "grant WRITE on " + f.database + ".** to user " + user);
+ sql(receiver, "grant SYSTEM on root.** to user " + user);
+ // Authorization errors may stop a Pipe; START resumes that same durable generation.
+ if ("STOPPED".equals(pipe(f.name).getState())) {
+ sql(sender, "start pipe " + f.name);
+ }
+ f.assertRows(0, 8);
+ assertEquals(generation, pipe(f.name).getCreationTime());
+ awaitDrained(f.name);
+ }
+ }
+ }
+
+ @Test
+ public void testTsFileInterruptedBeforeSealRemainsInvisibleThenResumes() throws Exception {
+ // Real pieces have reached disk, but no seal has been sent. A restart must not load the
+ // residue.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("file_piece", gate)) {
+ f.write(0, 16);
+ sql(sender, "flush");
+ try (PipeTransferGate.Hold seal = gate.holdNext(PipeTransferGate::isSeal, false)) {
+ f.create(gate, ", 'source.realtime.mode'='file'", ", 'sink.format'='tsfile'");
+ seal.awaitReached();
+ assertTrue("No piece reached the real receiver", gate.completedPieces() > 0);
+ assertFalse(
+ "No incomplete file was staged on the receiver", stagingFiles(receiver).isEmpty());
+ assertEquals(0, gate.completedSeals());
+ f.assertRows();
+ final DataNodeWrapper dn = receiver.getDataNodeWrapper(0);
+ try (Cleanup restart =
+ new Cleanup(
+ () -> {
+ if (!dn.isAlive()) {
+ dn.start();
+ }
+ })) {
+ dn.stopForcibly();
+ await().until(() -> !dn.isAlive());
+ seal.disconnect();
+ // Hold the retried seal too, so restart visibility is checked before a complete file
+ // can be loaded by the retry. A fast reconnect must not erase this failure boundary.
+ try (PipeTransferGate.Hold retry = gate.holdNext(PipeTransferGate::isSeal, false)) {
+ dn.start();
+ awaitDataNode(receiver, dn);
+ retry.awaitReached();
+ assertEquals(0, gate.completedSeals());
+ f.assertRows();
+ }
+ }
+ }
+ f.assertRows(0, 16);
+ f.assertSchema();
+ awaitDrained(f.name);
+ }
+ }
+
+ @Test
+ public void testTsFileLostSealResponseRetriesWithoutChangingFinalValues() throws Exception {
+ // The receiver has sealed and loaded the file; only its successful response is discarded.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("seal_ack", gate)) {
+ f.write(0, 16);
+ sql(sender, "flush");
+ // This case also verifies receiver auto-creation from the sealed file's schema.
+ sql(receiver, "drop timeseries " + f.device + ".*");
+ try (PipeTransferGate.Hold seal = gate.holdNext(PipeTransferGate::isSeal, true)) {
+ f.create(gate, "", ", 'sink.format'='tsfile'");
+ seal.awaitReached();
+ f.assertRows(0, 16);
+ final int sealsBeforeRetry = gate.completedSeals();
+ seal.disconnect();
+ await().until(() -> gate.completedSeals() > sealsBeforeRetry);
+ }
+ f.assertRows(0, 16);
+ awaitDrained(f.name);
+ f.assertSchema();
+ }
+ }
+
+ @Test
+ public void testBlockedPipeDoesNotStarveHealthyPipeAndDropReclaimsOwnedMemory() throws Exception {
+ // Both Pipes share sender tasks/receiver runtime; only one has an unacknowledged batch.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture blocked = new Fixture("blocked", gate);
+ Fixture healthy = new Fixture("healthy", null)) {
+ blocked.create(gate, "", "");
+ healthy.create(null, "", "");
+ blocked.write(0, 1);
+ blocked.assertRows(0, 1);
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ blocked.write(1, 2);
+ ack.awaitReached();
+ blocked.write(2, 8);
+ blocked.assertBacklog(0, 8, 0, 2);
+ await()
+ .untilAsserted(
+ () -> assertTrue(hasOwnedMemory(blocked.name, BaseEnv.TREE_SQL_DIALECT)));
+ healthy.write(0, 8);
+ healthy.assertRows(0, 8);
+ awaitDrained(healthy.name);
+ operateWhileHeld(
+ "drop pipe " + blocked.name,
+ ack,
+ () -> assertEquals("PRE_DELETE", pipe(blocked.name).getState()));
+ }
+ awaitNoPipeResources(blocked.name);
+ healthy.write(8, 12);
+ healthy.assertRows(0, 12);
+ awaitDrained(healthy.name);
+ }
+ }
+
+ @Test
+ public void testTreeSchemaConflictDoesNotBlockSubsequentOverwriteDeleteAndSchemaChanges()
+ throws Exception {
+ // Disable type conversion: a real schema conflict must be observable without silently
+ // converting the sender's INT64 value into the receiver's BOOLEAN series.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("tree_schema", gate)) {
+ sql(receiver, "create timeseries " + f.device + ".extra BOOLEAN");
+ f.create(
+ gate,
+ ", 'source.inclusion'='data,schema'",
+ ", 'sink.exception.data.convert-on-type-mismatch'='false'");
+ f.write(0, 2);
+ f.assertRows(0, 2);
+ awaitDrained(f.name);
+ final int ignoredConflict =
+ TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode();
+ final int rejectedSchemaBefore =
+ gate.rejectedRequests(
+ type -> type == PipeRequestType.TRANSFER_PLAN_NODE.getType(), ignoredConflict);
+ final int rejectedDataBefore =
+ gate.rejectedRequests(PipeTransferGate::isTablet, ignoredConflict);
+ sql(sender, "create timeseries " + f.device + ".extra INT64 tags(stage=old)");
+ await()
+ .until(
+ () ->
+ gate.rejectedRequests(
+ type -> type == PipeRequestType.TRANSFER_PLAN_NODE.getType(),
+ ignoredConflict)
+ > rejectedSchemaBefore);
+ sql(sender, "insert into " + f.device + "(time,extra) values(3,303)");
+ // Schema and data use independent sinks. Both requests must encounter the conflict before
+ // deleting the receiver series; otherwise a delayed INSERT can auto-create it and let the
+ // following ALTER succeed without exercising the missing-schema retry boundary.
+ await()
+ .until(
+ () ->
+ gate.rejectedRequests(PipeTransferGate::isTablet, ignoredConflict)
+ > rejectedDataBefore);
+ // A path-exists/type-mismatch response is an idempotent receiver conflict and the original
+ // DDL/DML is intentionally not replayed after the receiver schema is deleted. A subsequent
+ // alter of a missing series is a retryable user conflict, which gives us a deterministic
+ // recovery boundary for the same Pipe generation.
+ sql(receiver, "drop timeseries " + f.device + ".extra");
+ final int retryableConflict =
+ TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode();
+ final int rejectedBeforeAlter = gate.rejectedRequests(retryableConflict);
+ sql(
+ sender,
+ "alter timeseries "
+ + f.device
+ + ".extra upsert alias=renamed tags(stage=new) attributes(unit=ms)");
+ await().until(() -> gate.rejectedRequests(retryableConflict) > rejectedBeforeAlter);
+
+ // Repair the receiver with a compatible schema. The pending ALTER can now be retried on the
+ // same durable Pipe generation; the original idempotent-conflict INSERT is sent again below.
+ sql(receiver, "create timeseries " + f.device + ".extra INT64 tags(stage=repair)");
+ if ("STOPPED".equals(pipe(f.name).getState())) {
+ sql(sender, "start pipe " + f.name);
+ }
+ sql(sender, "insert into " + f.device + "(time,extra) values(3,303)");
+ awaitRows(
+ receiver,
+ "select extra from " + f.device,
+ new String[] {"Time", f.device + ".extra"},
+ Collections.singletonList("3,303,"));
+ sql(sender, "insert into " + f.device + "(time,v,flag) values(1,999,false)");
+ sql(sender, "delete from " + f.device + ".v where time=0");
+ awaitRows(
+ receiver,
+ "select v,flag from " + f.device + " where time<=1",
+ new String[] {"Time", f.device + ".v", f.device + ".flag"},
+ Arrays.asList("0,null,true,", "1,999,false,"));
+ await()
+ .untilAsserted(
+ () -> {
+ try (Connection c = connection(receiver);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show timeseries " + f.device + ".extra")) {
+ assertTrue(r.next());
+ assertEquals("INT64", r.getString("DataType"));
+ assertEquals("renamed", r.getString("Alias"));
+ assertNotNull(r.getString("Tags"));
+ assertNotNull(r.getString("Attributes"));
+ assertEquals(
+ JsonParser.parseString("{\"stage\":\"new\"}"),
+ JsonParser.parseString(r.getString("Tags")));
+ assertEquals(
+ JsonParser.parseString("{\"unit\":\"ms\"}"),
+ JsonParser.parseString(r.getString("Attributes")));
+ assertFalse(r.next());
+ }
+ });
+ awaitDrained(f.name);
+ }
+ }
+
+ @Test
+ public void testTableSchemaChangesAfterHeldBatchPreserveColumnsAndFinalRows() throws Exception {
+ // Table column lifecycle is distinct from Tree aliases; queued DDL follows a real in-flight
+ // row.
+ final String database = "p1_table_schema";
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("table_schema", gate);
+ Cleanup cleanup =
+ new Cleanup(
+ () -> tableSql(sender, "drop pipe if exists " + f.name),
+ () -> awaitNoPipeResources(f.name),
+ () -> tableSql(sender, "drop database if exists " + database),
+ () -> tableSql(receiver, "drop database if exists " + database))) {
+ tableSql(sender, "create database " + database);
+ tableSql(receiver, "create database " + database);
+ tableSql(sender, database, "create table t(id STRING TAG, v INT64 FIELD)");
+ tableSql(
+ sender,
+ database,
+ "create pipe "
+ + f.name
+ + " with source ('source.database-name'='"
+ + database
+ + "', 'source.table-name'='t', 'source.inclusion'='data,schema', 'source.realtime.mode'='log')"
+ + sink(gate, ""));
+ tableSql(sender, database, "insert into t(time,id,v) values(0,'business0',10)");
+ awaitTableRows(
+ database, "time,id,v", Arrays.asList("1970-01-01T00:00:00.000Z,business0,10,"));
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ tableSql(sender, database, "insert into t(time,id,v) values(1,'business1',11)");
+ ack.awaitReached();
+ tableSql(sender, database, "alter table t add column extra INT32 FIELD");
+ tableSql(sender, database, "insert into t(time,id,v,extra) values(2,'business2',12,102)");
+ awaitTableQuery(
+ sender,
+ database,
+ "select time,id,v,extra from t order by time,id",
+ new String[] {"time", "id", "v", "extra"},
+ Arrays.asList(
+ "1970-01-01T00:00:00.000Z,business0,10,null,",
+ "1970-01-01T00:00:00.001Z,business1,11,null,",
+ "1970-01-01T00:00:00.002Z,business2,12,102,"));
+ awaitTableRows(
+ database,
+ "time,id,v",
+ Arrays.asList(
+ "1970-01-01T00:00:00.000Z,business0,10,",
+ "1970-01-01T00:00:00.001Z,business1,11,"));
+ }
+ awaitTableQuery(
+ database,
+ "desc t",
+ new String[] {"ColumnName", "DataType", "Category"},
+ Arrays.asList(
+ "time,TIMESTAMP,TIME,", "id,STRING,TAG,", "v,INT64,FIELD,", "extra,INT32,FIELD,"));
+ awaitTableRows(
+ database,
+ "time,id,v,extra",
+ Arrays.asList(
+ "1970-01-01T00:00:00.000Z,business0,10,null,",
+ "1970-01-01T00:00:00.001Z,business1,11,null,",
+ "1970-01-01T00:00:00.002Z,business2,12,102,"));
+ tableSql(sender, database, "alter table t drop column extra");
+ tableSql(sender, database, "insert into t(time,id,v) values(1,'business1',999)");
+ tableSql(sender, database, "delete from t where time=0");
+ awaitTableRows(
+ database,
+ "time,id,v",
+ Arrays.asList(
+ "1970-01-01T00:00:00.001Z,business1,999,", "1970-01-01T00:00:00.002Z,business2,12,"));
+ awaitTableQuery(
+ database,
+ "desc t",
+ new String[] {"ColumnName", "DataType", "Category"},
+ Arrays.asList("time,TIMESTAMP,TIME,", "id,STRING,TAG,", "v,INT64,FIELD,"));
+ awaitDrained(f.name, BaseEnv.TABLE_SQL_DIALECT);
+ }
+ }
+
+ @Test
+ public void testRegionRedistributionAfterScaleOutPreservesInflightAndPendingRows()
+ throws Exception {
+ // Registration alone is insufficient: prove this database's populated Region changes owner.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("region_move", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final Map businessRegion = dataRegionOwners(f.database);
+ assertFalse("The business device must have an assigned DataRegion", businessRegion.isEmpty());
+ final int region = businessRegion.keySet().iterator().next();
+ final int oldOwner = businessRegion.get(region);
+ final List regionDevices = new ArrayList<>();
+ for (int i = 0; i < 32 && dataRegionOwners(f.database).size() < 2; i++) {
+ final String device = f.database + ".region_" + i;
+ regionDevices.add(device);
+ sql(sender, "create timeseries " + device + ".v INT64 encoding=PLAIN");
+ sql(
+ sender,
+ "insert into " + device + "(time,v) values(" + (100 + i) + "," + (17000 + i) + ")");
+ }
+ final Map before = dataRegionOwners(f.database);
+ assertTrue("The fixture must populate at least two DataRegions", before.size() >= 2);
+ for (int i = 0; i < regionDevices.size(); i++) {
+ awaitRows(
+ receiver,
+ "select v from " + regionDevices.get(i),
+ new String[] {"Time", regionDevices.get(i) + ".v"},
+ Collections.singletonList((100 + i) + "," + (17000 + i) + ","));
+ }
+ try (TemporaryDataNode added = new TemporaryDataNode(before, f.database)) {
+ added.register();
+ final int newOwner = added.getNodeId();
+ assertNotEquals(oldOwner, newOwner);
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ for (int i = 0; i < regionDevices.size(); i++) {
+ sql(
+ sender,
+ "insert into "
+ + regionDevices.get(i)
+ + "(time,v) values("
+ + (1000 + i)
+ + ","
+ + (27000 + i)
+ + ")");
+ }
+ sql(sender, "migrate region " + region + " from " + oldOwner + " to " + newOwner);
+ await()
+ .untilAsserted(
+ () ->
+ assertEquals(
+ Integer.valueOf(newOwner), dataRegionOwners(f.database).get(region)));
+ f.write(8, 12);
+ for (int i = 0; i < regionDevices.size(); i++) {
+ sql(
+ sender,
+ "insert into "
+ + regionDevices.get(i)
+ + "(time,v) values("
+ + (2000 + i)
+ + ","
+ + (37000 + i)
+ + ")");
+ }
+ ack.disconnect();
+ }
+ f.assertRows(0, 12);
+ f.assertRows(sender, 0, 12);
+ for (int i = 0; i < regionDevices.size(); i++) {
+ for (final BaseEnv env : Arrays.asList(sender, receiver)) {
+ awaitRows(
+ env,
+ "select v from " + regionDevices.get(i),
+ new String[] {"Time", regionDevices.get(i) + ".v"},
+ Arrays.asList(
+ (100 + i) + "," + (17000 + i) + ",",
+ (1000 + i) + "," + (27000 + i) + ",",
+ (2000 + i) + "," + (37000 + i) + ","));
+ }
+ }
+ final Map after = dataRegionOwners(f.database);
+ assertEquals(before.keySet(), after.keySet());
+ for (final Map.Entry entry : before.entrySet()) {
+ final int expectedOwner = entry.getKey() == region ? newOwner : entry.getValue();
+ assertEquals(Integer.valueOf(expectedOwner), after.get(entry.getKey()));
+ awaitRegionOwnerRunning(f.database, entry.getKey(), expectedOwner);
+ }
+ awaitRows(
+ receiver,
+ "select v,flag from " + f.device + " where time=8",
+ new String[] {"Time", f.device + ".v", f.device + ".flag"},
+ Collections.singletonList("8,1136,true,"));
+ awaitDrained(f.name);
+ f.assertSchema();
+ awaitRegionOwnerRunning(f.database, region, newOwner);
+ }
+ }
+ }
+
+ @Test
+ public void testAggregateProcessorRestartAfterLostAckRestoresOpenWindowExactly()
+ throws Exception {
+ // Closing one window persists the next open window. Crash with the second output
+ // unacknowledged.
+ final String output = "root.p1_aggregate_output";
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("aggregate_restart", gate);
+ Cleanup cleanup =
+ new Cleanup(
+ () -> sql(sender, "drop pipe if exists " + f.name),
+ () -> awaitNoPipeResources(f.name),
+ () -> sql(receiver, "delete database " + output))) {
+ sql(receiver, "create database " + output);
+ sql(receiver, "create timeseries " + output + ".business.v.mean DOUBLE encoding=PLAIN");
+ sql(
+ sender,
+ "create pipe "
+ + f.name
+ + " with source ('source.path'='"
+ + f.device
+ + ".v', 'source.realtime.mode'='log')"
+ + " with processor ('processor'='aggregate-processor', 'processor.operators'='avg',"
+ + " 'processor.output.measurements'='mean', 'processor.sliding.seconds'='10',"
+ + " 'processor.output.min-report-interval-seconds'='0', 'processor.output.database'='"
+ + output
+ + "')"
+ + sink(gate, ""));
+ final String insert = "insert into " + f.device + "(time,v) values";
+ // Keep the boundary-crossing row in its own source event so the serialized open window
+ // definitely contains the value at 10001 before the lost-ACK batch is injected.
+ sql(sender, insert + "(0,2),(1000,4)");
+ sql(sender, insert + "(10001,10)");
+ final String series = output + ".business.v.mean";
+ awaitRows(
+ receiver,
+ "select mean from " + output + ".business.v where time<10000",
+ new String[] {"Time", series},
+ Collections.singletonList("0,3.0,"));
+ awaitDrained(f.name);
+ // The aggregate processor emits the 10000 window only when the 20000 boundary is seen.
+ // The same event also serializes the next open window with its value at 20000.
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ sql(sender, insert + "(20000,30)");
+ ack.awaitReached();
+ awaitRows(
+ receiver,
+ "select mean from " + output + ".business.v where time<20000",
+ new String[] {"Time", series},
+ Arrays.asList("0,3.0,", "10000,10.0,"));
+ restartDataNode(sender, ack);
+ }
+ awaitRows(
+ receiver,
+ "select mean from " + output + ".business.v where time<20000",
+ new String[] {"Time", series},
+ Arrays.asList("0,3.0,", "10000,10.0,"));
+ if ("STOPPED".equals(pipe(f.name).getState())) {
+ sql(sender, "start pipe " + f.name);
+ }
+ awaitDrained(f.name);
+ // The new value differs from the restored one: losing or merging the restored open
+ // window twice changes 35.0 to 40.0 or 33.333..., even if the old output is overwritten.
+ sql(sender, insert + "(21000,40),(30001,50)");
+ awaitRows(
+ receiver,
+ "select mean from " + output + ".business.v where time<=20000",
+ new String[] {"Time", series},
+ Arrays.asList("0,3.0,", "10000,10.0,", "20000,35.0,"));
+ awaitRows(
+ sender,
+ "select v from " + f.device,
+ new String[] {"Time", f.device + ".v"},
+ Arrays.asList("0,2,", "1000,4,", "10001,10,", "20000,30,", "21000,40,", "30001,50,"));
+ awaitRows(
+ receiver,
+ "select mean from " + output + ".business.v",
+ new String[] {"Time", series},
+ Arrays.asList("0,3.0,", "10000,10.0,", "20000,35.0,"));
+ awaitDrained(f.name);
+ }
+ }
+
+ @Test
+ public void testForwardingDisabledBreaksReturnLoopWhilePeerPipeIsStoppedAndRecovered()
+ throws Exception {
+ // A return link receives local writes but excludes replicated events, including after restart.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ PipeTransferGate returnGate = new PipeTransferGate(sender.getDataNodeWrapper(0));
+ Fixture f = new Fixture("forward_loop", gate)) {
+ final String returning = f.name + "_return";
+ final int returnConnectionsBefore = receiverConnections(sender);
+ try (Cleanup cleanup =
+ new Cleanup(
+ () -> sql(receiver, "drop pipe if exists " + returning),
+ () -> awaitNoPipeResources(receiver, sender, returning, BaseEnv.TREE_SQL_DIALECT),
+ () ->
+ await()
+ .untilAsserted(
+ () -> {
+ assertEquals(0, returnGate.openConnections());
+ assertTrue(receiverConnections(sender) <= returnConnectionsBefore);
+ }))) {
+ f.create(gate, ", 'source.forwarding-pipe-requests'='false'", "");
+ sql(
+ receiver,
+ "create pipe "
+ + returning
+ + " with source ('source.path'='"
+ + f.database
+ + ".**', 'source.forwarding-pipe-requests'='false', 'source.realtime.mode'='log')"
+ + sink(sender, returnGate, ""));
+ final int reverseBefore = returnGate.completedDataRequests();
+ f.write(0, 2);
+ f.assertRows(0, 2);
+ awaitDrained(f.name);
+ awaitDrained(receiver, returning, BaseEnv.TREE_SQL_DIALECT);
+ assertQuietRequests(returnGate, reverseBefore);
+ // Snapshot before the fault: a leaked replay during START must not escape observation.
+ final int forwardBeforeStop = gate.completedDataRequests();
+ sql(sender, "stop pipe " + f.name);
+ sql(receiver, "insert into " + f.device + "(time,v,flag) values(10,1170,true)");
+ awaitRows(
+ sender,
+ "select v,flag from " + f.device,
+ new String[] {"Time", f.device + ".v", f.device + ".flag"},
+ Arrays.asList("0,1000,true,", "1,1017,false,", "10,1170,true,"));
+ awaitDrained(receiver, returning, BaseEnv.TREE_SQL_DIALECT);
+ assertTrue(returnGate.completedDataRequests() > reverseBefore);
+ final int reverseAfterLocalWrite = returnGate.completedDataRequests();
+ sql(sender, "start pipe " + f.name);
+ awaitDrained(f.name);
+ awaitDrained(receiver, returning, BaseEnv.TREE_SQL_DIALECT);
+ assertQuietRequests(gate, forwardBeforeStop);
+ assertQuietRequests(returnGate, reverseAfterLocalWrite);
+ f.write(2, 4);
+ f.assertRows(0, 4, 10, 11);
+ f.assertRows(sender, 0, 4, 10, 11);
+ awaitDrained(f.name);
+ awaitDrained(receiver, returning, BaseEnv.TREE_SQL_DIALECT);
+ assertQuietRequests(returnGate, reverseAfterLocalWrite);
+ f.assertSchema();
+ }
+ }
+ }
+
+ private static void assertQuietRequests(final PipeTransferGate gate, final int expected) {
+ await()
+ .during(3, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertEquals(
+ "Replicated events were forwarded again",
+ expected,
+ gate.completedDataRequests()));
+ }
+
+ @Test
+ public void testDropProcedureSurvivesConfigNodeRestartWithRealTransferOutstanding()
+ throws Exception {
+ // Restart the ConfigNode while DROP is persisted as PRE_DELETE and its DataNode operation is
+ // waiting for a real in-flight transfer to release.
+ try (PipeTransferGate gate = new PipeTransferGate(receiver.getDataNodeWrapper(0));
+ Fixture f = new Fixture("procedure_replay", gate)) {
+ f.create(gate, "", "");
+ f.write(0, 1);
+ f.assertRows(0, 1);
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ try (Cleanup cleanup =
+ new Cleanup(
+ () -> {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+ })) {
+ try (PipeTransferGate.Hold ack = gate.holdNext(PipeTransferGate::isTablet, true)) {
+ f.write(1, 2);
+ ack.awaitReached();
+ f.write(2, 8);
+ f.assertBacklog(0, 8, 0, 2);
+ final Future drop =
+ executor.submit(
+ () -> {
+ sql(sender, "drop pipe " + f.name);
+ return null;
+ });
+ await().untilAsserted(() -> assertEquals("PRE_DELETE", pipe(f.name).getState()));
+ restartConfigNode();
+ assertEquals("PRE_DELETE", pipe(f.name).getState());
+ ack.disconnect();
+ try {
+ drop.get(60, TimeUnit.SECONDS);
+ } catch (final java.util.concurrent.ExecutionException e) {
+ // Only a transport failure can be explained by losing the response during restart.
+ // Product/procedure errors must fail this test even if background cleanup later wins.
+ assertTrue("Unexpected DROP failure: " + e.getCause(), causedByTransport(e));
+ }
+ }
+ awaitNoPipeResources(f.name);
+ f.assertRows(0, 2);
+ }
+ }
+ }
+
+ private static boolean causedByTransport(final Throwable error) {
+ for (Throwable cause = error; cause != null; cause = cause.getCause()) {
+ if (cause instanceof TTransportException) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static final class TemporaryDataNode implements AutoCloseable {
+ private final int originalDataNodeCount = sender.getDataNodeWrapperList().size();
+ private final Map originalRegionOwners;
+ private final String database;
+ private DataNodeWrapper dataNode;
+ private TDataNodeLocation location;
+
+ private TemporaryDataNode(
+ final Map originalRegionOwners, final String database) {
+ this.originalRegionOwners = originalRegionOwners;
+ this.database = database;
+ }
+
+ private void register() throws Exception {
+ dataNode = sender.generateRandomDataNodeWrapper();
+ sender.registerNewDataNode(dataNode, true);
+ location = findLocation();
+ assertNotNull("The newly registered DataNode is missing from SHOW CLUSTER", location);
+ }
+
+ private int getNodeId() {
+ assertNotNull("The temporary DataNode has not been registered", location);
+ return location.getDataNodeId();
+ }
+
+ private TDataNodeLocation findLocation() throws Exception {
+ if (sender.getDataNodeWrapperList().size() <= originalDataNodeCount) {
+ return null;
+ }
+ dataNode = sender.getDataNodeWrapper(originalDataNodeCount);
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) sender.getLeaderConfigNodeConnection()) {
+ return client.showCluster().getDataNodeList().stream()
+ .filter(n -> n.getClientRpcEndPoint().getPort() == dataNode.getPort())
+ .findFirst()
+ .orElse(null);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (dataNode == null && sender.getDataNodeWrapperList().size() > originalDataNodeCount) {
+ dataNode = sender.getDataNodeWrapper(originalDataNodeCount);
+ }
+ if (dataNode == null) {
+ return;
+ }
+
+ new Cleanup(
+ this::restoreRegions,
+ this::removeFromCluster,
+ () -> dataNode.stopForcibly(),
+ () -> await().until(() -> !dataNode.isAlive()),
+ () -> {
+ assertFalse("The temporary DataNode is still alive", dataNode.isAlive());
+ dataNode.destroyDir();
+ },
+ () -> {
+ if (!dataNode.isAlive()) {
+ Files.deleteIfExists(Paths.get(EnvUtils.getLockFilePath(dataNode.getPort())));
+ sender.getDataNodeWrapperList().remove(dataNode);
+ }
+ })
+ .close();
+ }
+
+ private void restoreRegions() throws Exception {
+ if (location == null) {
+ location = findLocation();
+ }
+ if (location == null) {
+ return; // Startup failed before registration; physical resources still need cleanup.
+ }
+ final int nodeId = location.getDataNodeId();
+ final int fallbackOwner = originalRegionOwners.values().iterator().next();
+ for (final Map.Entry entry : dataRegionOwners(database).entrySet()) {
+ if (entry.getValue() == nodeId) {
+ final int targetOwner = originalRegionOwners.getOrDefault(entry.getKey(), fallbackOwner);
+ sql(
+ sender,
+ "migrate region " + entry.getKey() + " from " + nodeId + " to " + targetOwner);
+ await()
+ .untilAsserted(
+ () ->
+ assertEquals(
+ Integer.valueOf(targetOwner),
+ dataRegionOwners(database).get(entry.getKey())));
+ awaitRegionOwnerRunning(database, entry.getKey(), targetOwner);
+ }
+ }
+ }
+
+ private void removeFromCluster() throws Exception {
+ if (location == null) {
+ location = findLocation();
+ }
+ if (location == null) {
+ return;
+ }
+ final int nodeId = location.getDataNodeId();
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) sender.getLeaderConfigNodeConnection()) {
+ final org.apache.iotdb.common.rpc.thrift.TSStatus status =
+ client
+ .removeDataNode(new TDataNodeRemoveReq(Collections.singletonList(location)))
+ .getStatus();
+ assertEquals(
+ status.toString(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+ }
+ await()
+ .untilAsserted(
+ () -> {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) sender.getLeaderConfigNodeConnection()) {
+ assertTrue(
+ client.showCluster().getDataNodeList().stream()
+ .noneMatch(n -> n.getDataNodeId() == nodeId));
+ }
+ assertTrue(
+ dataRegionOwners(database).values().stream().noneMatch(id -> id == nodeId));
+ });
+ }
+ }
+
+ private static Map dataRegionOwners(final String database) throws Exception {
+ final Map owners = new HashMap<>();
+ // The scale-out fixture may have a temporary DataNode in the environment list while it is
+ // being removed. Query through the original DataNode so an intentionally stopped temporary
+ // endpoint cannot make this metadata assertion fail.
+ try (Connection c = connection(sender.getDataNodeWrapper(0), BaseEnv.TREE_SQL_DIALECT);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show regions")) {
+ while (r.next()) {
+ if (database.equals(r.getString("Database")) && "DataRegion".equals(r.getString("Type"))) {
+ if (r.getLong("TimeSlotNum") > 0) {
+ owners.put(r.getInt("RegionId"), r.getInt("DataNodeId"));
+ }
+ }
+ }
+ }
+ return owners;
+ }
+
+ private static void awaitRegionOwnerRunning(
+ final String database, final int regionId, final int expectedOwner) {
+ await()
+ .untilAsserted(
+ () -> {
+ boolean found = false;
+ try (Connection c =
+ connection(sender.getDataNodeWrapper(0), BaseEnv.TREE_SQL_DIALECT);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show regions")) {
+ while (r.next()) {
+ if (database.equals(r.getString("Database"))
+ && "DataRegion".equals(r.getString("Type"))
+ && r.getInt("RegionId") == regionId
+ && r.getLong("TimeSlotNum") > 0) {
+ found = true;
+ assertEquals(expectedOwner, r.getInt("DataNodeId"));
+ assertEquals("Running", r.getString("Status"));
+ }
+ }
+ }
+ assertTrue("Region " + regionId + " was not visible", found);
+ });
+ }
+
+ private static void operateWhileHeld(
+ final String operation,
+ final PipeTransferGate.Hold held,
+ final ThrowingRunnable persistedState)
+ throws Exception {
+ // Closing a sync sink waits for its transfer lock. Observe the ConfigNode-side transition
+ // before
+ // discarding the ACK, then require the original caller to complete successfully.
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ final Future result =
+ executor.submit(
+ () -> {
+ sql(sender, operation);
+ return null;
+ });
+ try (Cleanup cleanup =
+ new Cleanup(
+ held::close,
+ () -> {
+ executor.shutdownNow();
+ assertTrue(
+ "Pipe operation executor did not terminate",
+ executor.awaitTermination(10, TimeUnit.SECONDS));
+ })) {
+ await().untilAsserted(persistedState);
+ held.disconnect();
+ result.get(60, TimeUnit.SECONDS);
+ }
+ }
+
+ private static org.awaitility.core.ConditionFactory await() {
+ return Awaitility.await()
+ .pollInSameThread()
+ .pollInterval(1, TimeUnit.SECONDS)
+ .atMost(90, TimeUnit.SECONDS);
+ }
+
+ private static TShowPipeInfo pipe(final String name) throws Exception {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) sender.getLeaderConfigNodeConnection()) {
+ final TShowPipeResp result =
+ client.showPipe(
+ new TShowPipeReq()
+ .setPipeName(name)
+ .setWhereClause(true)
+ .setIsTableModel("p1_table_schema".equals(name))
+ .setUserName(SessionConfig.DEFAULT_USER));
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode());
+ assertEquals("Expected one generation of " + name, 1, result.getPipeInfoListSize());
+ return result.getPipeInfoList().get(0);
+ }
+ }
+
+ private static void awaitDrained(final String name) {
+ awaitDrained(name, BaseEnv.TREE_SQL_DIALECT);
+ }
+
+ private static void awaitDrained(final String name, final String dialect) {
+ awaitDrained(sender, name, dialect);
+ }
+
+ private static void awaitDrained(final BaseEnv env, final String name, final String dialect) {
+ await()
+ .during(3, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ boolean found = false;
+ for (final DataNodeWrapper dataNode : env.getDataNodeWrapperList()) {
+ try (Connection connection = connection(dataNode, dialect);
+ Statement statement = statement(connection);
+ ResultSet result = statement.executeQuery("show pipes")) {
+ while (result.next()) {
+ if (name.equals(result.getString("ID"))) {
+ found = true;
+ assertEquals("RUNNING", result.getString("State"));
+ assertEquals("0", result.getString("RemainingEventCount"));
+ }
+ }
+ }
+ }
+ assertTrue("Pipe " + name + " was not visible on any sender DataNode", found);
+ });
+ }
+
+ private static void awaitNoReceiver(final BaseEnv env, final String name) {
+ await()
+ .untilAsserted(
+ () -> {
+ try (Connection c = connection(env);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show receivers")) {
+ while (r.next()) {
+ final String ids = r.getString("PipeIDs");
+ assertTrue(ids, ids == null || !ids.contains(name + "@"));
+ }
+ }
+ });
+ }
+
+ private static boolean hasOwnedMemory(final String name, final String dialect) throws Exception {
+ return hasOwnedMemory(sender, name, dialect);
+ }
+
+ private static boolean hasOwnedMemory(final BaseEnv env, final String name, final String dialect)
+ throws Exception {
+ boolean found = false;
+ for (final DataNodeWrapper dn : env.getDataNodeWrapperList()) {
+ try (Connection c = connection(dn, dialect);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show pipe memory")) {
+ while (r.next()) {
+ found |= name.equals(r.getString("pipe_name"));
+ }
+ }
+ }
+ return found;
+ }
+
+ private static void awaitNoPipeResources(final String name) {
+ awaitNoPipeResources(
+ sender,
+ receiver,
+ name,
+ "p1_table_schema".equals(name) ? BaseEnv.TABLE_SQL_DIALECT : BaseEnv.TREE_SQL_DIALECT);
+ }
+
+ private static void awaitNoPipeResources(
+ final BaseEnv source, final BaseEnv target, final String name, final String dialect) {
+ awaitNoReceiver(target, name);
+ await()
+ .untilAsserted(
+ () -> {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) source.getLeaderConfigNodeConnection()) {
+ final TShowPipeResp result =
+ client.showPipe(
+ new TShowPipeReq()
+ .setUserName(SessionConfig.DEFAULT_USER)
+ .setIsTableModel(BaseEnv.TABLE_SQL_DIALECT.equals(dialect)));
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode());
+ assertTrue(
+ result.getPipeInfoList().stream().noneMatch(p -> name.equals(p.getId())));
+ }
+ assertFalse(
+ "Owned queue/processor/sink memory remains for " + name,
+ hasOwnedMemory(source, name, dialect));
+ });
+ }
+
+ private static void restartConfigNode() throws Exception {
+ final ConfigNodeWrapper cn = sender.getConfigNodeWrapper(0);
+ cn.stopForcibly();
+ cn.start();
+ await()
+ .untilAsserted(
+ () -> {
+ try (SyncConfigNodeIServiceClient ignored =
+ (SyncConfigNodeIServiceClient) sender.getLeaderConfigNodeConnection()) {
+ assertNotNull(ignored);
+ }
+ });
+ }
+
+ private static void restartDataNode(final BaseEnv env, final PipeTransferGate.Hold outstanding)
+ throws Exception {
+ final DataNodeWrapper dn = env.getDataNodeWrapper(0);
+ try {
+ dn.stopForcibly();
+ await().until(() -> !dn.isAlive());
+ if (outstanding != null) {
+ outstanding.disconnect();
+ }
+ } finally {
+ if (!dn.isAlive()) {
+ dn.start();
+ }
+ }
+ awaitDataNode(env, dn);
+ }
+
+ private static void awaitDataNode(final BaseEnv env, final DataNodeWrapper dn) {
+ env.ensureNodeStatus(
+ Collections.singletonList(dn), Collections.singletonList(NodeStatus.Running));
+ await()
+ .until(
+ () -> {
+ try (Connection ignored = connection(dn, BaseEnv.TREE_SQL_DIALECT)) {
+ return true;
+ } catch (SQLException e) {
+ return false;
+ }
+ });
+ }
+
+ private static void sql(final BaseEnv env, final String sql) throws SQLException {
+ execute(env, BaseEnv.TREE_SQL_DIALECT, sql);
+ }
+
+ private static void tableSql(final BaseEnv env, final String sql) throws SQLException {
+ execute(env, BaseEnv.TABLE_SQL_DIALECT, sql);
+ }
+
+ private static void tableSql(final BaseEnv env, final String database, final String sql)
+ throws SQLException {
+ try (Connection connection = connection(env, BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = statement(connection)) {
+ statement.execute("use " + database);
+ statement.execute(sql);
+ }
+ }
+
+ private static void execute(final BaseEnv env, final String dialect, final String sql)
+ throws SQLException {
+ try (Connection connection = connection(env, dialect);
+ Statement statement = statement(connection)) {
+ statement.execute(sql);
+ }
+ }
+
+ private static Connection connection(final BaseEnv env) throws SQLException {
+ return connection(env, BaseEnv.TREE_SQL_DIALECT);
+ }
+
+ private static Connection connection(final DataNodeWrapper dn, final String dialect)
+ throws SQLException {
+ // Env connections use network_timeout=0. A query deadline cannot interrupt a blocked socket
+ // (including openSession/closeSession), so give every fresh connection its own finite timeout.
+ final Properties properties =
+ BaseEnv.constructProperties(
+ SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, dialect);
+ properties.setProperty(Config.NETWORK_TIMEOUT, Integer.toString(SOCKET_TIMEOUT_MS));
+ properties.setProperty(Config.TIME_ZONE, "+00:00");
+ return DriverManager.getConnection("jdbc:iotdb://" + dn.getIpAndPortString() + "/", properties);
+ }
+
+ private static Connection connection(final BaseEnv env, final String dialect)
+ throws SQLException {
+ final List reads = new ArrayList<>();
+ final DataNodeWrapper writer = env.getDataNodeWrapper(0);
+ final Connection write = connection(writer, dialect);
+ try {
+ for (final DataNodeWrapper dn : env.getDataNodeWrapperList()) {
+ reads.add(
+ new NodeConnection(
+ dn.getIpAndPortString(),
+ NodeConnection.NodeRole.DATA_NODE,
+ NodeConnection.ConnectionRole.READ,
+ connection(dn, dialect)));
+ }
+ // Retain the standard IT wrapper and unordered assertion helper: every DataNode is checked.
+ return new ClusterTestConnection(
+ new NodeConnection(
+ writer.getIpAndPortString(),
+ NodeConnection.NodeRole.DATA_NODE,
+ NodeConnection.ConnectionRole.WRITE,
+ write),
+ reads,
+ (AbstractEnv) env);
+ } catch (final SQLException | RuntimeException | Error e) {
+ try {
+ final List close = new ArrayList<>();
+ close.add(write::close);
+ reads.forEach(n -> close.add(() -> n.getUnderlyingConnection().close()));
+ new Cleanup(close.toArray(new ThrowingRunnable[0])).close();
+ } catch (final Exception | Error cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ throw e;
+ }
+ }
+
+ private static Statement statement(final Connection connection) throws SQLException {
+ final Statement statement = connection.createStatement();
+ try {
+ statement.setQueryTimeout(QUERY_TIMEOUT_SECONDS);
+ return statement;
+ } catch (final SQLException e) {
+ try {
+ statement.close();
+ } catch (final SQLException cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ throw e;
+ }
+ }
+
+ private static int receiverConnections(final BaseEnv env) throws SQLException {
+ int count = 0;
+ for (final DataNodeWrapper dn : env.getDataNodeWrapperList()) {
+ try (Connection c = connection(dn, BaseEnv.TREE_SQL_DIALECT);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show receivers")) {
+ while (r.next()) {
+ count += r.getInt("ConnectionCount");
+ }
+ }
+ }
+ return count;
+ }
+
+ private static Set stagingFiles(final BaseEnv env) throws Exception {
+ final Set files = new java.util.HashSet<>();
+ for (final DataNodeWrapper dn : env.getDataNodeWrapperList()) {
+ // This fixture retains the default system/receiver directory under each isolated node.
+ final Path directory = Paths.get(dn.getDataNodeDir(), "system", "pipe", "receiver");
+ if (Files.exists(directory)) {
+ try (Stream paths = Files.walk(directory)) {
+ files.addAll(paths.filter(Files::isRegularFile).collect(Collectors.toSet()));
+ }
+ }
+ }
+ return files;
+ }
+
+ /** Run every cleanup and let try-with-resources preserve the original test failure. */
+ private static final class Cleanup implements AutoCloseable {
+ private final ThrowingRunnable[] actions;
+
+ private Cleanup(final ThrowingRunnable... actions) {
+ this.actions = actions;
+ }
+
+ @Override
+ public void close() throws Exception {
+ Throwable failure = null;
+ for (final ThrowingRunnable action : actions) {
+ try {
+ action.run();
+ } catch (final Throwable e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
+ if (failure instanceof Error) {
+ throw (Error) failure;
+ }
+ if (failure != null) {
+ throw (Exception) failure;
+ }
+ }
+ }
+
+ private static String sink(final PipeTransferGate gate, final String extra) {
+ return sink(receiver, gate, extra);
+ }
+
+ private static String sink(
+ final BaseEnv target, final PipeTransferGate gate, final String extra) {
+ final DataNodeWrapper dn = target.getDataNodeWrapper(0);
+ return " with sink ('sink'='iotdb-thrift-sync-sink', 'sink.ip'='"
+ + dn.getIp()
+ + "', 'sink.port'='"
+ + (gate == null ? dn.getPort() : gate.getPort())
+ + "', 'sink.batch.enable'='false', 'sink.parallel.tasks'='1', 'sink.leader-cache.enable'='false',"
+ + " 'sink.exception.others.retry-max-time-seconds'='-1',"
+ + " 'sink.exception.conflict.resolve-strategy'='retry',"
+ + " 'sink.exception.conflict.retry-max-time-seconds'='-1'"
+ + extra
+ + ")";
+ }
+
+ private static void awaitTableRows(
+ final String database, final String columns, final List expected) {
+ awaitTableQuery(
+ database, "select " + columns + " from t order by time,id", columns.split(","), expected);
+ }
+
+ private static void awaitTableQuery(
+ final String database,
+ final String query,
+ final String[] headers,
+ final List expected) {
+ awaitTableQuery(receiver, database, query, headers, expected);
+ }
+
+ private static void awaitTableQuery(
+ final BaseEnv env,
+ final String database,
+ final String query,
+ final String[] headers,
+ final List expected) {
+ await()
+ .untilAsserted(
+ () -> {
+ try (Connection connection = connection(env, BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = statement(connection)) {
+ statement.execute("use " + database);
+ try (ResultSet result = statement.executeQuery(query)) {
+ TestUtils.assertResultSetEqualUnordered(
+ result, String.join(",", headers) + ",", expected);
+ }
+ }
+ });
+ }
+
+ private static void awaitRows(
+ final BaseEnv env, final String query, final String[] headers, final List expected) {
+ awaitRows(env, BaseEnv.TREE_SQL_DIALECT, query, headers, expected);
+ }
+
+ private static void awaitRows(
+ final BaseEnv env,
+ final String dialect,
+ final String query,
+ final String[] headers,
+ final List expected) {
+ await()
+ .alias("Rows on " + (env == sender ? "sender" : "receiver") + ": " + query)
+ .untilAsserted(
+ () -> {
+ try (Connection connection = connection(env, dialect);
+ Statement statement = statement(connection);
+ ResultSet result = statement.executeQuery(query)) {
+ TestUtils.assertResultSetEqualUnordered(
+ result, String.join(",", headers) + ",", expected);
+ }
+ });
+ }
+
+ private static final class Fixture implements AutoCloseable {
+ private final String name;
+ private final String database;
+ private final String device;
+ private final PipeTransferGate gate;
+ private final int receiverConnectionsBefore;
+ private final Set stagingFilesBefore;
+
+ private Fixture(final String suffix, final PipeTransferGate gate) throws Exception {
+ name = "p1_" + suffix;
+ database = "root." + name;
+ device = database + ".business";
+ this.gate = gate;
+ receiverConnectionsBefore = receiverConnections(receiver);
+ stagingFilesBefore = stagingFiles(receiver);
+ // Explicit compatible schema prevents auto-create privileges from confounding permission ITs.
+ final List rollback = new ArrayList<>();
+ try {
+ for (final BaseEnv env : Arrays.asList(sender, receiver)) {
+ sql(env, "create database " + database);
+ rollback.add(() -> sql(env, "delete database " + database));
+ sql(env, "create timeseries " + device + ".v INT64 encoding=PLAIN");
+ sql(env, "create timeseries " + device + ".flag BOOLEAN encoding=PLAIN");
+ }
+ } catch (final SQLException | RuntimeException | Error e) {
+ try {
+ new Cleanup(rollback.toArray(new ThrowingRunnable[0])).close();
+ } catch (final Exception | Error cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ throw e;
+ }
+ }
+
+ private void create(final PipeTransferGate gate, final String source, final String sink)
+ throws SQLException {
+ sql(
+ sender,
+ "create pipe "
+ + name
+ + " with source ('source.path'='"
+ + database
+ + ".**'"
+ + (source.contains("source.realtime.mode") ? "" : ", 'source.realtime.mode'='log'")
+ + (source.contains("source.inclusion") ? "" : ", 'source.inclusion'='data'")
+ + source
+ + ")"
+ + sink(gate, sink));
+ }
+
+ private void write(final int from, final int to) throws SQLException {
+ try (Connection c = connection(sender);
+ Statement s = statement(c)) {
+ for (int id = from; id < to; id++) {
+ s.execute(
+ "insert into "
+ + device
+ + "(time,v,flag) values("
+ + id
+ + ","
+ + (1000 + 17 * id)
+ + ","
+ + (id % 2 == 0)
+ + ")");
+ }
+ }
+ }
+
+ private void assertRows(final long... ranges) {
+ assertRows(receiver, ranges);
+ }
+
+ private void assertBacklog(final int from, final int to, final long... visibleRanges) {
+ // A held successful response (or a rejected request) proves the uncommitted transfer.
+ // Check the durable sender writes and the receiver gap directly: SHOW PIPES estimates
+ // exclude some source/assigner queues and are not a barrier for an in-flight request.
+ assertRows(sender, from, to);
+ assertRows(visibleRanges);
+ }
+
+ private void assertRows(final BaseEnv env, final long... ranges) {
+ final List expected = new ArrayList<>();
+ for (int i = 0; i < ranges.length; i += 2) {
+ for (long id = ranges[i]; id < ranges[i + 1]; id++) {
+ expected.add(id + "," + (1000 + 17 * id) + "," + (id % 2 == 0) + ",");
+ }
+ }
+ awaitRows(
+ env,
+ "select v,flag from " + device,
+ new String[] {"Time", device + ".v", device + ".flag"},
+ expected);
+ }
+
+ private void assertSchema() {
+ await()
+ .untilAsserted(
+ () -> {
+ try (Connection c = connection(receiver);
+ Statement s = statement(c);
+ ResultSet r = s.executeQuery("show timeseries " + device + ".*")) {
+ final Map actual = new HashMap<>();
+ while (r.next()) {
+ assertEquals(
+ null, actual.put(r.getString("Timeseries"), r.getString("DataType")));
+ assertEquals("PLAIN", r.getString("Encoding"));
+ }
+ assertEquals(Map.of(device + ".v", "INT64", device + ".flag", "BOOLEAN"), actual);
+ }
+ });
+ }
+
+ @Override
+ public void close() throws Exception {
+ new Cleanup(
+ () -> sql(sender, "drop pipe if exists " + name),
+ () -> awaitNoPipeResources(name),
+ () -> {
+ if (gate != null) {
+ // Check while the relay is still open: closing the test socket must not hide a
+ // leak.
+ await()
+ .untilAsserted(
+ () -> {
+ assertEquals("Pipe left live relay sockets", 0, gate.openConnections());
+ assertTrue(
+ "Receiver connections did not return to baseline",
+ receiverConnections(receiver) <= receiverConnectionsBefore);
+ });
+ }
+ },
+ () ->
+ await()
+ .untilAsserted(
+ () -> {
+ final Set residue = stagingFiles(receiver);
+ residue.removeAll(stagingFilesBefore);
+ assertTrue(
+ "New receiver staging files remain after DROP: " + residue,
+ residue.isEmpty());
+ }),
+ () -> sql(sender, "delete database " + database),
+ () -> sql(receiver, "delete database " + database))
+ .close();
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeTransferFaultInjector.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeTransferFaultInjector.java
new file mode 100644
index 0000000000000..09abf115b1ede
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeTransferFaultInjector.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced;
+
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.commons.service.NoopServerContext;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.IClientRPCService;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TMessage;
+import org.apache.thrift.protocol.TMessageType;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.server.ServerContext;
+import org.apache.thrift.server.TServerEventHandler;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.Assert;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.awaitility.Awaitility.await;
+
+/** A real Thrift relay with deterministic failure boundaries, without changing the receiver. */
+final class PipeTransferFaultInjector implements AutoCloseable {
+
+ enum Mode {
+ PASS,
+ PAUSE_BEFORE_FORWARD,
+ REJECT_BEFORE_FORWARD,
+ LOSE_RESPONSE
+ }
+
+ private final DataNodeWrapper receiver;
+ private final TServerSocket socket;
+ private final TThreadPoolServer server;
+ // TThreadPoolServer retains a worker for each open connection, including idle sink clients.
+ // A fixed pool can leave new connections waiting forever behind those idle connections.
+ private final ExecutorService executor = Executors.newCachedThreadPool();
+ private final Map clients = new ConcurrentHashMap<>();
+ private final Set senderTransports = ConcurrentHashMap.newKeySet();
+ private final AtomicInteger pausedRequests = new AtomicInteger();
+ private final AtomicInteger rejectedRequests = new AtomicInteger();
+ private final AtomicInteger lostResponses = new AtomicInteger();
+ private final AtomicInteger committedResponses = new AtomicInteger();
+ private final AtomicInteger forwardingFailures = new AtomicInteger();
+ private final AtomicInteger successfulDataRequests = new AtomicInteger();
+ private final Map failedRequests = new ConcurrentHashMap<>();
+ private final CountDownLatch release = new CountDownLatch(1);
+ private final CountDownLatch releaseResponseLoss = new CountDownLatch(1);
+ private final Thread serverThread;
+ private volatile Mode mode;
+
+ PipeTransferFaultInjector(final DataNodeWrapper receiver, final Mode mode) throws Exception {
+ this.receiver = receiver;
+ this.mode = mode;
+ socket = new TServerSocket(new InetSocketAddress(receiver.getIp(), 0));
+ server =
+ new TThreadPoolServer(
+ new TThreadPoolServer.Args(socket)
+ .processor(this::process)
+ .protocolFactory(new TBinaryProtocol.Factory())
+ .transportFactory(DeepCopyRpcTransportFactory.INSTANCE)
+ .executorService(executor));
+ server.setServerEventHandler(
+ new TServerEventHandler() {
+ @Override
+ public void preServe() {}
+
+ @Override
+ public ServerContext createContext(final TProtocol input, final TProtocol output) {
+ senderTransports.add(input.getTransport());
+ return NoopServerContext.INSTANCE;
+ }
+
+ @Override
+ public void deleteContext(
+ final ServerContext context, final TProtocol input, final TProtocol output) {
+ // Keep receiver sessions paired with sender connections, including injected failures.
+ final IoTDBSyncClient client = clients.remove(input.getTransport());
+ if (client != null) {
+ client.invalidate();
+ }
+ senderTransports.remove(input.getTransport());
+ }
+
+ @Override
+ public void processContext(
+ final ServerContext context, final TTransport input, final TTransport output) {}
+ });
+ serverThread = new Thread(server::serve, "pipe-transfer-fault-injector");
+ serverThread.setDaemon(true);
+ serverThread.start();
+ try {
+ await().atMost(10, TimeUnit.SECONDS).until(server::isServing);
+ } catch (final RuntimeException | Error e) {
+ try {
+ close();
+ } catch (final Exception cleanupFailure) {
+ e.addSuppressed(cleanupFailure);
+ }
+ throw e;
+ }
+ }
+
+ String getEndpoint() {
+ return receiver.getIp() + ":" + socket.getServerSocket().getLocalPort();
+ }
+
+ void passThrough() {
+ mode = Mode.PASS;
+ release.countDown();
+ releaseResponseLoss.countDown();
+ }
+
+ void allowResponseLoss() {
+ releaseResponseLoss.countDown();
+ }
+
+ void setMode(final Mode mode) {
+ this.mode = mode;
+ }
+
+ int getPausedRequests() {
+ return pausedRequests.get();
+ }
+
+ int getRejectedRequests() {
+ return rejectedRequests.get();
+ }
+
+ int getLostResponses() {
+ return lostResponses.get();
+ }
+
+ int getCommittedResponses() {
+ return committedResponses.get();
+ }
+
+ int getForwardingFailures() {
+ return forwardingFailures.get();
+ }
+
+ int getSuccessfulDataRequests() {
+ return successfulDataRequests.get();
+ }
+
+ boolean hasRetriedFailedRequest() {
+ return failedRequests.values().stream().anyMatch(attempts -> attempts.get() > 1);
+ }
+
+ private void recordFailedRequest(final TPipeTransferReq request) {
+ // Serialization may advance body buffers; immutable copies preserve the retry identity.
+ failedRequests
+ .computeIfAbsent(request.deepCopy(), ignored -> new AtomicInteger())
+ .incrementAndGet();
+ }
+
+ private void process(final TProtocol input, final TProtocol output) throws TException {
+ final TMessage message = input.readMessageBegin();
+ if (!"pipeTransfer".equals(message.name)) {
+ throw new TException("Only pipeTransfer is supported by this test relay");
+ }
+ final IClientRPCService.pipeTransfer_args args = new IClientRPCService.pipeTransfer_args();
+ args.read(input);
+ input.readMessageEnd();
+ final TPipeTransferReq request = args.getReq();
+ final TPipeTransferReq requestIdentity = request.deepCopy();
+ // Pause/reject data only. Handshake and schema requests must still reach the real receiver.
+ final boolean dataRequest =
+ request.getType() >= PipeRequestType.TRANSFER_TABLET_INSERT_NODE.getType()
+ && request.getType() <= PipeRequestType.TRANSFER_TABLET_BATCH_V2.getType();
+ final Mode requestMode = mode;
+ if (dataRequest && requestMode == Mode.PAUSE_BEFORE_FORWARD) {
+ pausedRequests.incrementAndGet();
+ try {
+ if (!release.await(120, TimeUnit.SECONDS)) {
+ throw new TTransportException("Timed out waiting for the test to release a data request");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TTransportException(e);
+ }
+ }
+ if (dataRequest && requestMode == Mode.REJECT_BEFORE_FORWARD) {
+ rejectedRequests.incrementAndGet();
+ recordFailedRequest(requestIdentity);
+ input.getTransport().close();
+ throw new TTransportException("Injected failure before forwarding the data request");
+ }
+
+ IoTDBSyncClient client = clients.get(input.getTransport());
+ final TPipeTransferResp response;
+ try {
+ if (client == null) {
+ client =
+ new IoTDBSyncClient(
+ new ThriftClientProperty.Builder().setConnectionTimeoutMs(5000).build(),
+ receiver.getIp(),
+ receiver.getPort(),
+ false,
+ null,
+ null);
+ clients.put(input.getTransport(), client);
+ }
+ response = client.pipeTransfer(request);
+ } catch (final TException e) {
+ forwardingFailures.incrementAndGet();
+ if (client != null) {
+ client.invalidate();
+ clients.remove(input.getTransport());
+ }
+ throw e;
+ }
+ final boolean successfulDataRequest =
+ dataRequest
+ && response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ if (successfulDataRequest) {
+ successfulDataRequests.incrementAndGet();
+ }
+ if (successfulDataRequest && requestMode == Mode.LOSE_RESPONSE) {
+ // Hold the successful response until the test has flushed and reconciled the receiver.
+ // Only then lose the response, making the persisted-write/absent-ACK boundary deterministic.
+ committedResponses.incrementAndGet();
+ try {
+ if (!releaseResponseLoss.await(120, TimeUnit.SECONDS)) {
+ throw new TTransportException("Timed out waiting to lose a committed response");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TTransportException(e);
+ }
+ lostResponses.incrementAndGet();
+ recordFailedRequest(requestIdentity);
+ input.getTransport().close();
+ throw new TTransportException("Injected response loss after receiver success");
+ }
+ final IClientRPCService.pipeTransfer_result result =
+ new IClientRPCService.pipeTransfer_result().setSuccess(response);
+ output.writeMessageBegin(new TMessage(message.name, TMessageType.REPLY, message.seqid));
+ result.write(output);
+ output.writeMessageEnd();
+ output.getTransport().flush();
+ }
+
+ @Override
+ public void close() throws Exception {
+ passThrough();
+ server.stop();
+ socket.close();
+ for (final TTransport senderTransport : senderTransports) {
+ senderTransport.close();
+ }
+ executor.shutdownNow();
+ try {
+ serverThread.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("Test relay did not stop", serverThread.isAlive());
+ Assert.assertTrue(
+ "Test relay workers did not stop", executor.awaitTermination(10, TimeUnit.SECONDS));
+ } finally {
+ clients.values().forEach(IoTDBSyncClient::invalidate);
+ clients.clear();
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
index 0f29879caad79..543ac1e03615e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.relational.it.db.it;
+import org.apache.iotdb.db.it.utils.DeletionTestUtils;
import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.isession.ITableSession;
import org.apache.iotdb.isession.SessionConfig;
@@ -36,6 +37,7 @@
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.write.record.Tablet;
@@ -68,6 +70,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -458,6 +461,119 @@ public void testDeleteDataByAttributeFilterWithTagAndTimeRange() throws SQLExcep
}
}
+ @Test
+ public void testAttributeDeleteScopeDoesNotDriftAfterAttributeUpdate() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ connection.setClientInfo("time_zone", "+00:00");
+ statement.execute("use test");
+ statement.execute(
+ "CREATE TABLE delete_attr_stable(device_id STRING TAG, site STRING TAG, "
+ + "color STRING ATTRIBUTE, value INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO delete_attr_stable(time,device_id,site,color,value) VALUES "
+ + "(1,'d1','north','red',101),(2,'d1','north','red',102),"
+ + "(1,'d1','south','red',201),(2,'d1','south','red',202),"
+ + "(1,'d2','north','blue',301),(2,'d2','north','blue',302),"
+ + "(1,'d3','','blue',501),(2,'d3','','blue',502)");
+ statement.execute(
+ "INSERT INTO delete_attr_stable(time,device_id,site,value) "
+ + "VALUES (1,'d3',null,401),(2,'d3',null,402)");
+ statement.execute(
+ "CREATE TABLE delete_attr_keep(device_id STRING TAG, site STRING TAG, "
+ + "color STRING ATTRIBUTE, value INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO delete_attr_keep(time,device_id,site,color,value) "
+ + "VALUES (1,'d1','north','red',901),(2,'d1','north','red',902)");
+ statement.execute("FLUSH");
+ DeletionTestUtils.assertSharedTsFile(
+ "test",
+ IDeviceID.Factory.DEFAULT_FACTORY.create(
+ new String[] {"delete_attr_stable", "d1", "north"}));
+ List flushedSequenceFiles = DeletionTestUtils.getClosedTsFiles("test", true);
+ List flushedUnsequenceFiles = DeletionTestUtils.getClosedTsFiles("test", false);
+ statement.execute(
+ "INSERT INTO delete_attr_stable(time,device_id,site,value) VALUES "
+ + "(3,'d1','north',103),(3,'d1','south',203),(3,'d2','north',303),"
+ + "(3,'d3',null,403),(3,'d3','',503)");
+ assertEquals(flushedSequenceFiles, DeletionTestUtils.getClosedTsFiles("test", true));
+ assertEquals(flushedUnsequenceFiles, DeletionTestUtils.getClosedTsFiles("test", false));
+
+ statement.execute(
+ "DELETE FROM delete_attr_stable WHERE color='red' AND site='north' AND time <= 2");
+ assertAttributeDeleteRows(statement, false, false);
+
+ // Change both sides of the original predicate after DELETE has completed. The original
+ // matching devices remain fixed, while previously unmatched devices keep their old rows.
+ statement.execute(
+ "UPDATE delete_attr_stable SET color='blue' WHERE device_id='d1' AND site='north'");
+ statement.execute(
+ "UPDATE delete_attr_stable SET color='red' WHERE device_id='d2' AND site='north'");
+ statement.execute(
+ "UPDATE delete_attr_stable SET color='red' WHERE device_id='d3' AND site IS NULL");
+ assertAttributeDeleteRows(statement, true, false);
+ statement.execute(
+ "INSERT INTO delete_attr_stable(time,device_id,site,color,value) "
+ + "VALUES (2,'d1','north','blue',1202)");
+ statement.execute("FLUSH");
+ assertAttributeDeleteRows(statement, true, true);
+ }
+ }
+
+ private static void assertAttributeDeleteRows(
+ Statement statement, boolean attributesUpdated, boolean rewritten) throws SQLException {
+ List expected = new ArrayList<>();
+ if (rewritten) {
+ expected.add("1970-01-01T00:00:00.002Z,d1,north,blue,1202,");
+ }
+ expected.add(
+ "1970-01-01T00:00:00.003Z,d1,north," + (attributesUpdated ? "blue" : "red") + ",103,");
+ for (int time = 1; time <= 3; time++) {
+ expected.add("1970-01-01T00:00:00.00" + time + "Z,d1,south,red," + (200 + time) + ",");
+ }
+ for (int time = 1; time <= 3; time++) {
+ expected.add(
+ "1970-01-01T00:00:00.00"
+ + time
+ + "Z,d2,north,"
+ + (attributesUpdated ? "red" : "blue")
+ + ","
+ + (300 + time)
+ + ",");
+ }
+ for (int time = 1; time <= 3; time++) {
+ expected.add(
+ "1970-01-01T00:00:00.00"
+ + time
+ + "Z,d3,null,"
+ + (attributesUpdated ? "red" : "null")
+ + ","
+ + (400 + time)
+ + ",");
+ }
+ for (int time = 1; time <= 3; time++) {
+ expected.add("1970-01-01T00:00:00.00" + time + "Z,d3,,blue," + (500 + time) + ",");
+ }
+ try (ResultSet rows =
+ statement.executeQuery(
+ "SELECT time,device_id,site,color,value FROM delete_attr_stable "
+ + "ORDER BY device_id,site NULLS FIRST,time")) {
+ TestUtils.assertResultSetEqual(
+ rows, "time,device_id,site,color,value,", expected.toArray(new String[0]));
+ }
+ try (ResultSet rows =
+ statement.executeQuery(
+ "SELECT time,device_id,site,color,value FROM delete_attr_keep ORDER BY time")) {
+ TestUtils.assertResultSetEqual(
+ rows,
+ "time,device_id,site,color,value,",
+ new String[] {
+ "1970-01-01T00:00:00.001Z,d1,north,red,901,",
+ "1970-01-01T00:00:00.002Z,d1,north,red,902,"
+ });
+ }
+ }
+
@Test
public void testDeleteDataByAttributeFilterWithOrAndNull() throws SQLException {
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
@@ -948,6 +1064,67 @@ public void testDeleteDataByAttributeFilterRejectsTooManyDevices() throws SQLExc
}
}
+ @Test
+ public void testDeleteRangesPreservePartitionGapsAndCompleteTagIdentity() throws SQLException {
+ final long[] times = {-1, 0, 1, 998, 999, 1000, 1001, 1002, 1003, 1998, 1999, 2000, 2001};
+ final Set deleted = Set.of(-1L, 999L, 1000L, 1001L, 1002L, 1999L, 2000L);
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE test");
+ statement.execute(
+ "CREATE TABLE delete_boundary(tag1 STRING TAG,tag2 STRING TAG,s1 INT64 FIELD,s2 INT64 FIELD)");
+ for (long time : times) {
+ for (String tag : new String[] {"null", "''"}) {
+ statement.execute(
+ "INSERT INTO delete_boundary(time,tag1,tag2,s1,s2) VALUES("
+ + time
+ + ",'a',"
+ + tag
+ + ","
+ + (time + 100)
+ + ","
+ + (time + 10000)
+ + ")");
+ }
+ }
+ statement.execute("FLUSH");
+ statement.execute(
+ "INSERT INTO delete_boundary(time,tag1,s1) VALUES(999,'a',9999),(1001,'a',10001)");
+ final String delete = "DELETE FROM delete_boundary WHERE tag1='a' AND tag2 IS NULL AND ";
+ statement.execute(delete + "time < 0");
+ statement.execute(delete + "time >= 999 AND time <= 1000");
+ statement.execute(delete + "time >= 1000 AND time <= 1001");
+ statement.execute(delete + "time = 1002");
+ statement.execute(delete + "time > 1998 AND time < 2001");
+ statement.execute("INSERT INTO delete_boundary(time,tag1,s1) VALUES(1500,'a',15000)");
+ statement.execute(delete + "time >= 999 AND time <= 1000");
+
+ final List expected = new ArrayList<>();
+ for (long time : times) {
+ expected.add(time + ",a,," + (time + 100) + "," + (time + 10000) + ",");
+ if (!deleted.contains(time)) {
+ expected.add(time + ",a,null," + (time + 100) + "," + (time + 10000) + ",");
+ }
+ }
+ expected.add("1500,a,null,15000,null,");
+ assertPartitionBoundaryRows(statement, expected);
+ // A rewrite after the final retry is a new row; old tombstones must not remove it.
+ statement.execute("INSERT INTO delete_boundary(time,tag1,s1,s2) VALUES(1000,'a',900,901)");
+ expected.add("1000,a,null,900,901,");
+ statement.execute("FLUSH");
+ assertPartitionBoundaryRows(statement, expected);
+ }
+ }
+
+ private static void assertPartitionBoundaryRows(Statement statement, List expected)
+ throws SQLException {
+ try (ResultSet rows =
+ statement.executeQuery(
+ "SELECT cast(time AS INT64) AS t,tag1,tag2,s1,s2 FROM delete_boundary")) {
+ TestUtils.assertResultSetEqualUnordered(rows, "t,tag1,tag2,s1,s2,", expected);
+ }
+ }
+
@Test
public void test() throws SQLException {
int testId = 2;
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java
index c4444c4bf55f8..a50f9ce17f1d5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.relational.it.db.it;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.it.utils.TsFileTableGenerator;
@@ -28,7 +32,9 @@
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
@@ -45,9 +51,11 @@
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
@@ -560,6 +568,204 @@ public void testLoadWithAllFieldsNullRows() throws Exception {
}
}
+ @Test
+ public void testQueryFallbackRestoresTableMetadataAndReadWriteAfterRestart() throws Exception {
+ final String target = "load_p1_target";
+ final String other = "load_p1_other";
+ final String table = "recovered";
+ final File source = new File(tmpDir, "610-0-0-0.tsfile");
+ final long markerOffset = writeTableRecoverySource(source, table);
+ try (final TsFileSequenceReader reader = new TsFileSequenceReader(source.getAbsolutePath())) {
+ reader.position(markerOffset);
+ Assert.assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker());
+ }
+ try (final RandomAccessFile file = new RandomAccessFile(source, "rw")) {
+ file.seek(markerOffset);
+ file.writeByte(0x7f);
+ }
+
+ try (final Connection connection =
+ EnvFactory.getEnv()
+ .getConnection(
+ EnvFactory.getEnv().getDataNodeWrapper(0),
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD,
+ BaseEnv.TABLE_SQL_DIALECT);
+ final Statement statement = connection.createStatement()) {
+ statement.execute("create database " + target);
+ statement.execute("use " + target);
+ // The source has a TAG prefix and a different FIELD order, and introduces v.
+ statement.execute(
+ "create table "
+ + table
+ + " (t1 STRING TAG,t2 STRING TAG,tail STRING TAG,n INT64 FIELD,extra INT32 FIELD)");
+ statement.execute("create database " + other);
+ statement.execute("use " + other);
+ // Disable proactive tablet conversion: the invalid later marker must fail actual splitting.
+ statement.execute(
+ String.format(
+ "load '%s' with ('database'='%s','on-success'='delete',"
+ + "'tablet-conversion-threshold'='-1')",
+ source.getAbsolutePath(), target));
+ Assert.assertTrue(
+ EnvFactory.getEnv()
+ .getDataNodeWrapper(0)
+ .logContains(
+ DataNodeQueryMessages
+ .LOAD_SUCCESSFULLY_CONVERTED_TSFILE_ARG_INTO_TABLETS_AND_INSERTED
+ .replace("{}", source.getAbsolutePath())));
+ }
+
+ // The coordinator is pinned for the log assertion; verify the result through every DataNode.
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ final Statement statement = connection.createStatement()) {
+ statement.execute("use " + other);
+ try (final ResultSet rows = statement.executeQuery("show tables")) {
+ Assert.assertFalse("LOAD must retain its explicit target database", rows.next());
+ }
+ statement.execute("use " + target);
+ assertRecoveredTableSchema(statement, table);
+ assertRecoveredTableRows(statement, table, false);
+ assertRecoveredTableDevices(statement, table, false);
+ Assert.assertFalse(source.exists());
+ Assert.assertFalse(new File(source + TsFileResource.RESOURCE_SUFFIX).exists());
+ statement.execute("flush");
+ }
+
+ TestUtils.restartCluster(EnvFactory.getEnv());
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ final Statement statement = connection.createStatement()) {
+ statement.execute("use " + target);
+ assertRecoveredTableSchema(statement, table);
+ assertRecoveredTableRows(statement, table, false);
+ assertRecoveredTableDevices(statement, table, false);
+ // Merge into a loaded row: retaining v proves the null trailing TAG resolves to the same
+ // device identity, which writes at fresh timestamps alone cannot establish.
+ statement.execute("insert into " + table + "(time,t1,t2,n) values(20000,'a','b',55)");
+ statement.execute(
+ "insert into " + table + "(time,t1,t2,n,v) values(20001,'a','b',60,'after')");
+ statement.execute(
+ "insert into "
+ + table
+ + "(time,t1,t2,tail,n,extra,v) values(20002,'a','b','tail',70,7,'new-tail')");
+ assertRecoveredTableRows(statement, table, true);
+ assertRecoveredTableDevices(statement, table, true);
+ statement.execute("use " + other);
+ try (final ResultSet rows = statement.executeQuery("show tables")) {
+ Assert.assertFalse(rows.next());
+ }
+ }
+ }
+
+ private static long writeTableRecoverySource(final File file, final String table)
+ throws Exception {
+ final List schemas =
+ Arrays.asList(
+ new MeasurementSchema("t1", TSDataType.STRING),
+ new MeasurementSchema("t2", TSDataType.STRING),
+ new MeasurementSchema("v", TSDataType.STRING),
+ new MeasurementSchema("n", TSDataType.INT64));
+ final List names = Arrays.asList("t1", "t2", "v", "n");
+ final List types =
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.STRING, TSDataType.INT64);
+ final List categories =
+ Arrays.asList(
+ ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD);
+ try (final TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTableSchema(new TableSchema(table, schemas, categories));
+ final Tablet anchor = new Tablet(table, names, types, categories, 1);
+ anchor.addTimestamp(0, 0);
+ anchor.addValue(0, "t1", "anchor");
+ anchor.addValue(0, "t2", "x");
+ anchor.addValue(0, "v", "anchor");
+ anchor.addValue(0, "n", -1L);
+ writer.writeTable(anchor);
+ writer.flush();
+ final long markerOffset = writer.getIOWriter().getPos();
+ final Tablet tablet = new Tablet(table, names, types, categories, 5);
+ final long[] times = {9999, 10000, 10001, 19999, 20000};
+ final String[] firstTags = {"a", "a", "a", null, "a"};
+ final String[] secondTags = {"b", null, "b", "b", "b"};
+ final String[] values = {"one", null, "three", null, "five"};
+ final Long[] numbers = {10L, 20L, null, null, 50L};
+ for (int row = 0; row < times.length; row++) {
+ tablet.addTimestamp(row, times[row]);
+ tablet.addValue(row, "t1", firstTags[row]);
+ tablet.addValue(row, "t2", secondTags[row]);
+ tablet.addValue(row, "v", values[row]);
+ if (numbers[row] != null) {
+ tablet.addValue(row, "n", numbers[row].longValue());
+ }
+ }
+ writer.writeTable(tablet);
+ return markerOffset;
+ }
+ }
+
+ private static void assertRecoveredTableSchema(final Statement statement, final String table)
+ throws SQLException {
+ final String[][] expected = {
+ {"time", "TIMESTAMP", "TIME"}, {"t1", "STRING", "TAG"}, {"t2", "STRING", "TAG"},
+ {"tail", "STRING", "TAG"}, {"n", "INT64", "FIELD"}, {"extra", "INT32", "FIELD"},
+ {"v", "STRING", "FIELD"}
+ };
+ try (final ResultSet rows = statement.executeQuery("describe " + table)) {
+ for (final String[] row : expected) {
+ Assert.assertTrue(rows.next());
+ for (int column = 0; column < row.length; column++) {
+ Assert.assertEquals(row[column], rows.getString(column + 1));
+ }
+ }
+ Assert.assertFalse(rows.next());
+ }
+ }
+
+ private static void assertRecoveredTableRows(
+ final Statement statement, final String table, final boolean afterWrites)
+ throws SQLException {
+ final List expected =
+ new ArrayList<>(
+ Arrays.asList(
+ new Object[] {0L, "anchor", "x", null, -1L, null, "anchor"},
+ new Object[] {9999L, "a", "b", null, 10L, null, "one"},
+ new Object[] {10000L, "a", null, null, 20L, null, null},
+ new Object[] {10001L, "a", "b", null, null, null, "three"},
+ new Object[] {19999L, null, "b", null, null, null, null},
+ new Object[] {20000L, "a", "b", null, afterWrites ? 55L : 50L, null, "five"}));
+ if (afterWrites) {
+ expected.add(new Object[] {20001L, "a", "b", null, 60L, null, "after"});
+ expected.add(new Object[] {20002L, "a", "b", "tail", 70L, 7, "new-tail"});
+ }
+ try (final ResultSet rows =
+ statement.executeQuery(
+ "select time,t1,t2,tail,n,extra,v from " + table + " order by time")) {
+ for (final Object[] row : expected) {
+ Assert.assertTrue(Arrays.toString(row), rows.next());
+ Assert.assertEquals(row[0], rows.getLong(1));
+ for (int column = 1; column < row.length; column++) {
+ Assert.assertEquals(Arrays.toString(row), row[column], rows.getObject(column + 1));
+ }
+ }
+ Assert.assertFalse(rows.next());
+ }
+ }
+
+ private static void assertRecoveredTableDevices(
+ final Statement statement, final String table, final boolean afterWrites)
+ throws SQLException {
+ final List expected =
+ new ArrayList<>(
+ Arrays.asList("anchor,x,null,", "a,b,null,", "a,null,null,", "null,b,null,"));
+ if (afterWrites) {
+ expected.add("a,b,tail,");
+ }
+ try (final ResultSet rows = statement.executeQuery("show devices from " + table)) {
+ TestUtils.assertResultSetEqualUnordered(rows, "t1,t2,tail,", expected);
+ }
+ }
+
private static class SchemaConfig {
private static final String DATABASE_0 = "root";
private static final String TABLE_0 = "test";
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java
index d24c4737c25e5..151548da6f368 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBRestartTableIT.java
@@ -21,15 +21,22 @@
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.it.utils.DeletionTestUtils;
import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.write.record.Tablet;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -50,8 +57,10 @@
import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -92,6 +101,112 @@ public void tearDown() throws Exception {
.setSchemaRegionConsensusProtocolClass("org.apache.iotdb.consensus.simple.SimpleConsensus");
}
+ @Test
+ public void testPartialAndRejectedFieldsSurviveCrash() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE partial_recovery");
+ statement.execute("USE partial_recovery");
+ statement.execute(
+ "CREATE TABLE accepted(site STRING TAG, device STRING TAG, n INT64 FIELD, v INT64 FIELD)");
+ statement.execute("CREATE TABLE rejected(device STRING TAG, n INT64 FIELD)");
+ SQLException partial =
+ assertThrows(
+ SQLException.class,
+ () ->
+ statement.execute(
+ "INSERT INTO accepted(time,site,device,n,v) VALUES(1,'s','d',true,42)"));
+ assertEquals(507, partial.getErrorCode());
+ assertTrue(partial.getMessage(), partial.getMessage().contains("[n]"));
+ // Explicit NULL fields produce a real row; rejected fields must not produce one.
+ statement.execute("INSERT INTO accepted(time,site,device,n,v) VALUES(2,'s','d',null,null)");
+ }
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE partial_recovery");
+ for (boolean withTag : new boolean[] {true, false}) {
+ Tablet tablet =
+ new Tablet(
+ "rejected",
+ withTag ? Arrays.asList("device", "n") : Arrays.asList("n"),
+ withTag
+ ? Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN)
+ : Arrays.asList(TSDataType.BOOLEAN),
+ withTag
+ ? Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)
+ : Arrays.asList(ColumnCategory.FIELD),
+ 1);
+ tablet.addTimestamp(0, withTag ? 1 : 2);
+ if (withTag) {
+ tablet.addValue("device", 0, "rejected-device");
+ }
+ tablet.addValue("n", 0, true);
+ StatementExecutionException rejected =
+ assertThrows(StatementExecutionException.class, () -> session.insert(tablet));
+ assertEquals(507, rejected.getStatusCode());
+ assertTrue(
+ rejected.getMessage(),
+ rejected
+ .getMessage()
+ .contains(DataNodeQueryMessages.NO_COLUMN_OTHER_THAN_TIME_PRESENT_PLEASE_CHECK));
+ }
+ }
+ assertPartialRecoveryData(false);
+ TestUtils.stopForciblyAndRestartDataNodes();
+ assertPartialRecoveryData(false);
+
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE partial_recovery");
+ session.executeNonQueryStatement(
+ "INSERT INTO accepted(time,site,device,n) VALUES(1,'s','d',7)");
+ session.executeNonQueryStatement(
+ "INSERT INTO rejected(time,device,n) VALUES(1,'rejected-device',8)");
+ session.executeNonQueryStatement("INSERT INTO rejected(time,n) VALUES(2,9)");
+ session.executeNonQueryStatement("FLUSH");
+ }
+ assertPartialRecoveryData(true);
+ }
+
+ private static void assertPartialRecoveryData(boolean repaired) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE partial_recovery");
+ try (ResultSet result =
+ statement.executeQuery("SELECT time,site,device,n,v FROM accepted ORDER BY time")) {
+ for (int time = 1; time <= 2; time++) {
+ assertTrue(result.next());
+ assertEquals(time, result.getLong(1));
+ assertEquals("s", result.getString(2));
+ assertEquals("d", result.getString(3));
+ assertEquals(repaired && time == 1 ? Long.valueOf(7) : null, result.getObject(4));
+ assertEquals(time == 1 ? Long.valueOf(42) : null, result.getObject(5));
+ }
+ assertFalse(result.next());
+ }
+ try (ResultSet result =
+ statement.executeQuery("SELECT time,device,n FROM rejected ORDER BY time")) {
+ if (repaired) {
+ assertTrue(result.next());
+ assertEquals(1, result.getLong(1));
+ assertEquals("rejected-device", result.getString(2));
+ assertEquals(8, result.getLong(3));
+ assertTrue(result.next());
+ assertEquals(2, result.getLong(1));
+ assertNull(result.getString(2));
+ assertEquals(9, result.getLong(3));
+ }
+ assertFalse(result.next());
+ }
+ }
+ if (!repaired) {
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE partial_recovery");
+ try (SessionDataSet devices = session.executeQueryStatement("SHOW DEVICES FROM rejected")) {
+ assertFalse("Rejected FIELDs must not create a phantom device", devices.hasNext());
+ }
+ }
+ }
+ }
+
@Test
public void testRestart() throws SQLException {
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataP0RecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataP0RecoveryIT.java
new file mode 100644
index 0000000000000..1480459cd6037
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataP0RecoveryIT.java
@@ -0,0 +1,746 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.schema;
+
+import org.apache.iotdb.db.it.utils.MetadataFaultTestUtils;
+import org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.Fault;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.fault.metadata.MetadataTestHooks;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.FENCE_MS;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.TIMEOUT_SECONDS;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.assertAllRows;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.assertFenced;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.assertRows;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.awaitAllRows;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.connect;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.execute;
+import static org.apache.iotdb.db.it.utils.MetadataFaultTestUtils.interruptProcedure;
+import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Metadata P0 fault windows missing from normal DDL/device tests: live but fenced client nodes,
+ * durable real-procedure failover, interrupted blacklists, object reuse and attribute backlogs. The
+ * Memory-only table schema engine uses one class-scoped 3CN/3DN cluster with Ratis RF=3.
+ */
+@RunWith(IoTDBTestRunner.class)
+@Category(TableClusterIT.class)
+public class IoTDBTableMetadataP0RecoveryIT {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ MetadataFaultTestUtils.configureCluster();
+ EnvFactory.getEnv().getConfig().getCommonConfig().setSchemaEngineMode("Memory");
+ EnvFactory.getEnv().initClusterEnvironment(3, 3);
+ }
+
+ /** P0-1/P0-2: a recovered real procedure must also respect the isolated node's lease. */
+ @Test
+ public void testConfigNodeFailoverRetainsLeaseSafetyForUnackedNode() throws Exception {
+ final String database = "p0_failover_lease";
+ final DataNodeWrapper victim = EnvFactory.getEnv().getDataNodeWrapper(2);
+ try (Connection connection = connect(2, TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement();
+ Fault isolation = new Fault(victim, MetadataTestHooks.CONFIG_NODE_RPC);
+ Fault heartbeat = new Fault(victim, "heartbeat-rejected").arm("observe")) {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table " + database + ".t(id string tag, old int32, kept int32)",
+ "insert into " + database + ".t(time,id,old,kept) values(1,'d',10,20)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,old,kept from " + database + ".t",
+ "id,old,kept,",
+ "d,10,20,");
+ interruptProcedure(
+ "DropTableColumnProcedure",
+ "INVALIDATE_CACHE",
+ false,
+ () -> execute(0, TABLE_SQL_DIALECT, "alter table " + database + ".t drop column old"),
+ () -> {
+ isolation.arm("fail");
+ heartbeat.awaitHit();
+ });
+ // The new leader has recovered the same procedure and committed a definition the victim
+ // never acknowledged. Its existing physical client session must therefore be fenced.
+ assertTrue(victim.isAlive());
+ assertFenced(statement, "insert into " + database + ".t(time,id,old) values(2,'d',999)");
+ assertRows(
+ 0,
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "kept,INT32,FIELD,");
+ isolation.release();
+ awaitAllRows(
+ TABLE_SQL_DIALECT, "select id,kept from " + database + ".t", "id,kept,", "d,20,");
+ statement.execute("insert into " + database + ".t(time,id,kept) values(3,'d',30)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kept from " + database + ".t",
+ "id,kept,",
+ "d,20,",
+ "d,30,");
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ /** P0-1: this endpoint keeps its process, JDBC session and inter-DataNode links throughout. */
+ @Test
+ public void testLivePartitionedNodeFencesAndRetriesFailedResync() throws Exception {
+ final String database = "p0_table_lease";
+ final DataNodeWrapper victim = EnvFactory.getEnv().getDataNodeWrapper(2);
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ try (Connection connection = connect(2, TABLE_SQL_DIALECT);
+ Statement victimStatement = connection.createStatement()) {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table "
+ + database
+ + ".t (id string tag, a string attribute, old int32, kept int32)",
+ "insert into " + database + ".t(time,id,a,old,kept) values(1,'d','v1',10,20)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,old,kept from " + database + ".t where id='d'",
+ "id,a,old,kept,",
+ "d,v1,10,20,");
+ try (Fault isolation = new Fault(victim, MetadataTestHooks.CONFIG_NODE_RPC);
+ Fault heartbeat = new Fault(victim, "heartbeat-rejected").arm("observe");
+ Fault invalidation = new Fault(victim, "cache-broadcast-rejected").arm("observe");
+ Fault recovery = new Fault(victim, MetadataTestHooks.LEASE_RECOVERY).arm("fail")) {
+ final long disconnectedAt = System.nanoTime();
+ isolation.arm("fail");
+ heartbeat.awaitHit();
+ // A separate connection owns the background statement and propagates failures via Future.
+ final Future ddl =
+ executor.submit(
+ () -> {
+ execute(0, TABLE_SQL_DIALECT, "alter table " + database + ".t drop column old");
+ return null;
+ });
+ invalidation.awaitHit();
+ await()
+ .during(2, TimeUnit.SECONDS)
+ .atMost(3, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertFalse("Unacked, unfenced node was skipped", ddl.isDone()));
+ await()
+ .atMost(TIMEOUT_SECONDS, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertFenced(
+ victimStatement, "select old from " + database + ".t where id='d'"));
+ assertTrue("The partitioned process was stopped", victim.isAlive());
+ assertFenced(
+ victimStatement, "insert into " + database + ".t(time,id,old) values(2,'d',999)");
+ ddl.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ assertTrue(
+ "DDL crossed the fence boundary too early",
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - disconnectedAt) >= FENCE_MS - 2000);
+ assertRows(
+ 0,
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "a,STRING,ATTRIBUTE,",
+ "kept,INT32,FIELD,");
+ isolation.release();
+ recovery.awaitHit();
+ // A heartbeat alone must not re-enable service when metadata pulling failed.
+ assertFenced(victimStatement, "select kept from " + database + ".t where id='d'");
+ recovery.release();
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,kept from " + database + ".t",
+ "id,a,kept,",
+ "d,v1,20,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "a,STRING,ATTRIBUTE,",
+ "kept,INT32,FIELD,");
+ victimStatement.execute("insert into " + database + ".t(time,id,kept) values(3,'d',30)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kept from " + database + ".t",
+ "id,kept,",
+ "d,20,",
+ "d,30,");
+ }
+ } finally {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /** P0-2: PRE_CREATE is durable, PRE_RELEASE is still a rollback-capable boundary. */
+ @Test
+ public void testPreCreatedTableRollsBackAfterConfigNodeFailover() throws Exception {
+ final String database = "p0_create_rollback";
+ try {
+ execute(0, TABLE_SQL_DIALECT, "create database " + database);
+ interruptProcedure(
+ "CreateTableProcedure",
+ "PRE_RELEASE",
+ true,
+ () ->
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create table " + database + ".t(id string tag, ghost int32)"));
+ awaitAllRows(TABLE_SQL_DIALECT, "show tables from " + database, "TableName,TTL(ms),");
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create table if not exists " + database + ".t(id string tag, v boolean)",
+ "insert into " + database + ".t(time,id,v) values(1,'d',true)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "v,BOOLEAN,FIELD,");
+ assertAllRows(TABLE_SQL_DIALECT, "select id,v from " + database + ".t", "id,v,", "d,true,");
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /** P0-2: the committed catalog survives loss of the original leader/client response. */
+ @Test
+ public void testCommittedCreateContinuesAfterConfigNodeFailover() throws Exception {
+ final String database = "p0_create_commit";
+ try {
+ execute(0, TABLE_SQL_DIALECT, "create database " + database);
+ interruptProcedure(
+ "CreateTableProcedure",
+ "COMMIT_RELEASE",
+ false,
+ () ->
+ execute(
+ 0, TABLE_SQL_DIALECT, "create table " + database + ".t(id string tag, v int32)"));
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "v,INT32,FIELD,");
+ // Resolve the unknown response using the documented idempotent form, after observing commit.
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create table if not exists " + database + ".t(id string tag, v int32)",
+ "insert into " + database + ".t(time,id,v) values(1,'d',17)");
+ assertAllRows(TABLE_SQL_DIALECT, "select id,v from " + database + ".t", "id,v,", "d,17,");
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /**
+ * P0-3: both blacklist rollback and deletion continuation operate on actual cross-region keys.
+ */
+ @Test
+ public void testCrossRegionDeviceDeleteRecoversBlacklistAndDataDeletion() throws Exception {
+ final String database = "p0_delete_devices";
+ final List devices = new ArrayList<>();
+ final List expected = new ArrayList<>();
+ try {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table "
+ + database
+ + ".t(id string tag, kind string tag, a string attribute, v int32)",
+ "insert into "
+ + database
+ + ".t(time,id,kind,a,v) values(1,'keep','keep','protected',77)");
+ // Few stable keys suffice; verify their real partition mapping rather than node count.
+ for (int i = 0; i < 16; i++) {
+ final String id = "delete_" + i;
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "insert into "
+ + database
+ + ".t(time,id,kind,a,v) values(1,'"
+ + id
+ + "','delete','old',"
+ + i
+ + ")");
+ devices.add(IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"t", id, "delete"}));
+ expected.add(id + ",delete,old," + i + ",");
+ }
+ MetadataFaultTestUtils.assertSpansRegions(database, devices);
+ expected.add("keep,keep,protected,77,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kind,a,v from " + database + ".t",
+ "id,kind,a,v,",
+ expected.toArray(new String[0]));
+ final String delete = "delete devices from " + database + ".t where kind='delete'";
+ interruptProcedure(
+ "DeleteDevicesProcedure",
+ "CLEAN_DATANODE_SCHEMA_CACHE",
+ true,
+ () -> execute(0, TABLE_SQL_DIALECT, delete));
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kind,a,v from " + database + ".t",
+ "id,kind,a,v,",
+ expected.toArray(new String[0]));
+ assertAllRows(
+ TABLE_SQL_DIALECT, "count devices from " + database + ".t", "count(devices),", "17,");
+ interruptProcedure(
+ "DeleteDevicesProcedure",
+ "DELETE_DEVICE_SCHEMA",
+ false,
+ () -> execute(0, TABLE_SQL_DIALECT, delete));
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "show devices from " + database + ".t",
+ "id,kind,a,",
+ "keep,keep,protected,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kind,a,v from " + database + ".t",
+ "id,kind,a,v,",
+ "keep,keep,protected,77,");
+ assertAllRows(
+ TABLE_SQL_DIALECT, "count devices from " + database + ".t", "count(devices),", "1,");
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ delete,
+ "insert into "
+ + database
+ + ".t(time,id,kind,a,v) values(2,'delete_0','delete','new',100)",
+ "insert into " + database + ".t(time,id,kind,v) values(2,'keep','keep',78)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,kind,a,v from " + database + ".t",
+ "id,kind,a,v,",
+ "keep,keep,protected,77,",
+ "keep,keep,protected,78,",
+ "delete_0,delete,new,100,");
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /**
+ * P0-2/P0-4: async DROP recovery, physical mod progress, shared-file retention and name reuse.
+ */
+ @Test
+ public void testAsyncDropFailoverAndSameNameRecreateBeforeOldNodeReturns() throws Exception {
+ final String database = "p0_drop_recreate";
+ final DataNodeWrapper victim = EnvFactory.getEnv().getDataNodeWrapper(2);
+ try {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table "
+ + database
+ + ".t(id string tag, a string attribute, v int32) with(ttl=10000000000000)",
+ "create table " + database + ".retained(id string tag, a string attribute, v int32)",
+ "insert into " + database + ".t(time,id,a,v) values(1,'old','obsolete',10)",
+ "insert into " + database + ".retained(time,id,a,v) values(1,'keep','protected',77)");
+ // IoTConsensus replication is asynchronous; flush only after both replicas have the fixture.
+ MetadataFaultTestUtils.awaitDataReplication(database);
+ execute(0, TABLE_SQL_DIALECT, "flush");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "old,obsolete,10,");
+ victim.stopForcibly();
+ assertFalse(victim.isAlive());
+ interruptProcedure(
+ "DropTableProcedure",
+ "DELETE_DATA",
+ false,
+ () -> {
+ execute(0, TABLE_SQL_DIALECT, "drop table " + database + ".t");
+ // DROP returns asynchronously; retain the client operation until the paused procedure
+ // ends.
+ MetadataFaultTestUtils.awaitDropSuccess(database, "t");
+ });
+ MetadataFaultTestUtils.awaitDropSuccess(database, "t");
+ // Reuse the name only after confirmed deletion, while the old DataNode is still offline.
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create table " + database + ".t(id string tag, a string attribute, v boolean)",
+ "insert into " + database + ".t(time,id,a,v) values(2,'new','fresh',true)");
+ victim.start();
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "a,STRING,ATTRIBUTE,",
+ "v,BOOLEAN,FIELD,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "show tables from " + database,
+ "TableName,TTL(ms),",
+ "t,INF,",
+ "retained,INF,");
+ assertAllRows(
+ TABLE_SQL_DIALECT, "show devices from " + database + ".t", "id,a,", "new,fresh,");
+ assertAllRows(
+ TABLE_SQL_DIALECT, "select id,a,v from " + database + ".t", "id,a,v,", "new,fresh,true,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".retained",
+ "id,a,v,",
+ "keep,protected,77,");
+ execute(
+ 2,
+ TABLE_SQL_DIALECT,
+ "insert into " + database + ".t(time,id,v) values(3,'new',false)",
+ "insert into " + database + ".retained(time,id,v) values(3,'keep',78)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "new,fresh,true,",
+ "new,fresh,false,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".retained",
+ "id,a,v,",
+ "keep,protected,77,",
+ "keep,protected,78,");
+ } finally {
+ if (!victim.isAlive()) {
+ victim.start();
+ }
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /** P0-5: a real captured v2 batch is overtaken by NULL/v3 and a SchemaRegion leader change. */
+ @Test
+ public void testAttributeBacklogAndDelayedBatchSurviveSchemaLeaderChange() throws Exception {
+ final String database = "p0_attribute_backlog";
+ DataNodeWrapper oldLeader = null;
+ try {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table " + database + ".t(id string tag, a string attribute, v int32)",
+ "insert into "
+ + database
+ + ".t(time,id,a,v) values(1,'d','v1',10),(1,'keep','protected',77)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where id='d'",
+ "id,a,v,",
+ "d,v1,10,");
+ final int regionId = MetadataFaultTestUtils.tableDeviceRegion(database, "t", "d");
+ oldLeader = MetadataFaultTestUtils.schemaLeader(database, regionId);
+ final int target = oldLeader == EnvFactory.getEnv().getDataNodeWrapper(2) ? 1 : 2;
+ final DataNodeWrapper victim = EnvFactory.getEnv().getDataNodeWrapper(target);
+ try (Fault delayed = new Fault(victim, MetadataTestHooks.ATTRIBUTE_UPDATE).arm("pause")) {
+ execute(0, TABLE_SQL_DIALECT, "update " + database + ".t set a='v2' where id='d'");
+ final String batch = delayed.awaitHit();
+ try (Fault delivered =
+ new Fault(victim, "attribute-update-returned." + batch).arm("observe")) {
+ // Hold the captured old call and reject its retries, leaving the last confirmed values
+ // in the replicated updater backlog when leadership changes.
+ delayed.arm("fail");
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "update " + database + ".t set a=NULL where id='d'",
+ "update " + database + ".t set a='v3' where id='d'");
+ delayed.awaitFailure();
+ oldLeader.stopForcibly();
+ MetadataFaultTestUtils.awaitDifferentSchemaLeader(database, regionId, oldLeader);
+ delayed.disarm("fail");
+ oldLeader.start();
+ // Stop all attribute writes. The new leader must drain the existing backlog itself.
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where id='d'",
+ "id,a,v,",
+ "d,v3,10,");
+ delayed.release();
+ delivered.awaitHit();
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "show devices from " + database + ".t",
+ "id,a,",
+ "d,v3,",
+ "keep,protected,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "count devices from " + database + ".t where a='v3'",
+ "count(devices),",
+ "1,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where a='v3'",
+ "id,a,v,",
+ "d,v3,10,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where id='keep'",
+ "id,a,v,",
+ "keep,protected,77,");
+ }
+ }
+ } finally {
+ if (oldLeader != null && !oldLeader.isAlive()) {
+ oldLeader.start();
+ }
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /** P0-4/5: deliver a captured old attribute request after a supported DELETE/INSERT lifecycle. */
+ @Test
+ public void testDelayedAttributeCannotPolluteRecreatedDevice() throws Exception {
+ final String database = "p0_attribute_recreate";
+ try {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create database " + database,
+ "create table " + database + ".t(id string tag, a string attribute, v int32)",
+ "insert into "
+ + database
+ + ".t(time,id,a,v) values(1,'d','old',10),(1,'keep','protected',77)");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "d,old,10,",
+ "keep,protected,77,");
+ final int regionId = MetadataFaultTestUtils.tableDeviceRegion(database, "t", "d");
+ final DataNodeWrapper leader = MetadataFaultTestUtils.schemaLeader(database, regionId);
+ final DataNodeWrapper victim =
+ EnvFactory.getEnv().getDataNodeWrapperList().stream()
+ .filter(node -> node != leader)
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ try (Fault delayed = new Fault(victim, MetadataTestHooks.ATTRIBUTE_UPDATE).arm("pause")) {
+ execute(0, TABLE_SQL_DIALECT, "update " + database + ".t set a='obsolete' where id='d'");
+ final String batch = delayed.awaitHit();
+ try (Fault delivered =
+ new Fault(victim, "attribute-update-returned." + batch).arm("observe")) {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "delete devices from " + database + ".t where id='d'",
+ "insert into " + database + ".t(time,id,a,v) values(2,'d','fresh',20)");
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "d,fresh,20,",
+ "keep,protected,77,");
+ delayed.release();
+ delivered.awaitHit();
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "show devices from " + database + ".t",
+ "id,a,",
+ "d,fresh,",
+ "keep,protected,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "count devices from " + database + ".t where a='obsolete'",
+ "count(devices),",
+ "0,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "d,fresh,20,",
+ "keep,protected,77,");
+ }
+ }
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ /**
+ * P0-6: SchemaRegion snapshot payload plus a separate log tail, including NULL and recreation.
+ */
+ @Test
+ public void testSchemaSnapshotAndLogTailRestoreDevicesAndAttributes() throws Exception {
+ final String database = "p0_table_snapshot";
+ try {
+ MetadataFaultTestUtils.createSingleSchemaRegionDatabase(database, TABLE_SQL_DIALECT);
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "create table " + database + ".t(id string tag, a string attribute, v int32)",
+ "create table " + database + ".retained(id string tag, a string attribute, v int32)",
+ "insert into "
+ + database
+ + ".t(time,id,a,v) values(1,'d','baseline',10),(1,'n',NULL,11),"
+ + "(1,'gone','deleted',12),(1,'reborn','old',13)",
+ "insert into " + database + ".retained(time,id,a,v) values(1,'keep','protected',77)",
+ "delete devices from " + database + ".t where id='gone'");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "show devices from " + database + ".t",
+ "id,a,",
+ "d,baseline,",
+ "n,null,",
+ "reborn,old,");
+ final int regionId = MetadataFaultTestUtils.tableDeviceRegion(database, "t", "d");
+ assertTrue(
+ MetadataFaultTestUtils.schemaRegions(database).stream()
+ .allMatch(info -> info.getConsensusGroupId().getId() == regionId));
+ MetadataFaultTestUtils.installSchemaSnapshot(
+ database,
+ regionId,
+ () -> {
+ // Metadata-only log entries in the same region trigger a real snapshot and log purge.
+ for (int i = 0; i < 32; i++) {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "update " + database + ".t set a='baseline" + i + "' where id='d'");
+ }
+ },
+ () -> {
+ execute(
+ 0,
+ TABLE_SQL_DIALECT,
+ "update " + database + ".t set a='tail' where id='d'",
+ "update " + database + ".t set a=NULL where id='reborn'",
+ "delete devices from " + database + ".t where id='reborn'",
+ "insert into "
+ + database
+ + ".t(time,id,a,v) values(2,'reborn','fresh',20),"
+ + "(2,'added','new',21)");
+ },
+ () -> assertSnapshotTableState(database));
+ // First reconcile the old manifest; only then prove the recovered leader accepts new work.
+ execute(0, TABLE_SQL_DIALECT, "update " + database + ".t set a='after' where id='d'");
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where id='d'",
+ "id,a,v,",
+ "d,after,10,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".retained",
+ "id,a,v,",
+ "keep,protected,77,");
+ } finally {
+ execute(0, TABLE_SQL_DIALECT, "drop database if exists " + database);
+ }
+ }
+
+ private static void assertSnapshotTableState(final String database) throws Exception {
+ awaitAllRows(
+ TABLE_SQL_DIALECT,
+ "show devices from " + database + ".t",
+ "id,a,",
+ "d,tail,",
+ "n,null,",
+ "reborn,fresh,",
+ "added,new,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t",
+ "id,a,v,",
+ "d,tail,10,",
+ "n,null,11,",
+ "reborn,fresh,20,",
+ "added,new,21,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "count devices from " + database + ".t where a is null",
+ "count(devices),",
+ "1,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".t where a='tail'",
+ "id,a,v,",
+ "d,tail,10,");
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "select id,a,v from " + database + ".retained",
+ "id,a,v,",
+ "keep,protected,77,");
+ // Catalog definitions are ConfigNode state, checked separately from the SchemaRegion payload.
+ assertAllRows(
+ TABLE_SQL_DIALECT,
+ "describe " + database + ".t",
+ "ColumnName,DataType,Category,",
+ "time,TIMESTAMP,TIME,",
+ "id,STRING,TAG,",
+ "a,STRING,ATTRIBUTE,",
+ "v,INT32,FIELD,");
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataRecoveryIT.java
new file mode 100644
index 0000000000000..d218b9c4609b4
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableMetadataRecoveryIT.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.schema;
+
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.iotdb.util.MetadataRecoveryTestUtils.*;
+import static org.junit.Assert.*;
+
+/** P1-4/5: a previously warmed DataNode misses catalog broadcasts and then serves clients again. */
+@RunWith(IoTDBTestRunner.class)
+@Category(TableClusterIT.class)
+public class IoTDBTableMetadataRecoveryIT {
+ @BeforeClass
+ public static void before() {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaEngineMode("Memory")
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaReplicationFactor(3)
+ .setDataReplicationFactor(3);
+ EnvFactory.getEnv().getConfig().getConfigNodeConfig().setMetadataLeaseFenceMs(20000);
+ EnvFactory.getEnv().initClusterEnvironment(3, 3);
+ }
+
+ @AfterClass
+ public static void after() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testDatabaseTtlInheritanceAfterMissedBroadcast() throws Exception {
+ String database = "p1_ttl";
+ long day = TimeUnit.DAYS.toMillis(1);
+ long old = System.currentTimeMillis() - 2 * day;
+ long recent = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1);
+ DataNodeWrapper stale = node(2);
+ try (Connection healthy = connection(node(0), BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(
+ healthy,
+ "CREATE DATABASE " + database + " WITH (ttl=" + 7 * day + ")",
+ "USE " + database,
+ "CREATE TABLE inherited(dev STRING TAG,v INT32 FIELD)",
+ "CREATE TABLE overridden(dev STRING TAG,v INT32 FIELD) WITH (ttl=" + 30 * day + ")");
+ for (String table : new String[] {"inherited", "overridden"}) {
+ execute(
+ healthy,
+ "INSERT INTO "
+ + table
+ + "(time,dev,v) VALUES("
+ + old
+ + ",'d',1),("
+ + recent
+ + ",'d',2)");
+ }
+ for (DataNodeWrapper dataNode : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ try (Connection warm = connection(dataNode, BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(warm, "USE " + database);
+ assertRows(
+ warm,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM inherited",
+ "ts,dev,v,",
+ old + ",d,1,",
+ recent + ",d,2,");
+ }
+ }
+ stale.stopForcibly();
+ assertFalse(stale.isAlive());
+ execute(healthy, "ALTER DATABASE " + database + " SET PROPERTIES ttl=" + day);
+ assertTtls(healthy, day, 30 * day);
+ assertRows(
+ healthy,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM inherited",
+ "ts,dev,v,",
+ recent + ",d,2,");
+ assertRows(
+ healthy,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM overridden",
+ "ts,dev,v,",
+ old + ",d,1,",
+ recent + ",d,2,");
+ stale.start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ try (Connection recovered = connection(stale, BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(recovered, "USE " + database);
+ // Check recovery before doing any new write; another DDL or cache clear must not repair it.
+ assertTtls(recovered, day, 30 * day);
+ assertRows(
+ recovered,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM inherited",
+ "ts,dev,v,",
+ recent + ",d,2,");
+ assertRows(
+ recovered,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM overridden",
+ "ts,dev,v,",
+ old + ",d,1,",
+ recent + ",d,2,");
+ try {
+ execute(recovered, "INSERT INTO inherited(time,dev,v) VALUES(" + (old + 1) + ",'d',3)");
+ fail("Recovered node must enforce the new inherited TTL");
+ } catch (SQLException e) {
+ assertEquals(e.getMessage(), TSStatusCode.OUT_OF_TTL.getStatusCode(), e.getErrorCode());
+ }
+ execute(
+ recovered,
+ "INSERT INTO overridden(time,dev,v) VALUES(" + (old + 1) + ",'d',3)",
+ "INSERT INTO inherited(time,dev,v) VALUES(" + (recent + 1) + ",'d',4)");
+ }
+ assertRows(
+ healthy,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM inherited",
+ "ts,dev,v,",
+ recent + ",d,2,",
+ (recent + 1) + ",d,4,");
+ assertRows(
+ healthy,
+ "SELECT CAST(time AS INT64) AS ts,dev,v FROM overridden",
+ "ts,dev,v,",
+ old + ",d,1,",
+ (old + 1) + ",d,3,",
+ recent + ",d,2,");
+ } finally {
+ if (!stale.isAlive()) {
+ stale.start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ }
+ try (Connection cleanup = connection(node(0), BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(cleanup, "DROP DATABASE IF EXISTS " + database);
+ }
+ }
+ }
+
+ private void assertTtls(Connection connection, long inherited, long overridden)
+ throws SQLException {
+ Map actual = new HashMap<>();
+ try (Statement statement = connection.createStatement();
+ ResultSet tables = statement.executeQuery("SHOW TABLES")) {
+ while (tables.next()) {
+ assertNull(actual.put(tables.getString(1), Long.parseLong(tables.getString(2))));
+ }
+ }
+ Map expected = new HashMap<>();
+ expected.put("inherited", inherited);
+ expected.put("overridden", overridden);
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testViewAutoSyncAfterMissedRenameReplaceAndDisable() throws Exception {
+ String treeDb = "root.p1_views";
+ String viewDb = "p1_views";
+ DataNodeWrapper stale = node(2);
+ try (Connection tree = connection(node(0), BaseEnv.TREE_SQL_DIALECT);
+ Connection table = connection(node(0), BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(
+ tree,
+ "CREATE DATABASE " + treeDb,
+ "CREATE TIMESERIES " + treeDb + ".old.d.seed INT32",
+ "INSERT INTO " + treeDb + ".old.d(time,seed) VALUES(1,1)");
+ execute(
+ table,
+ "CREATE DATABASE " + viewDb,
+ "USE " + viewDb,
+ "CREATE VIEW v(dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**",
+ "CREATE VIEW disabled(dev STRING TAG) WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".old.**");
+ try (Connection warmTree = connection(stale, BaseEnv.TREE_SQL_DIALECT);
+ Connection warmTable = connection(stale, BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(warmTable, "USE " + viewDb);
+ assertRows(
+ warmTable,
+ "SELECT CAST(time AS INT64) AS ts,dev,seed FROM v",
+ "ts,dev,seed,",
+ "1,d,1,");
+ // Creating a measurement on this node warms its view matching index as well as the query
+ // cache.
+ execute(warmTree, "CREATE TIMESERIES " + treeDb + ".old.d.warm INT32");
+ awaitColumns(
+ warmTable,
+ "v",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "seed:INT32:FIELD",
+ "warm:INT32:FIELD");
+ }
+ stale.stopForcibly();
+ assertFalse(stale.isAlive());
+ execute(
+ table,
+ "ALTER VIEW v RENAME TO renamed",
+ "CREATE OR REPLACE VIEW renamed(dev STRING TAG) WITH (auto_sync_schema=true) AS "
+ + treeDb
+ + ".current.**",
+ "ALTER VIEW disabled SET PROPERTIES auto_sync_schema=false");
+ execute(
+ tree,
+ "CREATE TIMESERIES " + treeDb + ".current.d.during INT32",
+ "INSERT INTO " + treeDb + ".current.d(time,during) VALUES(2,2)",
+ "CREATE TIMESERIES " + treeDb + ".old.d.ignored_during INT32");
+ awaitColumns(table, "renamed", "time:TIMESTAMP:TIME", "dev:STRING:TAG", "during:INT32:FIELD");
+ stale.start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ try (Connection recoveredTable = connection(stale, BaseEnv.TABLE_SQL_DIALECT);
+ Connection recoveredTree = connection(stale, BaseEnv.TREE_SQL_DIALECT)) {
+ execute(recoveredTable, "USE " + viewDb);
+ awaitColumns(
+ recoveredTable,
+ "renamed",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "during:INT32:FIELD");
+ assertRows(
+ recoveredTable,
+ "SELECT CAST(time AS INT64) AS ts,dev,during FROM renamed",
+ "ts,dev,during,",
+ "2,d,2,");
+ execute(
+ recoveredTree,
+ "CREATE TIMESERIES " + treeDb + ".current.d.after INT64",
+ "INSERT INTO " + treeDb + ".current.d(time,after) VALUES(3,3)",
+ "CREATE TIMESERIES " + treeDb + ".old.d.ignored_after BOOLEAN");
+ awaitColumns(
+ recoveredTable,
+ "renamed",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "during:INT32:FIELD",
+ "after:INT64:FIELD");
+ assertColumns(
+ recoveredTable,
+ "disabled",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "seed:INT32:FIELD",
+ "warm:INT32:FIELD");
+ assertMissingView(recoveredTable, "v");
+ }
+ for (DataNodeWrapper dataNode : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ try (Connection read = connection(dataNode, BaseEnv.TABLE_SQL_DIALECT)) {
+ execute(read, "USE " + viewDb);
+ awaitColumns(
+ read,
+ "renamed",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "during:INT32:FIELD",
+ "after:INT64:FIELD");
+ assertColumns(
+ read,
+ "disabled",
+ "time:TIMESTAMP:TIME",
+ "dev:STRING:TAG",
+ "seed:INT32:FIELD",
+ "warm:INT32:FIELD");
+ assertMissingView(read, "v");
+ assertRows(
+ read,
+ "SELECT CAST(time AS INT64) AS ts,dev,during,after FROM renamed",
+ "ts,dev,during,after,",
+ "2,d,2,null,",
+ "3,d,null,3,");
+ assertRows(
+ read,
+ "SELECT CAST(time AS INT64) AS ts,dev,seed FROM disabled",
+ "ts,dev,seed,",
+ "1,d,1,");
+ }
+ }
+ } finally {
+ if (!stale.isAlive()) {
+ stale.start();
+ EnvFactory.getAbstractEnv().checkClusterStatusWithoutUnknown();
+ }
+ try (Connection table = connection(node(0), BaseEnv.TABLE_SQL_DIALECT);
+ Connection tree = connection(node(0), BaseEnv.TREE_SQL_DIALECT)) {
+ execute(table, "DROP DATABASE IF EXISTS " + viewDb);
+ execute(tree, "DELETE DATABASE " + treeDb);
+ }
+ }
+ }
+
+ static void awaitColumns(Connection connection, String table, String... columns) {
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertColumns(connection, table, columns));
+ }
+
+ static void assertColumns(Connection connection, String table, String... columns)
+ throws SQLException {
+ Map actual = new HashMap<>();
+ try (Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("DESCRIBE " + table)) {
+ while (result.next()) {
+ assertNull(
+ actual.put(
+ result.getString("ColumnName"),
+ result.getString("DataType") + ":" + result.getString("Category")));
+ }
+ }
+ Map expected = new HashMap<>();
+ for (String column : columns) {
+ String[] parts = column.split(":", 2);
+ expected.put(parts[0], parts[1]);
+ }
+ assertEquals(expected, actual);
+ }
+
+ private void assertMissingView(Connection connection, String view) throws SQLException {
+ try {
+ execute(connection, "DESCRIBE " + view);
+ fail("Old view name must not remain in the cache");
+ } catch (SQLException e) {
+ assertEquals(e.getMessage(), TSStatusCode.TABLE_NOT_EXISTS.getStatusCode(), e.getErrorCode());
+ }
+ }
+
+ private static DataNodeWrapper node(int index) {
+ return EnvFactory.getEnv().getDataNodeWrapper(index);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRegionWriteIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRegionWriteIT.java
new file mode 100644
index 0000000000000..832b068852fc1
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRegionWriteIT.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.session.it;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.session.Session;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(IoTDBTestRunner.class)
+@Category(ClusterIT.class)
+public class IoTDBSessionRegionWriteIT {
+ private final String[] databases = new String[2];
+
+ @BeforeClass
+ public static void setUp() {
+ // Single replicas make one Region unwritable without disabling the other Region.
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+ .setSchemaReplicationFactor(1)
+ .setDataReplicationFactor(1)
+ .setRegionGroupAllocatePolicy("GREEDY")
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1)
+ .setTimePartitionInterval(1000)
+ .setWalMode("SYNC");
+ EnvFactory.getEnv().initClusterEnvironment(1, 2);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testSplitTabletsRejectOnlyUnwritableRegionAndRetry() throws Exception {
+ // A rejected Region must not roll back a successful peer or misalign the retried tablet.
+ Map owners = new HashMap<>();
+ Map regions = new HashMap<>();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ // Internal databases also own Regions. Select actual placements instead of assuming
+ // that the first two business databases must land on different DataNodes.
+ for (int candidate = 0; candidate < 8 && databases[1] == null; candidate++) {
+ String database = "root.split_" + candidate;
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + database + ".d.n INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d.t TEXT");
+ // Allocate both time partitions before the fault so that analysis can complete and
+ // the rejection occurs when executing the already-routed child request.
+ statement.execute(
+ "INSERT INTO "
+ + database
+ + ".d(time,n,t) VALUES(0,-1,'baseline'),(1000,-1,'baseline')");
+ try (ResultSet result = statement.executeQuery("SHOW REGIONS")) {
+ while (result.next()) {
+ if (database.equals(result.getString("Database"))
+ && "DataRegion".equals(result.getString("Type"))) {
+ assertEquals(
+ "The fixture requires one replica",
+ null,
+ owners.put(database, result.getInt("RpcPort")));
+ regions.put(database, result.getInt("RegionId"));
+ }
+ }
+ }
+ assertNotNull("Missing DataRegion for " + database, owners.get(database));
+ if (databases[0] == null) {
+ databases[0] = database;
+ } else if (!owners.get(databases[0]).equals(owners.get(database))) {
+ databases[1] = database;
+ }
+ }
+ }
+ assertNotNull("No separately hosted Region in " + owners, databases[1]);
+ assertNotEquals(regions.get(databases[0]), regions.get(databases[1]));
+ assertNotEquals(
+ "Both Regions must have different owners",
+ owners.get(databases[0]),
+ owners.get(databases[1]));
+ DataNodeWrapper healthy = nodeAtPort(owners.get(databases[0]));
+ DataNodeWrapper victim = nodeAtPort(owners.get(databases[1]));
+ try (Session session =
+ new Session.Builder()
+ .host(healthy.getIp())
+ .port(healthy.getPort())
+ .enableAutoFetch(false)
+ .enableRedirection(false)
+ .maxRetryCount(0)
+ .build()) {
+ session.open(false, 30_000);
+ try {
+ setStatus(victim, "READONLY");
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(victim), Collections.singletonList(NodeStatus.ReadOnly));
+ // One RPC, two actual Regions, unsorted timestamps, and sparse text in each partition.
+ Map batch = new LinkedHashMap<>();
+ batch.put(databases[1] + ".d", tablet(1));
+ batch.put(databases[0] + ".d", tablet(0));
+ StatementExecutionException failure =
+ assertThrows(StatementExecutionException.class, () -> session.insertTablets(batch));
+ assertEquals(TSStatusCode.SYSTEM_READ_ONLY.getStatusCode(), failure.getStatusCode());
+ // This API returns a Region-level failure, not individual row acknowledgements.
+ // The successful other Region proves that this was not whole-request prevalidation.
+ assertData(0, true);
+ assertData(1, false);
+ } finally {
+ setStatus(victim, "RUNNING");
+ }
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(victim), Collections.singletonList(NodeStatus.Running));
+ session.insertTablet(tablet(1));
+ assertData(0, true);
+ assertData(1, true);
+ session.executeNonQueryStatement("FLUSH");
+ }
+ TestUtils.stopForciblyAndRestartDataNodes();
+ assertData(0, true);
+ assertData(1, true);
+ }
+
+ private static DataNodeWrapper nodeAtPort(int port) {
+ return EnvFactory.getEnv().getDataNodeWrapperList().stream()
+ .filter(node -> node.getPort() == port)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("No DataNode at RPC port " + port));
+ }
+
+ @Test
+ public void testDeleteAcrossRegionsKeepsPartialSuccessAndSupportsSafeRetry() throws Exception {
+ final String[] targets = new String[2];
+ final Map owners = new HashMap<>();
+ final Map regions = new HashMap<>();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ for (int candidate = 0; candidate < 8 && targets[1] == null; candidate++) {
+ final String database = "root.delete_split_" + candidate;
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("CREATE TIMESERIES " + database + ".d.n INT64");
+ statement.execute("CREATE TIMESERIES " + database + ".d.t TEXT");
+ // Allocate both sides of the configured 1000-unit time-partition boundary before
+ // making an owner read-only. No schema or partition creation is needed by DELETE.
+ statement.execute(
+ "INSERT INTO "
+ + database
+ + ".d(time,n,t) VALUES"
+ + "(998,998,'keep-998'),(999,999,'keep-999'),(1000,1000,'keep-1000'),"
+ + "(1001,1001,'keep-1001'),(1002,1002,'keep-1002')");
+ try (ResultSet result = statement.executeQuery("SHOW REGIONS")) {
+ while (result.next()) {
+ if (database.equals(result.getString("Database"))
+ && "DataRegion".equals(result.getString("Type"))) {
+ assertEquals(null, owners.put(database, result.getInt("RpcPort")));
+ regions.put(database, result.getInt("RegionId"));
+ }
+ }
+ }
+ assertNotNull("Missing DataRegion for " + database, owners.get(database));
+ if (targets[0] == null) {
+ targets[0] = database;
+ } else if (!owners.get(targets[0]).equals(owners.get(database))) {
+ targets[1] = database;
+ }
+ }
+ statement.execute("FLUSH");
+ }
+ assertNotNull("No separately hosted Region in " + owners, targets[1]);
+ assertNotEquals(regions.get(targets[0]), regions.get(targets[1]));
+ assertNotEquals(owners.get(targets[0]), owners.get(targets[1]));
+ final DataNodeWrapper healthy = nodeAtPort(owners.get(targets[0]));
+ final DataNodeWrapper victim = nodeAtPort(owners.get(targets[1]));
+ final String delete =
+ "DELETE FROM "
+ + targets[0]
+ + ".d.n,"
+ + targets[1]
+ + ".d.n WHERE time >= 999 AND time <= 1001";
+ try (Session session =
+ new Session.Builder()
+ .host(healthy.getIp())
+ .port(healthy.getPort())
+ .enableAutoFetch(false)
+ .enableRedirection(false)
+ .maxRetryCount(0)
+ .build()) {
+ session.open(false, 30_000);
+ try {
+ setStatus(victim, "READONLY");
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(victim), Collections.singletonList(NodeStatus.ReadOnly));
+ StatementExecutionException failure =
+ assertThrows(
+ StatementExecutionException.class, () -> session.executeNonQueryStatement(delete));
+ assertEquals(TSStatusCode.SYSTEM_READ_ONLY.getStatusCode(), failure.getStatusCode());
+ // One original request really executed on one Region and was rejected on the other.
+ // Cross-Region deletion does not promise atomic rollback.
+ assertRegionDeleteRows(targets[0], true);
+ assertRegionDeleteRows(targets[1], false);
+ } finally {
+ setStatus(victim, "RUNNING");
+ }
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(victim), Collections.singletonList(NodeStatus.Running));
+ // No writes occur between attempts. Repeating this same absolute range cannot remove
+ // a later rewrite, unlike a retry interleaved with new writes within the deleted range.
+ session.executeNonQueryStatement(delete);
+ assertRegionDeleteRows(targets[0], true);
+ assertRegionDeleteRows(targets[1], true);
+ session.executeNonQueryStatement("FLUSH");
+ assertRegionDeleteRows(targets[0], true);
+ assertRegionDeleteRows(targets[1], true);
+ }
+ }
+
+ private static void assertRegionDeleteRows(String database, boolean deleted) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result =
+ statement.executeQuery("SELECT n,t FROM " + database + ".d ORDER BY time")) {
+ TestUtils.assertResultSetEqual(
+ result,
+ "Time," + database + ".d.n," + database + ".d.t,",
+ new String[] {
+ "998,998,keep-998,",
+ "999," + (deleted ? "null" : "999") + ",keep-999,",
+ "1000," + (deleted ? "null" : "1000") + ",keep-1000,",
+ "1001," + (deleted ? "null" : "1001") + ",keep-1001,",
+ "1002,1002,keep-1002,"
+ });
+ }
+ }
+
+ private static void setStatus(DataNodeWrapper node, String status) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(node);
+ Statement statement = connection.createStatement()) {
+ statement.execute("SET SYSTEM TO " + status + " ON LOCAL");
+ }
+ }
+
+ private Tablet tablet(int device) {
+ Tablet tablet =
+ new Tablet(
+ databases[device] + ".d",
+ Arrays.asList(
+ new MeasurementSchema("n", TSDataType.INT64),
+ new MeasurementSchema("t", TSDataType.TEXT)),
+ 4);
+ long[] times = {1002, 2, 1001, 1};
+ for (int row = 0; row < times.length; row++) {
+ long time = times[row];
+ tablet.addTimestamp(row, time);
+ tablet.addValue("n", row, device * 10000L + time);
+ tablet.addValue("t", row, time % 2 == 0 ? null : "device-" + device + "-time-" + time);
+ }
+ return tablet;
+ }
+
+ private void assertData(int device, boolean written) throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("SELECT n,t FROM " + databases[device] + ".d")) {
+ long[] times = written ? new long[] {0, 1, 2, 1000, 1001, 1002} : new long[] {0, 1000};
+ for (long time : times) {
+ assertTrue(result.next());
+ assertEquals(time, result.getLong(1));
+ boolean baseline = time == 0 || time == 1000;
+ assertEquals(baseline ? -1 : device * 10000L + time, result.getLong(2));
+ assertFalse(result.wasNull());
+ assertEquals(
+ baseline ? "baseline" : time % 2 == 0 ? null : "device-" + device + "-time-" + time,
+ result.getString(3));
+ }
+ assertFalse(result.next());
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
index 1ec46f9d5192c..5747f3d6ff4aa 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
@@ -115,6 +115,123 @@ public static void tearDownClass() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void testDeleteDataRangeAndRejectedPathKeepSessionUsable() throws Exception {
+ // Reuse the database covered by this fixture's existing per-method cleanup.
+ final String database = "root.sg";
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement("CREATE DATABASE " + database);
+ for (String device : Arrays.asList("api", "sql")) {
+ session.executeNonQueryStatement(
+ "CREATE ALIGNED TIMESERIES " + database + "." + device + "(s1 INT64,s2 INT64)");
+ }
+ session.executeNonQueryStatement(
+ "CREATE TIMESERIES " + database + ".second.s1 WITH DATATYPE=INT64");
+ session.executeNonQueryStatement(
+ "CREATE TIMESERIES " + database + ".empty.s1 WITH DATATYPE=INT64");
+ for (int time = 0; time <= 5; time++) {
+ for (String device : Arrays.asList("api", "sql")) {
+ session.executeNonQueryStatement(
+ "INSERT INTO "
+ + database
+ + "."
+ + device
+ + "(time,s1,s2) ALIGNED VALUES("
+ + time
+ + ","
+ + (100 + time)
+ + ","
+ + (200 + time)
+ + ")");
+ }
+ session.executeNonQueryStatement(
+ "INSERT INTO "
+ + database
+ + ".second(time,s1) VALUES("
+ + time
+ + ","
+ + (500 + time)
+ + ")");
+ if (time == 2) {
+ session.executeNonQueryStatement("FLUSH");
+ }
+ }
+
+ // The RPC parses every path before dispatch. A valid first path must not be partially
+ // deleted when a later path is rejected, and the original Session must remain usable.
+ StatementExecutionException rejected =
+ assertThrows(
+ StatementExecutionException.class,
+ () -> session.deleteData(Arrays.asList(database + ".api.s2", "root..invalid"), 0, 5));
+ assertEquals(TSStatusCode.ILLEGAL_PATH.getStatusCode(), rejected.getStatusCode());
+ assertTrue(rejected.getMessage().contains("root..invalid"));
+ final String[] original = {
+ "0,100,200", "1,101,201", "2,102,202", "3,103,203", "4,104,204", "5,105,205"
+ };
+ assertDeleteSessionRows(session, database + ".api", "s1,s2", original);
+ assertDeleteSessionRows(session, database + ".sql", "s1,s2", original);
+ assertDeleteSessionRows(
+ session,
+ database + ".second",
+ "s1",
+ new String[] {"0,500", "1,501", "2,502", "3,503", "4,504", "5,505"});
+
+ // The explicit interval is closed at both ends, spans multiple paths and leaves the
+ // other aligned measurement intact. A schema-only series is a successful no-op.
+ session.deleteData(
+ Arrays.asList(database + ".api.s1", database + ".second.s1", database + ".empty.s1"),
+ 1,
+ 3);
+ session.executeNonQueryStatement(
+ "DELETE FROM " + database + ".sql.s1 WHERE time >= 1 AND time <= 3");
+ final String[] partial = {
+ "0,100,200", "1,null,201", "2,null,202", "3,null,203", "4,104,204", "5,105,205"
+ };
+ assertDeleteSessionRows(session, database + ".api", "s1,s2", partial);
+ assertDeleteSessionRows(session, database + ".sql", "s1,s2", partial);
+ assertDeleteSessionRows(
+ session, database + ".second", "s1", new String[] {"0,500", "4,504", "5,505"});
+ assertDeleteSessionRows(session, database + ".empty", "s1", new String[0]);
+
+ // Both end-only overloads include the endpoint. Compare against equivalent SQL without
+ // replacing the Session after the rejected request or the successful deletions.
+ session.deleteData(database + ".api.s1", 0);
+ session.deleteData(Arrays.asList(database + ".api.s2", database + ".second.s1"), 4);
+ session.executeNonQueryStatement("DELETE FROM " + database + ".sql.s1 WHERE time <= 0");
+ session.executeNonQueryStatement("DELETE FROM " + database + ".sql.s2 WHERE time <= 4");
+ for (String device : Arrays.asList("api", "sql")) {
+ session.executeNonQueryStatement(
+ "INSERT INTO " + database + "." + device + "(time,s1) ALIGNED VALUES(2,902)");
+ }
+ session.executeNonQueryStatement(
+ "INSERT INTO " + database + ".second(time,s1) VALUES(2,802)");
+ session.executeNonQueryStatement("FLUSH");
+ final String[] rewritten = {"2,902,null", "4,104,null", "5,105,205"};
+ assertDeleteSessionRows(session, database + ".api", "s1,s2", rewritten);
+ assertDeleteSessionRows(session, database + ".sql", "s1,s2", rewritten);
+ assertDeleteSessionRows(session, database + ".second", "s1", new String[] {"2,802", "5,505"});
+ }
+ }
+
+ private static void assertDeleteSessionRows(
+ ISession session, String device, String measurements, String[] expected) throws Exception {
+ try (SessionDataSet data =
+ session.executeQueryStatement(
+ "SELECT " + measurements + " FROM " + device + " ORDER BY time")) {
+ List actual = new ArrayList<>();
+ while (data.hasNext()) {
+ RowRecord row = data.next();
+ StringBuilder values = new StringBuilder(Long.toString(row.getTimestamp()));
+ for (Field field : row.getFields()) {
+ values.append(',').append(field.getDataType() == null ? "null" : field.getLongV());
+ }
+ actual.add(values.toString());
+ }
+ assertEquals(device, Arrays.asList(expected), actual);
+ }
+ }
+
@Test
@Category({LocalStandaloneIT.class, ClusterIT.class})
public void insertPartialTabletTest() {
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java
index 8ab83b6233506..9b00d6fd1fb6d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/SessionSeedNodeFallbackIT.java
@@ -36,6 +36,10 @@
import org.apache.iotdb.session.Session;
import org.apache.iotdb.session.pool.SessionPool;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Before;
@@ -44,13 +48,19 @@
import org.junit.runner.RunWith;
import java.lang.reflect.Field;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -73,6 +83,9 @@ public void setUp() {
.getCommonConfig()
.setSchemaReplicationFactor(3)
.setDataReplicationFactor(3)
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1)
+ .setTimePartitionInterval(1000)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
EnvFactory.getEnv().initClusterEnvironment(1, 3);
@@ -83,6 +96,260 @@ public void tearDown() {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Test
+ public void testOriginalWriteClientsRecoverCachedRoutesAfterLeaderCrash() throws Exception {
+ String[] devices = {"root.route_first.d", "root.route_second.d"};
+ Map> expected = new HashMap<>();
+ for (String device : devices) {
+ expected.put(device, new TreeMap<>());
+ }
+ // Create Regions before choosing the business seed so its address cannot accidentally be
+ // the leader whose cached write connection we need to break.
+ try (Session setup =
+ new Session.Builder()
+ .host(endpoint(2).getIp())
+ .port(endpoint(2).getPort())
+ .maxRetryCount(0)
+ .build()) {
+ setup.open(false, RECOVERY_CONNECTION_TIMEOUT_MS);
+ for (String device : devices) {
+ setup.executeNonQueryStatement(
+ "CREATE DATABASE " + device.substring(0, device.lastIndexOf('.')));
+ setup.executeNonQueryStatement("CREATE TIMESERIES " + device + ".n INT64");
+ setup.executeNonQueryStatement("CREATE TIMESERIES " + device + ".t TEXT");
+ setup.executeNonQueryStatement(
+ "INSERT INTO "
+ + device
+ + "(time,n,t) VALUES(0,"
+ + routeNumber(device, 0, devices)
+ + ",'"
+ + device
+ + "-0')");
+ expected.get(device).put(0L, device + "-0");
+ }
+ }
+ Map initialLeaders = awaitWriteLeaders(devices, null, 2);
+ int seedIndex = initialLeaders.get(devices[0]).equals(endpoint(0)) ? 1 : 0;
+ TEndPoint seed = endpoint(seedIndex);
+ // The seed stays alive; the test specifically crashes a cached write leader.
+ SessionPool pool =
+ new SessionPool.Builder()
+ .host(seed.getIp())
+ .port(seed.getPort())
+ .maxSize(1)
+ .connectionTimeoutInMs(RECOVERY_CONNECTION_TIMEOUT_MS)
+ .waitToGetSessionTimeoutInMs(RECOVERY_CONNECTION_TIMEOUT_MS)
+ .maxRetryCount(0)
+ .build();
+ try (Session session =
+ new Session.Builder().host(seed.getIp()).port(seed.getPort()).maxRetryCount(0).build()) {
+ session.open(false, RECOVERY_CONNECTION_TIMEOUT_MS);
+ writeRoutePhase(session, pool, devices, 0, expected);
+ // Repeat identical keys to ensure both clients have received the routing recommendations.
+ writeRoutePhase(session, pool, devices, 0, expected);
+ Map leaders = awaitWriteLeaders(devices, null, seedIndex);
+ Map sessionRoutes = writeRoutes(session);
+ Map poolRoutes = writeRoutes(pool);
+ TEndPoint victim =
+ Arrays.stream(devices)
+ .map(device -> sessionRoutes.get(device))
+ .filter(route -> route != null && !route.equals(seed))
+ .filter(
+ route ->
+ Arrays.stream(devices)
+ .anyMatch(
+ device ->
+ route.equals(leaders.get(device))
+ && route.equals(poolRoutes.get(device))))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new AssertionError(
+ "No shared cached leader: session="
+ + sessionRoutes
+ + ", pool="
+ + poolRoutes
+ + ", leaders="
+ + leaders));
+ int victimIndex = -1;
+ for (int index = 0; index < 3; index++) {
+ if (endpoint(index).equals(victim)) {
+ victimIndex = index;
+ }
+ }
+ assertTrue(victimIndex >= 0);
+ EnvFactory.getEnv().getDataNodeWrapper(victimIndex).stopForcibly();
+ try {
+ awaitStatus(victimIndex, NodeStatus.Unknown);
+ awaitWriteLeaders(devices, victim, seedIndex);
+ // Fixed keys make an outer retry after a transport failure safe; only returned success
+ // enters the ledger. The same Session and size-one pool must recover their old routes.
+ writeRoutePhase(session, pool, devices, 1, expected);
+ assertRouteData(devices, expected, session);
+ assertEquals(0, pool.currentOccupiedSize());
+ assertEquals(1, pool.currentAvailableSize());
+ } finally {
+ EnvFactory.getEnv().startDataNode(victimIndex);
+ }
+ awaitStatus(victimIndex, NodeStatus.Running);
+ awaitWriteLeaders(devices, null, seedIndex);
+ writeRoutePhase(session, pool, devices, 2, expected);
+ session.executeNonQueryStatement("FLUSH");
+ assertRouteData(devices, expected, session);
+ assertEquals(0, pool.currentOccupiedSize());
+ // The environment checks every DataNode, including the recovered old leader.
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ for (String device : devices) {
+ try (ResultSet result = statement.executeQuery("SELECT n,t FROM " + device)) {
+ for (Map.Entry row : expected.get(device).entrySet()) {
+ assertTrue(result.next());
+ assertEquals((long) row.getKey(), result.getLong(1));
+ assertEquals(routeNumber(device, row.getKey(), devices), result.getLong(2));
+ assertEquals(row.getValue(), result.getString(3));
+ }
+ assertFalse(result.next());
+ }
+ }
+ }
+ } finally {
+ pool.close();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map writeRoutes(Object client) throws Exception {
+ Field field = client.getClass().getDeclaredField("deviceIdToEndpoint");
+ field.setAccessible(true);
+ return new HashMap<>((Map) field.get(client));
+ }
+
+ private Map awaitWriteLeaders(
+ String[] devices, TEndPoint excluded, int probeIndex) {
+ Map leaders = new HashMap<>();
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ leaders.clear();
+ Map runningReplicas = new HashMap<>();
+ Map regionIds = new HashMap<>();
+ try (Session probe =
+ new Session.Builder()
+ .host(endpoint(probeIndex).getIp())
+ .port(endpoint(probeIndex).getPort())
+ .maxRetryCount(0)
+ .build()) {
+ probe.open(false, RECOVERY_CONNECTION_TIMEOUT_MS);
+ try (SessionDataSet result = probe.executeQueryStatement("SHOW REGIONS")) {
+ SessionDataSet.DataIterator rows = result.iterator();
+ while (rows.next()) {
+ String device = rows.getString("Database") + ".d";
+ if (!Arrays.asList(devices).contains(device)
+ || !"DataRegion".equals(rows.getString("Type"))) {
+ continue;
+ }
+ int id = rows.getInt("RegionId");
+ Integer previous = regionIds.put(device, id);
+ assertTrue(
+ "Expected one DataRegion per database", previous == null || previous == id);
+ TEndPoint replica =
+ new TEndPoint(rows.getString("RpcAddress"), rows.getInt("RpcPort"));
+ if ("Running".equals(rows.getString("Status")) && !replica.equals(excluded)) {
+ runningReplicas.merge(device, 1, Integer::sum);
+ if ("Leader".equals(rows.getString("Role"))) {
+ leaders.put(device, replica);
+ }
+ }
+ }
+ }
+ }
+ assertEquals(2, new HashSet<>(regionIds.values()).size());
+ assertEquals(2, leaders.size());
+ for (String device : devices) {
+ assertEquals(
+ excluded == null ? 3 : 2, (int) runningReplicas.getOrDefault(device, 0));
+ }
+ });
+ return new HashMap<>(leaders);
+ }
+
+ private void writeRoutePhase(
+ Session session,
+ SessionPool pool,
+ String[] devices,
+ int phase,
+ Map> expected)
+ throws Exception {
+ for (int writer = 0; writer < 2; writer++) {
+ final int client = writer;
+ Awaitility.await()
+ .atMost(90, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Map batch = new HashMap<>();
+ for (String device : devices) {
+ Tablet tablet =
+ new Tablet(
+ device,
+ Arrays.asList(
+ new MeasurementSchema("n", TSDataType.INT64),
+ new MeasurementSchema("t", TSDataType.TEXT)),
+ 2);
+ for (int row = 0; row < 2; row++) {
+ long time = phase * 3000L + client * 100L + (row == 0 ? 1001 : 1);
+ tablet.addTimestamp(row, time);
+ tablet.addValue("n", row, routeNumber(device, time, devices));
+ tablet.addValue("t", row, row == 0 ? device + "-" + time : null);
+ }
+ batch.put(device, tablet);
+ }
+ try {
+ if (client == 0) {
+ session.insertTablets(batch);
+ } else {
+ pool.insertTablets(batch);
+ }
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ // Leader/Running heartbeats can precede write readiness. Both transport and
+ // server-side failures must retry these same idempotent keys until success.
+ throw new AssertionError("Original writer has not recovered its cached route", e);
+ }
+ });
+ for (String device : devices) {
+ for (int row = 0; row < 2; row++) {
+ long time = phase * 3000L + writer * 100L + (row == 0 ? 1001 : 1);
+ expected.get(device).put(time, row == 0 ? device + "-" + time : null);
+ }
+ }
+ }
+ }
+
+ private static long routeNumber(String device, long time, String[] devices) {
+ return (device.equals(devices[0]) ? 0 : 100_000L) + time;
+ }
+
+ private void assertRouteData(
+ String[] devices, Map> expected, Session session) throws Exception {
+ for (String device : devices) {
+ try (SessionDataSet result = session.executeQueryStatement("SELECT n,t FROM " + device)) {
+ for (Map.Entry row : expected.get(device).entrySet()) {
+ assertTrue(result.hasNext());
+ RowRecord actual = result.next();
+ assertEquals((long) row.getKey(), actual.getTimestamp());
+ assertEquals(
+ routeNumber(device, row.getKey(), devices), actual.getFields().get(0).getLongV());
+ assertEquals(
+ row.getValue(),
+ actual.getFields().get(1).getDataType() == null
+ ? null
+ : actual.getFields().get(1).getStringValue());
+ }
+ assertFalse(result.hasNext());
+ }
+ }
+ }
+
@Test
public void testQueriesDuringRollingRestart() throws Exception {
SessionPool pool =
diff --git a/integration-test/src/test/java/org/apache/iotdb/util/MetadataRecoveryTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/util/MetadataRecoveryTestUtils.java
new file mode 100644
index 0000000000000..30be001fa2b9a
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/util/MetadataRecoveryTestUtils.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.util;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppBaseConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Properties;
+
+import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer;
+import static org.junit.Assert.assertEquals;
+
+/** Helpers shared by metadata recovery ITs; all clients stay on the requested DataNode. */
+public final class MetadataRecoveryTestUtils {
+ public static final String TIMESERIES_HEADER =
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,";
+
+ private MetadataRecoveryTestUtils() {}
+
+ public static Connection connection(DataNodeWrapper node, String dialect) throws SQLException {
+ return EnvFactory.getEnv()
+ .getConnection(node, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, dialect);
+ }
+
+ public static void execute(Connection connection, String... sql) throws SQLException {
+ try (Statement statement = connection.createStatement()) {
+ statement.setQueryTimeout(90);
+ for (String command : sql) {
+ statement.execute(command);
+ }
+ }
+ }
+
+ public static void assertRows(Connection connection, String sql, String header, String... rows)
+ throws SQLException {
+ try (Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery(sql)) {
+ TestUtils.assertResultSetEqualUnordered(result, header, Arrays.asList(rows));
+ }
+ }
+
+ public static void property(Object config, String key, String value) {
+ Properties properties = new Properties();
+ properties.setProperty(key, value);
+ ((MppBaseConfig) config).updateProperties(properties);
+ }
+
+ public static int schemaRegion(String database, String path) throws Exception {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final var result =
+ client.getSchemaPartitionTable(
+ new TSchemaPartitionReq(generatePatternTreeBuffer(new String[] {path})));
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode());
+ final var slots = result.getSchemaPartitionTable().get(database);
+ assertEquals("Expected exactly one partition for " + path, 1, slots.size());
+ return slots.values().iterator().next().getId();
+ }
+ }
+}