Skip to content

Fix table TTL reset propagation to DataNodes #19110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Caideyipi wants to merge 2 commits into
base: master
Choose a base branch
from
+392 −9
Draft

Fix table TTL reset propagation to DataNodes #19110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d6a8447
Fix table TTL reset propagation to DataNodes
Caideyipi Nov 3, 2026
295c34f
Fix table TTL default reset consistency
Caideyipi Nov 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions ...c/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
Expand Down Expand Up @@ -94,7 +95,9 @@ public void testTableSync() throws Exception {
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());

final String dbName = "test";
TableModelUtils.createDatabase(senderEnv, dbName, 300);
// Keep rows alive while asynchronous metadata and data synchronization finish.
final long databaseTtl = TimeUnit.DAYS.toMillis(1);
TableModelUtils.createDatabase(senderEnv, dbName, databaseTtl);

TestUtils.executeNonQueries(
dbName,
Expand All @@ -115,7 +118,7 @@ public void testTableSync() throws Exception {
receiverEnv,
"show tables from test",
"TableName,TTL(ms),",
Collections.singleton("table1,300,"),
Collections.singleton("table1," + databaseTtl + ","),
dbName);

// Test devices
Expand Down Expand Up @@ -159,7 +162,7 @@ public void testTableSync() throws Exception {
receiverEnv,
"show tables from test",
"TableName,TTL(ms),",
Collections.singleton("table1,300,"),
Collections.singleton("table1," + databaseTtl + ","),
dbName);

TestUtils.assertDataEventuallyOnEnv(
Expand Down
89 changes: 89 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@

import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.isession.ITableSession;
import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
import org.apache.iotdb.itbase.env.BaseEnv;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
Expand All @@ -52,6 +55,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableColumnHeaders;
import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableDetailsColumnHeaders;
Expand Down Expand Up @@ -79,6 +83,91 @@ public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}

@Test
public void testResetTableTtlUsesDatabaseDefault() throws Exception {
final String database = "ttl_default_reset";
final long day = TimeUnit.DAYS.toMillis(1);
final long old = System.currentTimeMillis() - 2 * day;
final long recent = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1);
try (final Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
final Statement statement = connection.createStatement()) {
try {
statement.execute("CREATE DATABASE " + database + " WITH (ttl=" + 7 * day + ")");
statement.execute("USE " + database);
statement.execute("CREATE TABLE inherited(dev STRING TAG,v INT32 FIELD)");
statement.execute(
"CREATE TABLE overridden(dev STRING TAG,v INT32 FIELD) WITH (ttl=" + 30 * day + ")");
for (final String table : new String[] {"inherited", "overridden"}) {
statement.execute(
"INSERT INTO "
+ table
+ "(time,dev,v) VALUES("
+ old
+ ",'d',1),("
+ recent
+ ",'d',2)");
TestUtils.assertDataEventuallyOnEnv(
EnvFactory.getEnv(),
"SELECT v FROM " + table + " ORDER BY time",
"v,",
new HashSet<>(Arrays.asList("1,", "2,")),
30,
database,
null);
}

statement.execute("ALTER DATABASE " + database + " SET PROPERTIES ttl=" + day);
statement.execute("ALTER TABLE inherited SET PROPERTIES ttl=DEFAULT");
TestUtils.tableResultSetEqualTestUnordered(
"SHOW TABLES",
new String[] {"TableName", "TTL(ms)"},
new String[] {"inherited," + day + ",", "overridden," + 30 * day + ","},
database);
TestUtils.tableResultSetEqualTestUnordered(
"SELECT v FROM inherited", new String[] {"v"}, new String[] {"2,"}, database);
TestUtils.tableResultSetEqualTestUnordered(
"SELECT v FROM overridden", new String[] {"v"}, new String[] {"1,", "2,"}, database);
for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) {
try (final Connection writer =
EnvFactory.getEnv()
.getConnection(
node,
SessionConfig.DEFAULT_USER,
SessionConfig.DEFAULT_PASSWORD,
BaseEnv.TABLE_SQL_DIALECT);
final Statement write = writer.createStatement()) {
final SQLException exception =
Assert.assertThrows(
SQLException.class,
() ->
write.execute(
"INSERT INTO "
+ database
+ ".inherited(time,dev,v) VALUES("
+ (old + 1)
+ ",'d',3)"));
assertEquals(TSStatusCode.OUT_OF_TTL.getStatusCode(), exception.getErrorCode());
}
}

statement.execute("ALTER DATABASE " + database + " SET PROPERTIES ttl='INF'");
statement.execute("ALTER TABLE inherited SET PROPERTIES ttl=DEFAULT");
statement.execute("INSERT INTO inherited(time,dev,v) VALUES(" + (old + 1) + ",'d',3)");
TestUtils.assertDataEventuallyOnEnv(
EnvFactory.getEnv(),
"SELECT v FROM inherited WHERE v=3",
"v,",
Collections.singleton("3,"),
30,
database,
null);
} finally {
statement.execute("DROP DATABASE IF EXISTS " + database);
}
}
}

@Test
public void testManageTable() {
try (final Connection connection =
Expand Down
30 changes: 26 additions & 4 deletions ...ignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.exception.table.ColumnInAlterException;
import org.apache.iotdb.commons.exception.table.ColumnInDeletionException;
Expand Down Expand Up @@ -1968,7 +1969,8 @@
final Map<String, String> updatedProperties,
final boolean isTableView)
throws MetadataException {
final TsTable originalTable = getTableWithUsingStatusIfExists(database, tableName).orElse(null);
final TsTable originalTable =
getTableWithUsingStatusIfExists(database, tableName).map(TsTable::new).orElse(null);

if (Objects.isNull(originalTable)) {
return new Pair<>(
Expand All @@ -1994,12 +1996,32 @@
null);
}

// Resolve DEFAULT before comparing properties and preparing the table sent to DataNodes.
if (updatedProperties.containsKey(TsTable.TTL_PROPERTY)
&& Objects.isNull(updatedProperties.get(TsTable.TTL_PROPERTY))) {
try {
final TDatabaseSchema schema = getDatabaseSchemaByName(database);
updatedProperties.put(
TsTable.TTL_PROPERTY,
schema.isSetTTL() && schema.getTTL() != Long.MAX_VALUE
? String.valueOf(schema.getTTL())
: IoTDBConstant.TTL_INFINITE);
} catch (final DatabaseNotExistsException e) {
throw new MetadataException(e);
}
}

updatedProperties
.keySet()
.removeIf(
key ->
Objects.equals(
updatedProperties.get(key), originalTable.getPropValue(key).orElse(null)));
key -> {
if (TsTable.TTL_PROPERTY.equals(key)
&& IoTDBConstant.TTL_INFINITE.equalsIgnoreCase(updatedProperties.get(key))) {
return originalTable.getTableTTL() == Long.MAX_VALUE;
}
return Objects.equals(
updatedProperties.get(key), originalTable.getPropValue(key).orElse(null));
});
if (updatedProperties.isEmpty()) {
return new Pair<>(RpcUtils.SUCCESS_STATUS, null);
}
Expand Down Expand Up @@ -2105,7 +2127,7 @@
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
LOGGER.warn(
ManagerMessages
.LOG_FAILED_TO_INVALIDATE_LAST_CACHE_OF_DATABASE_ARG_ON_DATANODE_ARG_STATUS_ARG_E0B53CBD,

Check warning on line 2130 in iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Line is longer than 100 characters (found 113). 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaGkPfGb1zV7SpaJCXT0&open=AaGkPfGb1zV7SpaJCXT0&pullRequest=19110
database,
dataNodeId,
status);
Expand Down
16 changes: 14 additions & 2 deletions .../org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.confignode.procedure.impl.schema.table;

import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
Expand All @@ -35,7 +36,9 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

public class SetTablePropertiesProcedure extends AbstractSetPropertiesProcedure {

Expand Down Expand Up @@ -68,9 +71,18 @@ protected Pair<TSStatus, TsTable> updateProperties(final ConfigNodeProcedureEnv
@Override
protected ConfigPhysicalPlan createSetPropertiesPlan(
final Map<String, String> properties, final boolean isRollback) {
final Map<String, String> propertiesToSet =
isRollback
&& properties.containsKey(TsTable.TTL_PROPERTY)
&& Objects.isNull(properties.get(TsTable.TTL_PROPERTY))
? new HashMap<>(properties)
: properties;
if (propertiesToSet != properties) {
propertiesToSet.put(TsTable.TTL_PROPERTY, IoTDBConstant.TTL_INFINITE);
}
return this instanceof SetViewPropertiesProcedure
? new SetViewPropertiesPlan(database, tableName, properties)
: new SetTablePropertiesPlan(database, tableName, properties);
? new SetViewPropertiesPlan(database, tableName, propertiesToSet)
: new SetTablePropertiesPlan(database, tableName, propertiesToSet);
}

@Override
Expand Down
Loading
Loading