Skip to content

Normalize table DELETE predicate identifiers #19087

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jt2594838 merged 1 commit into from
Nov 3, 2026
Merged

Normalize table DELETE predicate identifiers #19087

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions ...gration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.iotdb.itbase.exception.ParallelRequestTimeoutException;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
Expand Down Expand Up @@ -935,16 +936,107 @@ public void testDeleteDataByAttributeFilterRejectsTooManyDevices() throws SQLExc
builder.append("(1, 'd").append(i).append("', 'same', ").append(i).append(')');
}
statement.execute(builder.toString());
statement.execute("FLUSH");
statement.execute(
"INSERT INTO delete_by_attr_limit(time,deviceId,attr1,s1) VALUES"
+ "(2,'d0','same',2000),(2,'d1000','same',3000)");
statement.execute(
"CREATE TABLE delete_by_attr_guard(deviceId STRING TAG, attr1 ATTRIBUTE, s1 INT32 FIELD)");
statement.execute(
"INSERT INTO delete_by_attr_guard(time,deviceId,attr1,s1) VALUES(1,'d0','same',9000)");
final List<String> expected = new ArrayList<>();
for (int i = 0; i <= 1000; i++) {
expected.add("1,d" + i + ",same," + i + ",");
}
expected.add("2,d0,same,2000,");
expected.add("2,d1000,same,3000,");

try {
statement.execute("DELETE FROM delete_by_attr_limit WHERE attr1 = 'same'");
fail("should not reach here!");
} catch (SQLException e) {
assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), e.getErrorCode());
assertTrue(e.getMessage().contains("Too many devices (1001)"));
assertTrue(e.getMessage().contains("limit is 1000"));
assertTrue(e.getMessage().contains("attr1"));
assertTrue(e.getMessage().contains("Please remove all attribute filters"));
}

// The analyzer rejects the complete operation before dispatching any data deletion.
// Check every key/value, including disk and memory data, rather than only the exception.
assertAttributeLimitRows(statement, expected);
// A narrower ATTRIBUTE + TAG predicate must work after the rejected request.
statement.execute(
"DELETE FROM delete_by_attr_limit WHERE attr1='same' AND deviceId='d0' AND time=1");
expected.remove("1,d0,same,0,");
statement.execute("FLUSH");
assertAttributeLimitRows(statement, expected);
try (ResultSet rows =
statement.executeQuery(
"SELECT cast(time AS INT64) AS t,deviceId,attr1,s1 FROM delete_by_attr_guard")) {
TestUtils.assertResultSetEqual(
rows, "t,deviceId,attr1,s1,", new String[] {"1,d0,same,9000,"});
}
}
}

@Test
public void testAttributeDeleteWithMixedCaseColumnNames() throws SQLException {
try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
Statement statement = connection.createStatement()) {
statement.execute("USE test");
statement.execute(
"CREATE TABLE delete_mixed_case(DeviceId STRING TAG, Attr1 ATTRIBUTE, s1 INT32 FIELD)");
statement.execute(
"INSERT INTO delete_mixed_case(time,DeviceId,Attr1,s1) VALUES"
+ "(1,'d0','same',10),(2,'d0','same',20),(1,'D0','same',30),"
+ "(1,'d1','Other',40),(2,'d1','Other',41),(1,'d2',null,50),"
+ "(1,'d3','Other',60),(1,'keep','same',70)");
final List<String> expected =
new ArrayList<>(
List.of(
"1,d0,same,10,",
"2,d0,same,20,",
"1,D0,same,30,",
"1,d1,Other,40,",
"2,d1,Other,41,",
"1,d2,null,50,",
"1,d3,Other,60,",
"1,keep,same,70,"));
final String[][] deletions = {
// This first request reproduced the missing-symbol NPE without a preceding rejection.
{"attr1='same' AND deviceId='d0' AND time=1", "1,d0,same,10,"},
{"ATTR1='same' AND DEVICEID='D0' AND TIME=1", "1,D0,same,30,"},
{"AtTr1 IS NULL AND DeViCeId='d2' AND TiMe=1", "1,d2,null,50,"},
{"AtTr1 LIKE 'Ot%' AND DeViCeId='d1' AND TiMe=1", "1,d1,Other,40,"},
{"AtTr1 IN ('Other') AND DeViCeId='d3' AND TiMe=1", "1,d3,Other,60,"},
{"AtTr1 IS NOT NULL AND DeViCeId='d0' AND TiMe=2", "2,d0,same,20,"}
};
for (String[] deletion : deletions) {
statement.execute("DELETE FROM delete_mixed_case WHERE " + deletion[0]);
assertTrue(expected.remove(deletion[1]));
assertMixedCaseAttributeRows(statement, expected);
}
statement.execute("FLUSH");
assertMixedCaseAttributeRows(statement, expected);
}
}

private static void assertMixedCaseAttributeRows(Statement statement, List<String> expected)
throws SQLException {
try (ResultSet rows =
statement.executeQuery(
"SELECT cast(time AS INT64) AS t,deviceId,attr1,s1 FROM delete_mixed_case")) {
TestUtils.assertResultSetEqualUnordered(rows, "t,deviceId,attr1,s1,", expected);
}
}

private static void assertAttributeLimitRows(Statement statement, List<String> expected)
throws SQLException {
try (ResultSet rows =
statement.executeQuery(
"SELECT cast(time AS INT64) AS t,deviceId,attr1,s1 FROM delete_by_attr_limit")) {
TestUtils.assertResultSetEqualUnordered(rows, "t,deviceId,attr1,s1,", expected);
}
}

Expand Down
14 changes: 8 additions & 6 deletions ...ore/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
Expand Down Expand Up @@ -643,7 +644,7 @@ private static PredicateParseResult parseIsNull(
throw new SemanticException(
DataNodeQueryMessages.LEFT_HAND_EXPRESSION_IS_NOT_AN_IDENTIFIER + leftHandExp);
}
String columnName = ((Identifier) leftHandExp).getValue();
String columnName = ((Identifier) leftHandExp).getValue().toLowerCase(Locale.ENGLISH);
final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName);
if (Objects.nonNull(columnSchema)
&& columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) {
Expand All @@ -669,7 +670,7 @@ private static PredicateParseResult parseIsNotNull(
throw new SemanticException(
DataNodeQueryMessages.LEFT_HAND_EXPRESSION_IS_NOT_AN_IDENTIFIER + leftHandExp);
}
String columnName = ((Identifier) leftHandExp).getValue();
String columnName = ((Identifier) leftHandExp).getValue().toLowerCase(Locale.ENGLISH);
final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName);
if (Objects.nonNull(columnSchema)
&& columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) {
Expand All @@ -695,7 +696,7 @@ private static PredicateParseResult parseLike(
throw new SemanticException(
DataNodeQueryMessages.LEFT_HAND_EXPRESSION_IS_NOT_AN_IDENTIFIER + leftHandExp);
}
String columnName = ((Identifier) leftHandExp).getValue();
String columnName = ((Identifier) leftHandExp).getValue().toLowerCase(Locale.ENGLISH);
final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName);
if (Objects.nonNull(columnSchema)
&& columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) {
Expand All @@ -721,7 +722,7 @@ private static PredicateParseResult parseIn(
throw new SemanticException(
DataNodeQueryMessages.LEFT_HAND_EXPRESSION_IS_NOT_AN_IDENTIFIER + leftHandExp);
}
String columnName = ((Identifier) leftHandExp).getValue();
String columnName = ((Identifier) leftHandExp).getValue().toLowerCase(Locale.ENGLISH);
final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName);
if (Objects.nonNull(columnSchema)
&& columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) {
Expand Down Expand Up @@ -815,7 +816,7 @@ private static PredicateParseResult parseComparison(
return PredicateParseResult.time(oldPredicate);
}
// tag predicate
String columnName = identifier.getValue();
String columnName = identifier.getValue().toLowerCase(Locale.ENGLISH);
final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName);
if (Objects.nonNull(columnSchema)
&& columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) {
Expand Down Expand Up @@ -926,7 +927,8 @@ public Expression rewriteIdentifier(
final Identifier node,
final Void context,
final ExpressionTreeRewriter<Void> treeRewriter) {
return new SymbolReference(node.getValue());
// DELETE bypasses normal column resolution; use the schema's canonical names.
return new SymbolReference(node.getValue().toLowerCase(Locale.ENGLISH));
}
},
expression);
Expand Down
Loading