From 5ddad2609a222ff8f593f9be8fc7f6d4ee4ec431 Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Sun, 14 Jun 2026 20:28:52 +1000 Subject: [PATCH] Add iotdb-thingsboard-table module for ThingsBoard on IoTDB Table Mode Implements ThingsBoard's historical telemetry TimeseriesDao SPI on Apache IoTDB 2.0.8 Table Mode (ITableSession SQL + Tablet writes). This first PR delivers the write + raw-read foundation: IoTDBTableBaseDao, an async bounded-queue batch writer, and the raw findAllAsync/remove read-delete path. Aggregation, latest telemetry and attributes land in later PRs. The module is inert by default: the live DAO/pool/writer/schema-bootstrap activate only on an explicit database.ts.type=iotdb-table plus iotdb.ts.experimental-raw-only=true opt-in, the auto-configuration is classpath-isolated from a non-ThingsBoard runtime, and ThingsBoard SPI types are a compile-only source surface excluded from the built jar (Strategy F). It is added to the reactor through a JDK-17 profile and overrides tsfile only within its own module pom, so the rest of the reactor is unaffected. Signed-off-by: Zihan Dai <99155080+PDGGK@users.noreply.github.com> --- iotdb-thingsboard-table/CI-NOTES.md | 49 + iotdb-thingsboard-table/README.md | 162 ++ .../docker-compose.test.yml | 94 ++ iotdb-thingsboard-table/pom.xml | 325 ++++ .../table/IoTDBTableAttributesDao.java | 42 + .../thingsboard/table/IoTDBTableBaseDao.java | 91 ++ .../thingsboard/table/IoTDBTableConfig.java | 129 ++ .../table/IoTDBTableConfiguration.java | 206 +++ .../IoTDBTableDaoShuttingDownException.java | 25 + .../thingsboard/table/IoTDBTableLabelDao.java | 42 + .../table/IoTDBTableLatestDao.java | 42 + .../table/IoTDBTablePendingSave.java | 71 + .../IoTDBTableRawOnlyEnabledCondition.java | 42 + .../IoTDBTableReadQueueFullException.java | 31 + .../table/IoTDBTableSaveIdentity.java | 22 + .../IoTDBTableSaveQueueFullException.java | 25 + .../table/IoTDBTableSchemaBootstrap.java | 148 ++ .../table/IoTDBTableTimeseriesDao.java | 489 ++++++ .../table/IoTDBTableTimeseriesWriter.java | 551 +++++++ .../IoTDBTableTimeseriesWriterStats.java | 29 + .../thingsboard/table/TypedKvValue.java | 94 ++ .../main/resources/META-INF/spring.factories | 20 + ...ot.autoconfigure.AutoConfiguration.imports | 19 + .../src/main/resources/schema-iotdb-table.sql | 44 + .../server/common/data/EntityType.java | 47 + .../server/common/data/HasVersion.java | 27 + .../server/common/data/id/EntityId.java | 39 + .../server/common/data/id/HasUUID.java | 27 + .../server/common/data/id/TenantId.java | 46 + .../server/common/data/id/UUIDBased.java | 70 + .../server/common/data/kv/Aggregation.java | 30 + .../common/data/kv/AggregationParams.java | 116 ++ .../common/data/kv/BaseDeleteTsKvQuery.java | 52 + .../common/data/kv/BaseReadTsKvQuery.java | 100 ++ .../server/common/data/kv/BaseTsKvQuery.java | 62 + .../server/common/data/kv/BasicKvEntry.java | 83 + .../server/common/data/kv/BasicTsKvEntry.java | 118 ++ .../common/data/kv/BooleanDataEntry.java | 69 + .../server/common/data/kv/DataType.java | 29 + .../common/data/kv/DeleteTsKvQuery.java | 27 + .../common/data/kv/DoubleDataEntry.java | 69 + .../server/common/data/kv/IntervalType.java | 29 + .../server/common/data/kv/JsonDataEntry.java | 69 + .../server/common/data/kv/KvEntry.java | 44 + .../server/common/data/kv/LongDataEntry.java | 69 + .../server/common/data/kv/ReadTsKvQuery.java | 37 + .../common/data/kv/ReadTsKvQueryResult.java | 76 + .../common/data/kv/StringDataEntry.java | 69 + .../server/common/data/kv/TsKvEntry.java | 38 + .../server/common/data/kv/TsKvQuery.java | 31 + .../server/common/data/query/TsValue.java | 51 + .../server/dao/timeseries/TimeseriesDao.java | 45 + .../IoTDBTableAutoConfigurationTest.java | 265 +++ .../table/IoTDBTableBaseDaoTest.java | 110 ++ .../table/IoTDBTableConfigTest.java | 180 ++ .../table/IoTDBTableContextStartupTest.java | 222 +++ .../table/IoTDBTableSchemaBootstrapTest.java | 186 +++ .../table/IoTDBTableTimeseriesDaoIT.java | 756 +++++++++ .../table/IoTDBTableTimeseriesDaoTest.java | 1449 +++++++++++++++++ .../table/StrategyFContractTest.java | 176 ++ pom.xml | 15 + 61 files changed, 7720 insertions(+) create mode 100644 iotdb-thingsboard-table/CI-NOTES.md create mode 100644 iotdb-thingsboard-table/README.md create mode 100644 iotdb-thingsboard-table/docker-compose.test.yml create mode 100644 iotdb-thingsboard-table/pom.xml create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAttributesDao.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDao.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfig.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfiguration.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableDaoShuttingDownException.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLabelDao.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTablePendingSave.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableRawOnlyEnabledCondition.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableReadQueueFullException.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveIdentity.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveQueueFullException.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrap.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriter.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriterStats.java create mode 100644 iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/TypedKvValue.java create mode 100644 iotdb-thingsboard-table/src/main/resources/META-INF/spring.factories create mode 100644 iotdb-thingsboard-table/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 iotdb-thingsboard-table/src/main/resources/schema-iotdb-table.sql create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/EntityType.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/HasVersion.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/EntityId.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/HasUUID.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/TenantId.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/UUIDBased.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/Aggregation.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/AggregationParams.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseDeleteTsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DataType.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DeleteTsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/IntervalType.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/KvEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/LongDataEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/StringDataEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvEntry.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvQuery.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/query/TsValue.java create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAutoConfigurationTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDaoTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfigTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableContextStartupTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrapTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/StrategyFContractTest.java diff --git a/iotdb-thingsboard-table/CI-NOTES.md b/iotdb-thingsboard-table/CI-NOTES.md new file mode 100644 index 00000000..cfa430b8 --- /dev/null +++ b/iotdb-thingsboard-table/CI-NOTES.md @@ -0,0 +1,49 @@ + + +# CI Notes + +The `iotdb-extras` parent reactor builds and tests this module only on JDK 17+: +the root pom adds it to `` through a profile activated by +`[17,)` (it compiles with Java 17 language features), so the root +build compiles and tests it on the 17/21 jobs and skips it on the 8/11 jobs. This +file is a developer reference of the local checks for the `iotdb-thingsboard-table` +module; it is not itself a GitHub Actions workflow. + +## Candidate Checks + +- Compile from the standalone module directory: + `mvn compile -DskipTests` +- Run unit tests: + `mvn test` +- Validate the local stack file: + `docker compose -f docker-compose.test.yml config` +- Run Docker-backed integration tests only when Docker is available: + `mvn -Piotdb-table-it verify` +- Start the optional local stack only when required environment values are set: + `TB_POSTGRES_USER= TB_POSTGRES_PASSWORD= IOTDB_USERNAME= IOTDB_PASSWORD= docker compose -f docker-compose.test.yml up -d` + +## Notes + +- Keep this file inside the module. Do not copy it to `.github/workflows`. +- Do not store passwords, tokens, or local hostnames in CI configuration. +- Keep the Docker image tags aligned with the versions exercised by this module's + integration-test profile. diff --git a/iotdb-thingsboard-table/README.md b/iotdb-thingsboard-table/README.md new file mode 100644 index 00000000..07dece9f --- /dev/null +++ b/iotdb-thingsboard-table/README.md @@ -0,0 +1,162 @@ + + +# IoTDB ThingsBoard Table + +## Overview + +`iotdb-thingsboard-table` is a ThingsBoard historical-telemetry DAO backend +built on Apache IoTDB 2.0.8 Table Mode. It lets a ThingsBoard deployment store +and serve time-series telemetry through IoTDB's table-session API instead of the +default Cassandra/SQL backends. The module targets ThingsBoard v4.3.1.2. Because +it compiles with Java 17 language features (records and others), the +`iotdb-extras` parent reactor builds and tests it only on JDK 17+: the root pom +adds it to `` through a profile activated by `[17,)`, so the +JDK 8/11 reactor jobs skip it while the 17/21 jobs build it as part of the root +project. + +## ThingsBoard SPI surface (Strategy F) + +The ThingsBoard DAO SPI and value types (`org.thingsboard.*`) are not published +to Maven Central, so they cannot be a normal compile dependency. Strategy F +treats them as a **compile-only source surface** under `src/provided/java`: just +enough of the ThingsBoard interfaces and value objects to compile against. The +maven-jar-plugin excludes `org/thingsboard/**` from the built jar, so these +compile-only types never ship and never shadow the real ThingsBoard classes. At +runtime the actual ThingsBoard classpath supplies them. This keeps the module +buildable in isolation while binding to the genuine ThingsBoard types on a real +deployment. + +The compile-only surface under `src/provided/java` was manually verified against +ThingsBoard `v4.3.1.2` (commit `c37fb509`): the `TimeseriesDao` SPI methods the +DAO consumes and the value-object accessors it reads were checked against the +upstream sources. A fully-automated check against the upstream artifact is not +possible because ThingsBoard's `dao`/`common-data` modules are not published to +Maven Central (the reason for Strategy F). As a guard against silent drift, +`StrategyFContractTest` pins the exact `TimeseriesDao` SPI method signatures the +DAO depends on, so any accidental edit to the local surface fails the build. + +## Scope + +This initial module delivers an inert-by-default foundation: `IoTDBTableBaseDao` +(session-pool lifecycle, schema/table bootstrap) and the +`IoTDBTableTimeseriesDao` write path (`save`), raw read, and delete. To exercise +it, set both `database.ts.type=iotdb-table` and +`iotdb.ts.experimental-raw-only=true`. Aggregation, latest telemetry, and +attribute/label DAOs are outside the current scope. + +> **This is an incremental / experimental backend.** Explicitly enabling it +> routes ThingsBoard historical telemetry through IoTDB Table Mode for **raw read +> + write + delete only**. **Time-bucketed aggregation is NOT implemented yet**; +> aggregation, latest telemetry, and attributes are outside the current scope. + +## Known limitations + +**Same-timestamp type change across separate flushes.** The writer collapses +duplicate `(tenant, entity, key, timestamp)` saves *within a single flush* so the +last write wins, but it does not yet defend against a same-`(tenant, entity, key, +timestamp)` save whose value *type* changes between two **separate** flushes (for +example a `LONG` written in one flush and a `STRING` written at the same +timestamp in a later flush). Because each typed value lands in its own column +(`long_v`, `str_v`, ...), that single point can end up with two non-null typed +columns. + +This is a deliberate current-scope decision: the cleanup that would prevent it (a +delete-then-insert overwrite on every save) is outside the current scope. The +behavior is **fail-fast, not silent**: a raw read of that one poisoned point +throws an `IllegalStateException` (the single-typed-column invariant enforced in +`IoTDBTableBaseDao`) rather than returning a wrong value. Every other point is +unaffected. This documented behavior is pinned by an integration test +(`IoTDBTableTimeseriesDaoIT`). + +## Configuration + +The backend is bound from `iotdb.*` Spring properties (see `IoTDBTableConfig`). +Key activation and operational flags: + +| Property | Default | Meaning | +| --- | --- | --- | +| `database.ts.type` | _(unset)_ | Set to `iotdb-table` as the ThingsBoard historical-timeseries backend selector. | +| `iotdb.ts.experimental-raw-only` | `false` | Explicit opt-in for this initial raw-only backend. Must be `true` together with `database.ts.type=iotdb-table`; write, raw read, and delete are implemented, while time-bucketed aggregation is outside the current scope. | +| `iotdb.host` / `iotdb.port` | `127.0.0.1` / `6667` | IoTDB node address. | +| `iotdb.username` / `iotdb.password` | `root` / `root` | IoTDB credentials. | +| `iotdb.database` | `thingsboard` | Target IoTDB database. | +| `iotdb.session-pool-size` | `8` | Table session pool size. | +| `iotdb.schema.bootstrap` | `true` | When `true`, the module runs an idempotent startup bootstrap that reads `schema-iotdb-table.sql` from the classpath and creates the `telemetry` / `entity_attributes` tables (and database) on a fresh IoTDB before the first write. Set to `false` if you manage the schema out-of-band. | + +The module is a Spring Boot **auto-configuration** +(`IoTDBTableConfiguration`), registered via +`META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports` +(and `META-INF/spring.factories` as a fallback), so it activates in a real +ThingsBoard deployment without the host application having to component-scan +`org.apache.iotdb.extras`. + +## Build + +The module builds from the repository root as part of the reactor: + +```bash +# from the iotdb-extras repository root +mvn -pl iotdb-thingsboard-table -am clean test +``` + +It can also be built standalone from the module directory: + +```bash +cd iotdb-thingsboard-table +mvn compile -DskipTests +``` + +## Test + +Run the Java test scaffold from the module directory: + +```bash +mvn test +``` + +Run the Docker-backed integration tests only when required: + +```bash +mvn -Piotdb-table-it verify +``` + +Start the local integration stack with explicit environment values: + +```bash +TB_POSTGRES_USER= TB_POSTGRES_PASSWORD= \ + IOTDB_USERNAME= IOTDB_PASSWORD= \ + docker compose -f docker-compose.test.yml up -d +``` + +Stop and remove the local stack: + +```bash +docker compose -f docker-compose.test.yml down -v +``` + +## Status + +Initial module status: `IoTDBTableBaseDao` plus the `IoTDBTableTimeseriesDao` +write, raw-read, and delete paths are implemented behind +`database.ts.type=iotdb-table` and `iotdb.ts.experimental-raw-only=true`. +Without both properties, the module is inert. Aggregation, latest telemetry, and +attributes are outside the current scope. diff --git a/iotdb-thingsboard-table/docker-compose.test.yml b/iotdb-thingsboard-table/docker-compose.test.yml new file mode 100644 index 00000000..f0014849 --- /dev/null +++ b/iotdb-thingsboard-table/docker-compose.test.yml @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +services: + iotdb: + image: apache/iotdb:2.0.8-standalone + container_name: iotdb-table-test + environment: + IOTDB_USERNAME: ${IOTDB_USERNAME:?set IOTDB_USERNAME} + IOTDB_PASSWORD: ${IOTDB_PASSWORD:?set IOTDB_PASSWORD} + ports: + - "${IOTDB_RPC_PORT:-6667}:6667" + networks: + - tb-iotdb-test + healthcheck: + test: ["CMD-SHELL", "bash -ec ': >/dev/tcp/127.0.0.1/6667'"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + + postgres: + image: postgres:15 + container_name: tb-postgres-test + environment: + POSTGRES_DB: ${TB_POSTGRES_DB:-thingsboard} + POSTGRES_USER: ${TB_POSTGRES_USER:?set TB_POSTGRES_USER} + POSTGRES_PASSWORD: ${TB_POSTGRES_PASSWORD:?set TB_POSTGRES_PASSWORD} + ports: + - "${TB_POSTGRES_PORT:-5432}:5432" + networks: + - tb-iotdb-test + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\"", + ] + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + + thingsboard: + image: thingsboard/tb-node:4.3.1.2 + container_name: thingsboard-table-test + depends_on: + iotdb: + condition: service_healthy + postgres: + condition: service_healthy + environment: + TB_QUEUE_TYPE: ${TB_QUEUE_TYPE:-in-memory} + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${TB_POSTGRES_DB:-thingsboard} + SPRING_DATASOURCE_USERNAME: ${TB_POSTGRES_USER:?set TB_POSTGRES_USER} + SPRING_DATASOURCE_PASSWORD: ${TB_POSTGRES_PASSWORD:?set TB_POSTGRES_PASSWORD} + DATABASE_TS_TYPE: ${DATABASE_TS_TYPE:-sql} + DATABASE_TS_LATEST_TYPE: ${DATABASE_TS_LATEST_TYPE:-sql} + IOTDB_HOST: ${IOTDB_HOST:-iotdb} + IOTDB_PORT: ${IOTDB_PORT:-6667} + IOTDB_USERNAME: ${IOTDB_USERNAME:?set IOTDB_USERNAME} + IOTDB_PASSWORD: ${IOTDB_PASSWORD:?set IOTDB_PASSWORD} + INSTALL_TB: ${INSTALL_TB:-true} + LOAD_DEMO: ${LOAD_DEMO:-false} + ports: + - "${TB_HTTP_PORT:-8080}:8080" + networks: + - tb-iotdb-test + healthcheck: + test: ["CMD-SHELL", "bash -ec ': >/dev/tcp/127.0.0.1/8080'"] + interval: 15s + timeout: 5s + retries: 20 + start_period: 90s + +networks: + tb-iotdb-test: + driver: bridge diff --git a/iotdb-thingsboard-table/pom.xml b/iotdb-thingsboard-table/pom.xml new file mode 100644 index 00000000..70cdd983 --- /dev/null +++ b/iotdb-thingsboard-table/pom.xml @@ -0,0 +1,325 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-extras-parent + 2.0.4-SNAPSHOT + ../pom.xml + + iotdb-thingsboard-table + IoTDB Extras: ThingsBoard Table + IoTDB table-session integration support for ThingsBoard storage experiments. + + 17 + 17 + 17 + UTF-8 + 2.0.8 + + 2.3.0 + 32.1.3-jre + 3.0.2 + 5.10.2 + 1.18.42 + 5.21.0 + 1.21.3 + 4.3.1.2 + 3.13.0 + 3.3.0 + 3.2.5 + 3.2.5 + + + + org.apache.iotdb + iotdb-session + ${iotdb.version} + + + com.google.guava + guava + ${guava.version} + provided + + + org.springframework + spring-core + ${spring.version} + provided + + + org.springframework + spring-beans + provided + + + org.springframework + spring-context + provided + + + org.springframework.boot + spring-boot-autoconfigure + provided + + + jakarta.validation + jakarta.validation-api + provided + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.slf4j + slf4j-api + provided + + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + org.springframework.boot + spring-boot-test + ${spring-boot.version} + test + + + org.assertj + assertj-core + 3.22.0 + test + + + org.hibernate.validator + hibernate-validator + 8.0.2.Final + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + junit-jupiter + ${testcontainers.version} + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.plugin.version} + + ${maven.compiler.release} + + + org.projectlombok + lombok + ${lombok.version} + + + + + + default-compile + + + + ${project.basedir}/src/main/java + ${project.basedir}/src/provided/java + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + + + + org/thingsboard/** + org/apache/commons/** + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.plugin.version} + + + + default-test + none + + + integration-tests + none + + + unit-tests + + test + + test + + + **/*Test.java + + + **/*IT.java + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + check-dependencies + + + + com.google.guava:guava + + org.apache.iotdb:iotdb-session + + org.junit.jupiter:junit-jupiter + org.assertj:assertj-core + org.hibernate.validator:hibernate-validator + org.mockito:mockito-junit-jupiter + + + + org.apache.iotdb:isession + org.apache.tsfile:tsfile + org.apache.tsfile:common + org.apache.iotdb:service-rpc + org.springframework.boot:spring-boot + + org.junit.jupiter:junit-jupiter-api + + + + + + + + + + iotdb-table-it + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven.failsafe.plugin.version} + + + ${project.build.outputDirectory} + + **/*IT.java + + + + + run-integration-tests + integration-test + + integration-test + + + + verify-integration-tests + verify + + verify + + + + + + + + + diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAttributesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAttributesDao.java new file mode 100644 index 00000000..188419b9 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAttributesDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import lombok.extern.slf4j.Slf4j; + +/** + * Attribute DAO skeleton for the IoTDB Table Mode backend. + * + *

Spring activation is intentionally absent because upstream ThingsBoard does not expose an + * AttributesDao selector yet. The activation property and SPI binding should be introduced together + * when the attributes path is implemented. + * + *

Strategy F keeps this class free of ThingsBoard imports and interface clauses until the + * attribute path is implemented. + */ +@Slf4j +public class IoTDBTableAttributesDao extends IoTDBTableBaseDao { + // Not annotated @Repository yet: this must not auto-register until ThingsBoard exposes an + // AttributesDao selector and this module provides the matching implementation. + public IoTDBTableAttributesDao(ITableSessionPool tableSessionPool) { + super(tableSessionPool); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDao.java new file mode 100644 index 00000000..cca44013 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDao.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.StatementExecutionException; + +import lombok.extern.slf4j.Slf4j; + +/** + * Base class for IoTDB Table Mode DAOs; not a Spring bean itself. Concrete DAOs declare + * {@code @Repository} and the activation conditional. Holds the shared {@code ITableSessionPool} + * (wired via constructor injection) and provides type-mapping helpers used by concrete DAOs + * (TimeseriesDao, LatestDao, AttributesDao, LabelDao). + * + *

Strategy F keeps this class free of ThingsBoard imports and interface clauses; concrete DAOs + * bind to the ThingsBoard SPI types directly. + */ +@Slf4j +public class IoTDBTableBaseDao { + protected final ITableSessionPool tableSessionPool; + + public IoTDBTableBaseDao(ITableSessionPool tableSessionPool) { + this.tableSessionPool = tableSessionPool; + } + + /** + * Maps a single IoTDB Table Mode telemetry row's 5 typed FIELD columns to a TypedKvValue. Exactly + * one typed value column may be non-null because the telemetry schema stores exactly one typed + * value per row. Throws IllegalStateException if more than one is non-null (fail-fast on schema + * invariant violation). Returns TypedKvValue.empty() if all 5 are null (caller decides how to + * handle). The caller must call row.next() and receive true before invoking this method so the + * iterator is positioned on a row. + */ + public TypedKvValue getEntry(SessionDataSet.DataIterator row) throws StatementExecutionException { + boolean hasBoolean = !row.isNull("bool_v"); + boolean hasLong = !row.isNull("long_v"); + boolean hasDouble = !row.isNull("double_v"); + boolean hasString = !row.isNull("str_v"); + boolean hasJson = !row.isNull("json_v"); + + int valueCount = 0; + valueCount += hasBoolean ? 1 : 0; + valueCount += hasLong ? 1 : 0; + valueCount += hasDouble ? 1 : 0; + valueCount += hasString ? 1 : 0; + valueCount += hasJson ? 1 : 0; + + if (valueCount == 0) { + return TypedKvValue.empty(); + } + if (valueCount > 1) { + throw new IllegalStateException( + "IoTDB telemetry row has " + + valueCount + + " typed value columns set; the telemetry schema stores exactly one typed value " + + "column per row. " + + "Possible same-timestamp type-change bug or stale data."); + } + if (hasBoolean) { + return TypedKvValue.ofBoolean(row.getBoolean("bool_v")); + } + if (hasLong) { + return TypedKvValue.ofLong(row.getLong("long_v")); + } + if (hasDouble) { + return TypedKvValue.ofDouble(row.getDouble("double_v")); + } + if (hasString) { + return TypedKvValue.ofString(row.getString("str_v")); + } + return TypedKvValue.ofJson(row.getString("json_v")); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfig.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfig.java new file mode 100644 index 00000000..88eb6a65 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfig.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the IoTDB Table Mode DAO backend. Bound from {@code iotdb.*} in + * Spring application config. + */ +@Data +@Validated +@ConfigurationProperties(prefix = "iotdb") +public class IoTDBTableConfig { + + @NotBlank private String host = "127.0.0.1"; + + @Min(1) + @Max(65535) + private int port = 6667; + + @Min(1) + @Max(1024) + private int sessionPoolSize = 8; + + /** + * Target IoTDB database. The bootstrap splices this name verbatim into {@code CREATE DATABASE} / + * {@code USE} DDL, so it is constrained to the IoTDB identifier rule (letter or underscore first, + * then letters, digits or underscores) to reject names that could break or inject into that DDL. + */ + @NotBlank + @Pattern( + regexp = "^[A-Za-z_][A-Za-z0-9_]*$", + message = + "must be a valid IoTDB identifier: a letter or underscore followed by letters, digits or" + + " underscores") + private String database = "thingsboard"; + + /** + * Affects ThingsBoard storage data-point accounting only; it does not configure IoTDB physical + * retention. + */ + @Min(-1) + private long defaultTtlMs = -1L; + + @NotBlank private String username = "root"; + + @NotBlank private String password = "root"; + + @Min(100) + private int connectionTimeoutMs = 5000; + + private boolean enableCompression = false; + + @Valid private Ts ts = new Ts(); + + @Data + public static class Ts { + /** + * Explicit opt-in for the raw-only backend. This backend currently implements write, raw read, + * and delete only; time-bucketed aggregation is outside the current scope, so it is not + * production-complete and must be enabled deliberately. + */ + private boolean experimentalRawOnly = false; + + @Valid private Save save = new Save(); + @Valid private Read read = new Read(); + } + + @Data + public static class Read { + @Min(1) + private int threads = 4; + + @Min(1) + private int queueCapacity = 10000; + } + + @Data + public static class Save { + @Min(1) + private int batchSize = 500; + + @Min(1) + private long maxLingerMs = 20L; + + @Min(1) + private int queueCapacity = 50000; + + @Min(1) + private long shutdownDrainTimeoutMs = 5000L; + + @Min(1) + @Max(1) + private int flushThreads = 1; + + @Min(1) + private int retryMaxAttempts = 3; + + @Min(0) + private long retryInitialBackoffMs = 50L; + + @Min(0) + private long retryMaxBackoffMs = 1000L; + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfiguration.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfiguration.java new file mode 100644 index 00000000..cada2906 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfiguration.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.ResolvableType; +import org.springframework.util.ClassUtils; + +import java.util.List; + +/** + * Spring Boot auto-configuration entry point for the IoTDB Table Mode backend. + * + *

This class is registered via {@code + * META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports} (Spring Boot + * 3.x mechanism) and, as a belt-and-suspenders fallback, via {@code META-INF/spring.factories}, so + * the module activates in a real ThingsBoard deployment without the host application having to + * component-scan {@code org.apache.iotdb.extras}. + * + *

The {@code @Bean} methods below explicitly register the session pool, the timeseries writer, + * the schema bootstrap, and the {@code @Repository} {@link IoTDBTableTimeseriesDao}. Explicit bean + * methods are used in preference to {@code @ComponentScan}, which Spring deliberately filters out + * of auto-configuration classes (it would otherwise re-scan the host application's packages). Each + * bean stays under the same selected-and-explicitly-enabled activation guard, so this foundation is + * inert unless {@code database.ts.type=iotdb-table} and {@code iotdb.ts.experimental-raw-only=true} + * are both set. This initial module delivers only the timeseries backend; the latest-telemetry and + * label selectors return when those DAOs are implemented. + */ +@Slf4j +@AutoConfiguration +@ConditionalOnClass(name = IoTDBTableConfiguration.TIMESERIES_DAO_CLASS_NAME) +public class IoTDBTableConfiguration { + static final String IOTDB_TABLE_SESSION_POOL_BEAN_NAME = "iotdbThingsboardTableSessionPool"; + static final String IOTDB_TABLE_TIMESERIES_DAO_BEAN_NAME = "ioTDBTableTimeseriesDao"; + static final String TIMESERIES_DAO_CLASS_NAME = + "org.thingsboard.server.dao.timeseries.TimeseriesDao"; + + @Configuration(proxyBeanMethods = false) + @ConditionalOnClass(name = TIMESERIES_DAO_CLASS_NAME) + @Conditional(IoTDBTableRawOnlyEnabledCondition.class) + @EnableConfigurationProperties(IoTDBTableConfig.class) + static class EnabledRawOnlyConfiguration { + + // This module implements only the timeseries backend. The latest-telemetry + // (database.ts_latest.type) and label (iotdb.labels.enabled) selectors are intentionally NOT + // included here: those DAOs do not exist yet, so they must not spin up a session pool or schema + // bootstrap for a backend that has not shipped. Those conditions return when the corresponding + // DAOs are implemented. + @Bean(name = IOTDB_TABLE_SESSION_POOL_BEAN_NAME, destroyMethod = "close") + @ConditionalOnMissingBean(name = IOTDB_TABLE_SESSION_POOL_BEAN_NAME) + ITableSessionPool tableSessionPool(IoTDBTableConfig config) { + String nodeUrl = config.getHost() + ":" + config.getPort(); + ITableSessionPool pool = + new TableSessionPoolBuilder() + .nodeUrls(List.of(nodeUrl)) + .user(config.getUsername()) + .password(config.getPassword()) + .database(config.getDatabase()) + .maxSize(config.getSessionPoolSize()) + .connectionTimeoutInMs(config.getConnectionTimeoutMs()) + .enableIoTDBRpcCompression(config.isEnableCompression()) + .build(); + log.info( + "IoTDB Table Mode session pool initialized: nodeUrl={}, database={}, poolSize={}, compression={}, storageAccountingDefaultTtlMs={}", + nodeUrl, + config.getDatabase(), + config.getSessionPoolSize(), + config.isEnableCompression(), + config.getDefaultTtlMs()); + return pool; + } + + @Bean + IoTDBTableTimeseriesWriter timeseriesWriter( + @Qualifier(IOTDB_TABLE_SESSION_POOL_BEAN_NAME) ITableSessionPool tableSessionPool, + IoTDBTableConfig config) { + return new IoTDBTableTimeseriesWriter(tableSessionPool, config); + } + + /** + * Fails startup before any IoTDB pool/bootstrap/writer singleton is created if the explicit + * IoTDB backend selection conflicts with a host-provided TimeseriesDao. + */ + @Bean + static BeanFactoryPostProcessor timeseriesDaoConflictGuard() { + return new TimeseriesDaoConflictGuard(); + } + + /** + * Registers the historical-telemetry DAO. The bean name {@code ioTDBTableTimeseriesDao} matches + * the default component-scan name, and the string-based missing-bean guard avoids loading + * ThingsBoard classes while evaluating auto-configuration metadata. + */ + @Bean + @ConditionalOnBean(name = IOTDB_TABLE_SESSION_POOL_BEAN_NAME) + @ConditionalOnMissingBean(type = TIMESERIES_DAO_CLASS_NAME) + IoTDBTableTimeseriesDao ioTDBTableTimeseriesDao( + @Qualifier(IOTDB_TABLE_SESSION_POOL_BEAN_NAME) ITableSessionPool tableSessionPool, + IoTDBTableTimeseriesWriter timeseriesWriter, + IoTDBTableConfig config) { + return new IoTDBTableTimeseriesDao(tableSessionPool, timeseriesWriter, config); + } + + /** + * Idempotent startup schema bootstrap. Only registered when the IoTDB Table Mode backend is + * selected and explicitly enabled (same activation guard as the pool/DAO), the session pool + * bean is present, and {@code iotdb.schema.bootstrap} is not disabled (defaults to {@code + * true}). + */ + @Bean + @ConditionalOnBean(name = IOTDB_TABLE_SESSION_POOL_BEAN_NAME) + @ConditionalOnProperty( + name = "iotdb.schema.bootstrap", + havingValue = "true", + matchIfMissing = true) + IoTDBTableSchemaBootstrap schemaBootstrap( + @Qualifier(IOTDB_TABLE_SESSION_POOL_BEAN_NAME) ITableSessionPool tableSessionPool, + IoTDBTableConfig config) { + return new IoTDBTableSchemaBootstrap(tableSessionPool, config); + } + } + + private static final class TimeseriesDaoConflictGuard implements BeanFactoryPostProcessor { + @Override + public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) + throws BeansException { + Class timeseriesDaoType = resolveTimeseriesDaoClass(beanFactory); + for (String beanName : beanFactory.getBeanNamesForType(timeseriesDaoType, true, false)) { + if (!isIoTDBTimeseriesDaoBean(beanFactory, beanName)) { + throw new IllegalStateException( + "database.ts.type=iotdb-table with iotdb.ts.experimental-raw-only=true, but a " + + "non-IoTDB TimeseriesDao bean '" + + beanName + + "' is present; remove it or unset the IoTDB selector"); + } + } + } + + private static boolean isIoTDBTimeseriesDaoBean( + ConfigurableListableBeanFactory beanFactory, String beanName) { + Class beanType = resolveBeanType(beanFactory, beanName); + if (beanType == null) { + throw new IllegalStateException( + "database.ts.type=iotdb-table with iotdb.ts.experimental-raw-only=true, but " + + "TimeseriesDao bean '" + + beanName + + "' has no resolvable type; expose a concrete IoTDBTableTimeseriesDao type or " + + "remove the bean"); + } + return beanType != null && IoTDBTableTimeseriesDao.class.isAssignableFrom(beanType); + } + + private static Class resolveBeanType( + ConfigurableListableBeanFactory beanFactory, String beanName) { + Class beanType = beanFactory.getType(beanName, false); + if (beanType != null || !beanFactory.containsBeanDefinition(beanName)) { + return beanType; + } + BeanDefinition beanDefinition = beanFactory.getBeanDefinition(beanName); + ResolvableType resolvableType = beanDefinition.getResolvableType(); + return resolvableType == ResolvableType.NONE ? null : resolvableType.resolve(); + } + + private static Class resolveTimeseriesDaoClass(ConfigurableListableBeanFactory beanFactory) { + try { + return ClassUtils.forName(TIMESERIES_DAO_CLASS_NAME, beanFactory.getBeanClassLoader()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + "IoTDB Table Mode backend was enabled but TimeseriesDao is not on the classpath", e); + } + } + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableDaoShuttingDownException.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableDaoShuttingDownException.java new file mode 100644 index 00000000..fee1d89f --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableDaoShuttingDownException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +public class IoTDBTableDaoShuttingDownException extends RuntimeException { + public IoTDBTableDaoShuttingDownException(String message) { + super(message); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLabelDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLabelDao.java new file mode 100644 index 00000000..c3aabe9f --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLabelDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import lombok.extern.slf4j.Slf4j; + +/** + * Label DAO skeleton for an optional IoTDB Table Mode label path. + * + *

This class is intentionally not annotated as a Spring bean. The optional label path is not + * implemented here, so registering it now would expose an {@code iotdb.labels.enabled=true} + * selector that ThingsBoard cannot bind to a working DAO. + * + *

Strategy F keeps this class free of ThingsBoard imports and interface clauses until the + * optional label path is implemented. + */ +@Slf4j +public class IoTDBTableLabelDao extends IoTDBTableBaseDao { + // Not annotated @Repository yet: auto-registering it now would advertise + // iotdb.labels.enabled=true with no working DAO behind it. + public IoTDBTableLabelDao(ITableSessionPool tableSessionPool) { + super(tableSessionPool); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java new file mode 100644 index 00000000..60b0c2b2 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import lombok.extern.slf4j.Slf4j; + +/** + * Latest telemetry DAO skeleton for the IoTDB Table Mode backend. + * + *

This class is intentionally not annotated as a Spring bean. The latest-telemetry SPI binding + * is not implemented here, so registering it now would expose a {@code + * database.ts_latest.type=iotdb-table} selector that ThingsBoard cannot bind to a working DAO. + * + *

Strategy F keeps this class free of ThingsBoard imports and interface clauses until the + * latest-telemetry path is implemented. + */ +@Slf4j +public class IoTDBTableLatestDao extends IoTDBTableBaseDao { + // Not annotated @Repository yet: auto-registering it now would advertise + // database.ts_latest.type=iotdb-table with no working DAO behind it. + public IoTDBTableLatestDao(ITableSessionPool tableSessionPool) { + super(tableSessionPool); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTablePendingSave.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTablePendingSave.java new file mode 100644 index 00000000..30a01bef --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTablePendingSave.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import com.google.common.util.concurrent.SettableFuture; +import org.thingsboard.server.common.data.kv.DataType; + +import java.util.Objects; + +record IoTDBTablePendingSave( + String tenantId, + String entityType, + String entityId, + String key, + long ts, + DataType dataType, + Object value, + int dataPointDays, + SettableFuture future) { + + IoTDBTablePendingSave( + String tenantId, + String entityType, + String entityId, + String key, + long ts, + DataType dataType, + Object value, + int dataPointDays) { + this( + tenantId, + entityType, + entityId, + key, + ts, + dataType, + value, + dataPointDays, + SettableFuture.create()); + } + + IoTDBTablePendingSave { + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityType, "entityType"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(key, "key"); + Objects.requireNonNull(dataType, "dataType"); + Objects.requireNonNull(value, "value"); + Objects.requireNonNull(future, "future"); + } + + IoTDBTableSaveIdentity identity() { + return new IoTDBTableSaveIdentity(tenantId, entityType, entityId, key, ts); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableRawOnlyEnabledCondition.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableRawOnlyEnabledCondition.java new file mode 100644 index 00000000..32af620e --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableRawOnlyEnabledCondition.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * Enables the raw-only backend only when the ThingsBoard historical-timeseries selector is set + * case-insensitively and the explicit experimental opt-in is true. + */ +final class IoTDBTableRawOnlyEnabledCondition implements Condition { + private static final String SELECTOR_PROPERTY = "database.ts.type"; + private static final String SELECTOR_VALUE = "iotdb-table"; + private static final String EXPERIMENTAL_PROPERTY = "iotdb.ts.experimental-raw-only"; + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + String selector = context.getEnvironment().getProperty(SELECTOR_PROPERTY); + boolean selected = selector != null && SELECTOR_VALUE.equalsIgnoreCase(selector.trim()); + boolean experimental = + context.getEnvironment().getProperty(EXPERIMENTAL_PROPERTY, Boolean.class, false); + return selected && experimental; + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableReadQueueFullException.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableReadQueueFullException.java new file mode 100644 index 00000000..3725537a --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableReadQueueFullException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import java.util.concurrent.RejectedExecutionException; + +/** + * Thrown when the IoTDB Table Mode timeseries read queue is full and a read task cannot be + * enqueued. + */ +public class IoTDBTableReadQueueFullException extends RejectedExecutionException { + public IoTDBTableReadQueueFullException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveIdentity.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveIdentity.java new file mode 100644 index 00000000..f30a6269 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveIdentity.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +record IoTDBTableSaveIdentity( + String tenantId, String entityType, String entityId, String key, long ts) {} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveQueueFullException.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveQueueFullException.java new file mode 100644 index 00000000..a256d496 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSaveQueueFullException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +public class IoTDBTableSaveQueueFullException extends RuntimeException { + public IoTDBTableSaveQueueFullException(String message) { + super(message); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrap.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrap.java new file mode 100644 index 00000000..48b669f6 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrap.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Idempotent startup bootstrap for the IoTDB Table Mode schema. + * + *

On a fresh IoTDB the {@code telemetry} / {@code entity_attributes} tables (and their database) + * do not exist, so the very first write would fail. This initializer runs once the session pool + * bean is up, reads {@code schema-iotdb-table.sql} from the classpath, and executes its statements. + * The {@code CREATE DATABASE} statement is already {@code IF NOT EXISTS}; the {@code CREATE TABLE} + * statements are not, so an "already exists" failure on re-run is tolerated rather than propagated, + * making the bootstrap idempotent. + * + *

Gated behind {@code iotdb.schema.bootstrap} (default {@code true}) so operators who manage the + * schema out-of-band can disable it; see the module README. + */ +@Slf4j +public class IoTDBTableSchemaBootstrap implements InitializingBean { + + static final String SCHEMA_RESOURCE = "schema-iotdb-table.sql"; + private static final String DEFAULT_SCHEMA_DATABASE = "thingsboard"; + + /** + * IoTDB identifier rule (letter or underscore first, then letters, digits or underscores). The + * database name is spliced verbatim into {@code CREATE DATABASE} / {@code USE} DDL, so it is + * validated here as defense-in-depth in addition to the {@code @Pattern} bean-validation + * constraint on {@link IoTDBTableConfig#getDatabase()} (the bootstrap can be constructed + * directly, bypassing bean validation). + */ + private static final Pattern DATABASE_NAME_PATTERN = Pattern.compile("^[A-Za-z_][A-Za-z0-9_]*$"); + + private final ITableSessionPool tableSessionPool; + private final IoTDBTableConfig config; + + public IoTDBTableSchemaBootstrap(ITableSessionPool tableSessionPool, IoTDBTableConfig config) { + this.tableSessionPool = Objects.requireNonNull(tableSessionPool, "tableSessionPool"); + this.config = Objects.requireNonNull(config, "config"); + } + + @Override + public void afterPropertiesSet() throws Exception { + String database = requireValidDatabaseName(config.getDatabase()); + String schema = loadSchema(database); + int created = 0; + int skipped = 0; + try (ITableSession session = tableSessionPool.getSession()) { + for (String statement : schema.split(";")) { + String trimmed = statement.trim(); + if (trimmed.isEmpty()) { + continue; + } + try { + session.executeNonQueryStatement(trimmed); + created++; + } catch (Exception e) { + if (isAlreadyExists(e)) { + skipped++; + log.debug("IoTDB Table Mode schema object already exists, skipping: {}", trimmed, e); + } else { + throw e; + } + } + } + } + log.info( + "IoTDB Table Mode schema bootstrap complete for database '{}': {} statement(s) applied, " + + "{} already-present statement(s) skipped", + database, + created, + skipped); + } + + private static String requireValidDatabaseName(String database) { + if (database == null || !DATABASE_NAME_PATTERN.matcher(database).matches()) { + throw new IllegalStateException( + "Invalid IoTDB database name for schema bootstrap: '" + + database + + "'. It must be a valid IoTDB identifier (a letter or underscore followed by letters," + + " digits or underscores) because it is spliced into CREATE DATABASE / USE DDL."); + } + return database; + } + + private String loadSchema(String database) throws IOException { + try (InputStream stream = + IoTDBTableSchemaBootstrap.class.getClassLoader().getResourceAsStream(SCHEMA_RESOURCE)) { + if (stream == null) { + throw new IllegalStateException( + "IoTDB Table Mode schema resource not found on classpath: " + SCHEMA_RESOURCE); + } + String schema = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + // Strip block and line comments so split-on-';' never executes a comment fragment. + schema = schema.replaceAll("(?s)/\\*.*?\\*/", "").replaceAll("(?m)--.*$", ""); + if (!DEFAULT_SCHEMA_DATABASE.equals(database)) { + schema = + schema + .replace( + "CREATE DATABASE IF NOT EXISTS " + DEFAULT_SCHEMA_DATABASE + ";", + "CREATE DATABASE IF NOT EXISTS " + database + ";") + .replace("USE " + DEFAULT_SCHEMA_DATABASE + ";", "USE " + database + ";"); + } + return schema; + } + } + + private static boolean isAlreadyExists(Throwable t) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + String message = cause.getMessage(); + if (message != null) { + String lower = message.toLowerCase(Locale.ROOT); + if (lower.contains("already exist") || lower.contains("has already been")) { + return true; + } + } + } + return false; + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java new file mode 100644 index 00000000..a42aa674 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Conditional; +import org.springframework.stereotype.Repository; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Historical telemetry DAO for the IoTDB Table Mode backend. + * + *

Spring activation: database.ts.type=iotdb-table and iotdb.ts.experimental-raw-only=true. + * + *

Strategy F consumes ThingsBoard common-data types from the compile classpath and binds the + * real historical {@link TimeseriesDao} SPI. + * + *

This initial implementation delivers the batch WRITE path ({@link #save}), the RAW + * (non-aggregated) historical READ path ({@link #findAllAsync}) and the DELETE path ({@link + * #remove}), all driven through a bounded read thread-pool. The time-bucketed aggregation read path + * is not implemented; a positive-interval aggregation query still throws {@link + * UnsupportedOperationException}. + */ +@Slf4j +@Repository +@ConditionalOnBean(name = IoTDBTableConfiguration.IOTDB_TABLE_SESSION_POOL_BEAN_NAME) +@Conditional(IoTDBTableRawOnlyEnabledCondition.class) +public class IoTDBTableTimeseriesDao extends IoTDBTableBaseDao + implements TimeseriesDao, DisposableBean { + private static final long SECONDS_PER_DAY = 86400L; + private static final String TABLE_NAME = IoTDBTableTimeseriesWriter.TABLE_NAME; + + private final IoTDBTableTimeseriesWriter timeseriesWriter; + private final ThreadPoolExecutor readExecutor; + private final Set> readTasks = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean accepting = new AtomicBoolean(true); + private final AtomicBoolean destroyed = new AtomicBoolean(false); + private final long defaultTtlSeconds; + private final long shutdownDrainTimeoutMs; + + public IoTDBTableTimeseriesDao( + @Qualifier(IoTDBTableConfiguration.IOTDB_TABLE_SESSION_POOL_BEAN_NAME) + ITableSessionPool tableSessionPool, + IoTDBTableTimeseriesWriter timeseriesWriter, + IoTDBTableConfig config) { + super(tableSessionPool); + this.timeseriesWriter = Objects.requireNonNull(timeseriesWriter, "timeseriesWriter"); + this.defaultTtlSeconds = + config.getDefaultTtlMs() > 0L + ? TimeUnit.MILLISECONDS.toSeconds(config.getDefaultTtlMs()) + : 0L; + this.shutdownDrainTimeoutMs = config.getTs().getSave().getShutdownDrainTimeoutMs(); + int readThreads = config.getTs().getRead().getThreads(); + int readQueueCapacity = config.getTs().getRead().getQueueCapacity(); + int flushThreads = config.getTs().getSave().getFlushThreads(); + if (readThreads + flushThreads > config.getSessionPoolSize()) { + log.warn( + "IoTDB Table Mode read/write workers ({}) exceed session pool size ({}); " + + "reads or flushes may wait for sessions", + readThreads + flushThreads, + config.getSessionPoolSize()); + } + this.readExecutor = + new ThreadPoolExecutor( + readThreads, + readThreads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(readQueueCapacity), + readThreadFactory(), + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public ListenableFuture> findAllAsync( + TenantId tenantId, EntityId entityId, List queries) { + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(queries, "queries"); + if (!accepting.get()) { + return Futures.immediateFailedFuture(shuttingDownException()); + } + + // Reject blank telemetry keys before any read task is enqueued, mirroring save()'s fail-fast + // contract, so an invalid query never occupies a read-pool slot. + try { + for (ReadTsKvQuery query : queries) { + Objects.requireNonNull(query, "query"); + requireTelemetryKey(query.getKey()); + } + } catch (RuntimeException e) { + return Futures.immediateFailedFuture(e); + } + + List> futures = new ArrayList<>(queries.size()); + for (ReadTsKvQuery query : queries) { + futures.add(submitReadTask(() -> readQuery(tenantId, entityId, query))); + } + return Futures.allAsList(futures); + } + + @Override + public ListenableFuture save( + TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(tsKvEntry, "tsKvEntry"); + + try { + String key = requireTelemetryKey(tsKvEntry.getKey()); + // Mirror the read/delete shutdown-race guard: once the DAO has stopped accepting work, fail + // fast instead of enqueueing into a writer that is (or is about to be) draining/destroyed. + if (!accepting.get()) { + return Futures.immediateFailedFuture(shuttingDownException()); + } + return timeseriesWriter.enqueue( + new IoTDBTablePendingSave( + tenantId.getId().toString(), + entityId.getEntityType().name(), + entityId.getId().toString(), + key, + tsKvEntry.getTs(), + tsKvEntry.getDataType(), + typedValue(tsKvEntry), + dataPointDays(tsKvEntry, ttl))); + } catch (RuntimeException e) { + return Futures.immediateFailedFuture(e); + } + } + + @Override + public ListenableFuture savePartition( + TenantId tenantId, EntityId entityId, long ts, String key) { + // IoTDB Table Mode has no per-partition bookkeeping; the write path is partition-agnostic, so + // there is nothing to persist for a partition marker. Matches the contract ThingsBoard expects + // from a DAO that does not maintain a partitions table. + return Futures.immediateFuture(0); + } + + @Override + public ListenableFuture remove( + TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(query, "query"); + + // Reject blank telemetry keys before enqueueing the delete task, mirroring save()'s fail-fast + // contract, so an invalid delete never occupies a read-pool slot. + try { + requireTelemetryKey(query.getKey()); + } catch (RuntimeException e) { + return Futures.immediateFailedFuture(e); + } + + return submitReadTask( + () -> { + String sql = buildDeleteSql(tenantId, entityId, query); + try (ITableSession session = tableSessionPool.getSession()) { + session.executeNonQueryStatement(sql); + } + return null; + }); + } + + @Override + public void cleanup(long systemTtl) { + // No-op: physical retention is a table-level IoTDB property (TTL in ms), owned by the + // operator's schema (WITH (TTL=) or ALTER TABLE telemetry SET PROPERTIES TTL=), not + // driven from this per-call hook. IoTDB Table Mode TTL cannot honor a per-data-point ttl, so + // the module does not issue retention DDL here. + } + + public IoTDBTableTimeseriesWriterStats stats() { + return timeseriesWriter.stats(); + } + + @Override + public void destroy() { + if (!destroyed.compareAndSet(false, true)) { + return; + } + accepting.set(false); + IoTDBTableDaoShuttingDownException failure = shuttingDownException(); + for (Runnable dropped : readExecutor.shutdownNow()) { + failDroppedReadTask(dropped, failure); + } + try { + readExecutor.awaitTermination(shutdownDrainTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + for (ReadTask task : readTasks) { + task.fail(failure); + } + timeseriesWriter.destroy(); + } + + private ReadTsKvQueryResult readQuery(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) + throws Exception { + // ThingsBoard 4.3.1.2 (AbstractChunkedAggregationTimeseriesDao.findAllAsync) routes a query to + // the + // RAW findAllWithLimit path when aggregation == NONE OR interval < 1 -- a sub-1 interval is a + // valid + // TB query shape that returns raw telemetry, not an error. Only a positive-interval aggregation + // enters the bucketed path. + if (aggregationOf(query) == Aggregation.NONE || query.getInterval() < 1L) { + return readRawQuery(tenantId, entityId, query); + } + // The positive-interval, time-bucketed aggregation read path is not implemented; only the RAW + // (Aggregation.NONE or interval < 1) branch is implemented now. + throw new UnsupportedOperationException( + "Time-bucketed aggregation is not supported by this incremental IoTDB Table Mode backend" + + " yet; raw read, write and delete are available."); + } + + private ReadTsKvQueryResult readRawQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + String key = requireTelemetryKey(query.getKey()); + String order = sqlOrder(query.getOrder()); + if (query.getLimit() <= 0) { + return new ReadTsKvQueryResult(query.getId(), List.of(), query.getStartTs()); + } + + String sql = buildReadSql(tenantId, entityId, query, order); + List entries = new ArrayList<>(); + long lastEntryTs = query.getStartTs(); + boolean hasEntry = false; + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + TypedKvValue value = getEntry(row); + if (!value.hasValue()) { + continue; + } + long ts = row.getTimestamp("time").getTime(); + entries.add(new BasicTsKvEntry(ts, kvEntry(key, value))); + if (!hasEntry || ts > lastEntryTs) { + lastEntryTs = ts; + hasEntry = true; + } + } + } + return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); + } + + private String buildReadSql( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query, String order) { + String key = requireTelemetryKey(query.getKey()); + return "SELECT time, bool_v, long_v, double_v, str_v, json_v FROM " + + TABLE_NAME + + " WHERE tenant_id=" + + sqlString(tenantId.getId().toString()) + + " AND entity_type=" + + sqlString(entityId.getEntityType().name()) + + " AND entity_id=" + + sqlString(entityId.getId().toString()) + + " AND key=" + + sqlString(key) + + " AND time >= " + + query.getStartTs() + + " AND time < " + + query.getEndTs() + + " ORDER BY time " + + order + + " LIMIT " + + query.getLimit(); + } + + private String buildDeleteSql(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + String key = requireTelemetryKey(query.getKey()); + return "DELETE FROM " + + TABLE_NAME + + " WHERE tenant_id=" + + sqlString(tenantId.getId().toString()) + + " AND entity_type=" + + sqlString(entityId.getEntityType().name()) + + " AND entity_id=" + + sqlString(entityId.getId().toString()) + + " AND key=" + + sqlString(key) + + " AND time >= " + + query.getStartTs() + + " AND time < " + + query.getEndTs(); + } + + private static Aggregation aggregationOf(ReadTsKvQuery query) { + var aggregationParams = query.getAggParameters(); + Aggregation aggregation = aggregationParams == null ? null : aggregationParams.getAggregation(); + return aggregation == null ? Aggregation.NONE : aggregation; + } + + private KvEntry kvEntry(String key, TypedKvValue value) { + if (value.booleanValue() != null) { + return new BooleanDataEntry(key, value.booleanValue()); + } + if (value.longValue() != null) { + return new LongDataEntry(key, value.longValue()); + } + if (value.doubleValue() != null) { + return new DoubleDataEntry(key, value.doubleValue()); + } + if (value.stringValue() != null) { + return new StringDataEntry(key, value.stringValue()); + } + if (value.jsonValue() != null) { + return new JsonDataEntry(key, value.jsonValue()); + } + throw new IllegalArgumentException("Telemetry row does not contain a typed value"); + } + + private static String sqlOrder(String order) { + String normalized = Objects.requireNonNull(order, "order").trim().toUpperCase(Locale.ROOT); + if (!"ASC".equals(normalized) && !"DESC".equals(normalized)) { + throw new IllegalArgumentException("Unsupported IoTDB Table Mode read order: " + order); + } + return normalized; + } + + private static String sqlString(String value) { + return "'" + Objects.requireNonNull(value, "value").replace("'", "''") + "'"; + } + + private ListenableFuture submitReadTask(Callable callable) { + if (!accepting.get()) { + return Futures.immediateFailedFuture(shuttingDownException()); + } + ReadTask task = new ReadTask<>(callable); + readTasks.add(task); + try { + readExecutor.execute(task); + } catch (RejectedExecutionException e) { + if (!accepting.get() || readExecutor.isShutdown()) { + task.fail(shuttingDownException()); + } else { + task.fail( + new IoTDBTableReadQueueFullException( + "IoTDB Table Mode timeseries read queue is full", e)); + } + readTasks.remove(task); + return task.future(); + } + if (!accepting.get() && readExecutor.remove(task)) { + task.fail(shuttingDownException()); + readTasks.remove(task); + } + return task.future(); + } + + private void failDroppedReadTask(Runnable dropped, IoTDBTableDaoShuttingDownException failure) { + if (dropped instanceof ReadTask task) { + task.fail(failure); + readTasks.remove(task); + } + } + + private IoTDBTableDaoShuttingDownException shuttingDownException() { + return new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries DAO is shutting down"); + } + + private static String requireTelemetryKey(String key) { + if (key == null || key.trim().isEmpty()) { + throw new IllegalArgumentException("Telemetry key must not be blank"); + } + return key; + } + + private static ThreadFactory readThreadFactory() { + AtomicInteger sequence = new AtomicInteger(); + return runnable -> { + Thread thread = + new Thread(runnable, "iotdb-table-timeseries-read-worker-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + private int dataPointDays(TsKvEntry tsKvEntry, long ttl) { + long effectiveTtlSeconds = + ttl <= 0L + ? defaultTtlSeconds + : (defaultTtlSeconds > 0L ? Math.min(defaultTtlSeconds, ttl) : ttl); + long ttlDays = Math.max(1L, effectiveTtlSeconds / SECONDS_PER_DAY); + return Math.toIntExact((long) tsKvEntry.getDataPoints() * ttlDays); + } + + private Object typedValue(TsKvEntry tsKvEntry) { + DataType dataType = tsKvEntry.getDataType(); + return switch (dataType) { + case BOOLEAN -> requiredValue(tsKvEntry.getBooleanValue(), dataType); + case LONG -> requiredValue(tsKvEntry.getLongValue(), dataType); + case DOUBLE -> requiredValue(tsKvEntry.getDoubleValue(), dataType); + case STRING -> requiredValue(tsKvEntry.getStrValue(), dataType); + case JSON -> requiredValue(tsKvEntry.getJsonValue(), dataType); + }; + } + + private Object requiredValue(Optional value, DataType dataType) { + return value.orElseThrow( + () -> new IllegalArgumentException("Missing value for telemetry data type " + dataType)); + } + + private final class ReadTask implements Runnable { + private final Callable callable; + private final SettableFuture future = SettableFuture.create(); + + private ReadTask(Callable callable) { + this.callable = Objects.requireNonNull(callable, "callable"); + } + + @Override + public void run() { + try { + future.set(callable.call()); + } catch (Throwable t) { + future.setException(t); + } finally { + readTasks.remove(this); + } + } + + private ListenableFuture future() { + return future; + } + + private void fail(Throwable t) { + future.setException(t); + } + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriter.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriter.java new file mode 100644 index 00000000..8a1412fb --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriter.java @@ -0,0 +1,551 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.springframework.beans.factory.DisposableBean; +import org.thingsboard.server.common.data.kv.DataType; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +@Slf4j +public class IoTDBTableTimeseriesWriter implements DisposableBean { + static final String TABLE_NAME = "telemetry"; + static final List COLUMN_NAMES = + List.of( + "entity_type", + "tenant_id", + "key", + "entity_id", + "bool_v", + "long_v", + "double_v", + "str_v", + "json_v"); + static final List DATA_TYPES = + List.of( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT64, + TSDataType.DOUBLE, + TSDataType.STRING, + TSDataType.TEXT); + static final List COLUMN_CATEGORIES = + List.of( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + // Rate-limited because overload or shutdown can reject many points on the write hot path. + private static final long REJECT_WARN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); + private static final long NEVER_WARNED_NANOS = Long.MIN_VALUE; + // After the graceful drain window expires, wait briefly for the interrupted worker to stop. + private static final long FORCE_STOP_JOIN_TIMEOUT_MS = 1000L; + + private final ITableSessionPool tableSessionPool; + private final BlockingQueue queue; + private final LongSupplier nanoTime; + private final int batchSize; + private final long maxLingerNanos; + private final long shutdownDrainTimeoutMs; + private final int retryMaxAttempts; + private final long retryInitialBackoffMs; + private final long retryMaxBackoffMs; + private final AtomicBoolean accepting = new AtomicBoolean(true); + private final AtomicBoolean destroyed = new AtomicBoolean(false); + private final ConcurrentMap, IoTDBTablePendingSave> acceptedSaves = + new ConcurrentHashMap<>(); + private final AtomicLong enqueued = new AtomicLong(); + private final AtomicLong flushed = new AtomicLong(); + private final AtomicLong flushFailures = new AtomicLong(); + private final AtomicLong retries = new AtomicLong(); + private final AtomicLong rejectsFull = new AtomicLong(); + private final AtomicLong rejectsShutdown = new AtomicLong(); + private final AtomicLong lastRejectWarnNanos = new AtomicLong(NEVER_WARNED_NANOS); + private final AtomicLong shutdownFailedPending = new AtomicLong(); + private final AtomicLong queueDepth = new AtomicLong(); + private volatile boolean forceStopped; + private final Thread worker; + + public IoTDBTableTimeseriesWriter(ITableSessionPool tableSessionPool, IoTDBTableConfig config) { + this(tableSessionPool, config, true); + } + + IoTDBTableTimeseriesWriter( + ITableSessionPool tableSessionPool, IoTDBTableConfig config, boolean startWorker) { + this( + tableSessionPool, + config, + startWorker, + new ArrayBlockingQueue<>(config.getTs().getSave().getQueueCapacity()), + System::nanoTime); + } + + IoTDBTableTimeseriesWriter( + ITableSessionPool tableSessionPool, + IoTDBTableConfig config, + boolean startWorker, + BlockingQueue queue) { + this(tableSessionPool, config, startWorker, queue, System::nanoTime); + } + + IoTDBTableTimeseriesWriter( + ITableSessionPool tableSessionPool, + IoTDBTableConfig config, + boolean startWorker, + BlockingQueue queue, + LongSupplier nanoTime) { + this.tableSessionPool = Objects.requireNonNull(tableSessionPool, "tableSessionPool"); + Objects.requireNonNull(config, "config"); + IoTDBTableConfig.Save saveConfig = config.getTs().getSave(); + this.batchSize = saveConfig.getBatchSize(); + this.maxLingerNanos = TimeUnit.MILLISECONDS.toNanos(saveConfig.getMaxLingerMs()); + this.shutdownDrainTimeoutMs = saveConfig.getShutdownDrainTimeoutMs(); + this.retryMaxAttempts = saveConfig.getRetryMaxAttempts(); + this.retryInitialBackoffMs = saveConfig.getRetryInitialBackoffMs(); + this.retryMaxBackoffMs = saveConfig.getRetryMaxBackoffMs(); + this.queue = Objects.requireNonNull(queue, "queue"); + this.nanoTime = Objects.requireNonNull(nanoTime, "nanoTime"); + this.worker = new Thread(this::runFlushLoop, "iotdb-table-timeseries-flush-worker"); + this.worker.setDaemon(true); + if (startWorker) { + this.worker.start(); + } + } + + public ListenableFuture enqueue(IoTDBTablePendingSave pending) { + Objects.requireNonNull(pending, "pending"); + if (!accepting.get()) { + rejectsShutdown.incrementAndGet(); + warnRejectedSave("shutting-down"); + pending.future().setException(shuttingDownException()); + return pending.future(); + } + if (!queue.offer(pending)) { + rejectsFull.incrementAndGet(); + warnRejectedSave("queue-full"); + pending + .future() + .setException( + new IoTDBTableSaveQueueFullException( + "IoTDB Table Mode timeseries save queue is full")); + return pending.future(); + } + registerAccepted(pending); + enqueued.incrementAndGet(); + queueDepth.incrementAndGet(); + if (!accepting.get() && queue.remove(pending)) { + queueDepth.decrementAndGet(); + rejectsShutdown.incrementAndGet(); + warnRejectedSave("shutting-down"); + pending.future().setException(shuttingDownException()); + unregisterAccepted(pending); + } else if (pending.future().isDone()) { + unregisterAccepted(pending); + } + return pending.future(); + } + + public IoTDBTableTimeseriesWriterStats stats() { + return new IoTDBTableTimeseriesWriterStats( + enqueued.get(), + flushed.get(), + flushFailures.get(), + retries.get(), + rejectsFull.get(), + rejectsShutdown.get(), + shutdownFailedPending.get(), + queueDepth.get()); + } + + Thread workerThread() { + return worker; + } + + /** + * Stops accepting saves and gives the flush worker a bounded drain window. + * + *

If that window expires, shutdown enters forced-stop mode: the worker is interrupted, local + * batches that were already dequeued are failed, and every accepted future is completed or failed + * before this method returns. If the timeout races an insert that has already started, the write + * may still commit after its future is failed. Shutdown therefore remains at least once: callers + * must tolerate duplicate or uncertain final-batch writes. + */ + @Override + public void destroy() { + if (!destroyed.compareAndSet(false, true)) { + return; + } + accepting.set(false); + // Do NOT interrupt the worker up front. An in-flight retry backoff (Thread.sleep in + // sleepBeforeRetry) would throw InterruptedException, flushBatch's catch would fail the + // already-accepted batch, and the drain window would be wasted on exactly the + // transient-error-during-shutdown path it exists to protect. With accepting=false the worker + // observes shutdown within one poll cycle (<=100ms), lets any in-flight retry finish, and + // drains + // the queue + current batch on its own. Only if it is STILL alive after the drain timeout + // (below) + // do we interrupt and fail whatever remains. + try { + worker.join(shutdownDrainTimeoutMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (worker.isAlive()) { + IoTDBTableDaoShuttingDownException failure = + new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries DAO shutdown drain timed out"); + forceStopped = true; + failUnfinishedPending(failure); + worker.interrupt(); + try { + worker.join(FORCE_STOP_JOIN_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + failUnfinishedPending(failure); + } else if (!queue.isEmpty()) { + failUnfinishedPending( + new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries DAO stopped before draining pending writes")); + } + } + + private void runFlushLoop() { + List batch = new ArrayList<>(batchSize); + while (!forceStopped && (accepting.get() || !queue.isEmpty() || !batch.isEmpty())) { + try { + if (batch.isEmpty()) { + IoTDBTablePendingSave first = queue.poll(100L, TimeUnit.MILLISECONDS); + if (first == null) { + continue; + } + addToBatch(batch, first); + } + fillBatchUntilReady(batch); + if (!batch.isEmpty()) { + flushBatch(batch); + batch.clear(); + } + } catch (InterruptedException e) { + if (accepting.get()) { + Thread.currentThread().interrupt(); + failBatch( + batch, + new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries flush worker was interrupted")); + batch.clear(); + break; + } + if (forceStopped) { + failBatch(batch, forcedStopException()); + batch.clear(); + break; + } + } catch (RuntimeException e) { + failBatch(batch, e); + batch.clear(); + } catch (Throwable t) { + failBatch(batch, t); + batch.clear(); + } + } + if (forceStopped && !batch.isEmpty()) { + failBatch(batch, forcedStopException()); + batch.clear(); + } + } + + // Upper bound on each linger-poll slice, so an in-flight linger wait observes a concurrent + // shutdown + // (accepting=false) within one slice instead of waiting out a large maxLingerMs -- WITHOUT + // interrupting the worker (an interrupt would abort an in-flight retry backoff; see destroy()). + private static final long SHUTDOWN_OBSERVE_SLICE_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + + private void fillBatchUntilReady(List batch) throws InterruptedException { + long deadlineNanos = System.nanoTime() + maxLingerNanos; + while (batch.size() < batchSize) { + // Once shutdown has begun, stop lingering and flush the partial batch immediately: any items + // still queued are drained by the outer runFlushLoop (its loop condition includes + // !queue.isEmpty()), so no accepted write is lost and the drain window is not spent waiting + // out maxLingerMs. The accepted-saves registry lets a forced stop fail the local batch even + // before flushBatch begins. + if (forceStopped || !accepting.get()) { + return; + } + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0L) { + return; + } + // Poll in bounded slices so a shutdown that races an in-flight linger wait is observed within + // one slice. With the default small maxLingerMs the slice covers the whole wait (single poll, + // unchanged behaviour); only a large maxLingerMs is sliced. + long sliceNanos = Math.min(remainingNanos, SHUTDOWN_OBSERVE_SLICE_NANOS); + IoTDBTablePendingSave next = queue.poll(sliceNanos, TimeUnit.NANOSECONDS); + if (next == null) { + // Slice elapsed with no item: re-check accepting + the maxLinger deadline at the loop top. + continue; + } + addToBatch(batch, next); + drainAvailable(batch); + } + } + + private void drainAvailable(List batch) { + int limit = batchSize - batch.size(); + if (limit <= 0) { + return; + } + List drained = new ArrayList<>(limit); + int drainedCount = queue.drainTo(drained, limit); + if (drainedCount == 0) { + return; + } + queueDepth.addAndGet(-drainedCount); + batch.addAll(drained); + } + + private void addToBatch(List batch, IoTDBTablePendingSave pending) { + queueDepth.decrementAndGet(); + batch.add(pending); + } + + private void flushBatch(List rawBatch) { + if (forceStopped) { + failBatch(rawBatch, forcedStopException()); + return; + } + List insertBatch = deduplicateForInsert(rawBatch); + if (insertBatch.isEmpty()) { + return; + } + try { + Tablet tablet = buildTablet(insertBatch); + insertWithRetry(tablet); + flushed.addAndGet(insertBatch.size()); + completeBatch(rawBatch); + } catch (InterruptedException e) { + flushFailures.incrementAndGet(); + if (forceStopped) { + failBatch(rawBatch, forcedStopException()); + } else { + Thread.currentThread().interrupt(); + failBatch(rawBatch, e); + } + } catch (Throwable t) { + flushFailures.incrementAndGet(); + failBatch(rawBatch, t); + } + } + + private List deduplicateForInsert(List rawBatch) { + // Collapses duplicate (tenant, entity, key, time) saves within a single flush so the tablet + // honors the design's same-(tags, time) overwrite contract: the last write wins. Cross-flush + // same-time type changes are out of scope for this iteration -- a delete-then-insert defense is + // outside the current scope (see the module README "Known limitations"). The read path fails + // fast on a row that ends up with two typed columns, so this relaxation never silently returns + // wrong data. + Map lastByIdentity = + new LinkedHashMap<>(rawBatch.size()); + for (IoTDBTablePendingSave pending : rawBatch) { + IoTDBTableSaveIdentity identity = pending.identity(); + lastByIdentity.remove(identity); + lastByIdentity.put(identity, pending); + } + return new ArrayList<>(lastByIdentity.values()); + } + + Tablet buildTablet(List batch) { + Tablet tablet = + new Tablet(TABLE_NAME, COLUMN_NAMES, DATA_TYPES, COLUMN_CATEGORIES, batch.size()); + for (int row = 0; row < batch.size(); row++) { + IoTDBTablePendingSave pending = batch.get(row); + tablet.addTimestamp(row, pending.ts()); + tablet.addValue("entity_type", row, pending.entityType()); + tablet.addValue("tenant_id", row, pending.tenantId()); + tablet.addValue("key", row, pending.key()); + tablet.addValue("entity_id", row, pending.entityId()); + tablet.addValue( + "bool_v", row, pending.dataType() == DataType.BOOLEAN ? pending.value() : null); + tablet.addValue("long_v", row, pending.dataType() == DataType.LONG ? pending.value() : null); + tablet.addValue( + "double_v", row, pending.dataType() == DataType.DOUBLE ? pending.value() : null); + tablet.addValue("str_v", row, pending.dataType() == DataType.STRING ? pending.value() : null); + tablet.addValue("json_v", row, pending.dataType() == DataType.JSON ? pending.value() : null); + } + tablet.setRowSize(batch.size()); + return tablet; + } + + private void insertWithRetry(Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + long backoffMs = initialBackoffMs(retryInitialBackoffMs, retryMaxBackoffMs); + for (int attempt = 1; attempt <= retryMaxAttempts; attempt++) { + if (forceStopped) { + throw forcedStopException(); + } + boolean inserted = false; + try (ITableSession session = tableSessionPool.getSession()) { + if (forceStopped) { + throw forcedStopException(); + } + session.insert(tablet); + inserted = true; + return; + } catch (IoTDBConnectionException e) { + if (inserted) { + // close() failed after insert returned; do not replay a tablet that may be persisted. + return; + } + if (attempt >= retryMaxAttempts) { + throw e; + } + retries.incrementAndGet(); + sleepBeforeRetry(backoffMs); + backoffMs = nextBackoffMs(backoffMs); + } + } + throw new IllegalStateException("IoTDB insert retry loop exited without success or failure"); + } + + static long initialBackoffMs(long initialBackoffMs, long maxBackoffMs) { + if (maxBackoffMs <= 0L) { + return initialBackoffMs; + } + return Math.min(initialBackoffMs, maxBackoffMs); + } + + private IoTDBTableDaoShuttingDownException shuttingDownException() { + return new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries DAO is shutting down"); + } + + private IoTDBTableDaoShuttingDownException forcedStopException() { + return new IoTDBTableDaoShuttingDownException( + "IoTDB Table Mode timeseries DAO shutdown drain timed out"); + } + + private void warnRejectedSave(String reason) { + if (!shouldLogRejectedSaveWarning()) { + return; + } + log.warn( + "IoTDB Table Mode timeseries save rejected (reason={}, rejectsFull={}, rejectsShutdown={})", + reason, + rejectsFull.get(), + rejectsShutdown.get()); + } + + boolean shouldLogRejectedSaveWarning() { + long nowNanos = nanoTime.getAsLong(); + long previousNanos = lastRejectWarnNanos.get(); + if (previousNanos != NEVER_WARNED_NANOS + && nowNanos - previousNanos < REJECT_WARN_INTERVAL_NANOS) { + return false; + } + return lastRejectWarnNanos.compareAndSet(previousNanos, nowNanos); + } + + private void sleepBeforeRetry(long backoffMs) throws InterruptedException { + if (backoffMs > 0L) { + Thread.sleep(backoffMs); + } + } + + private long nextBackoffMs(long currentBackoffMs) { + if (retryMaxBackoffMs <= 0L) { + return 0L; + } + if (currentBackoffMs <= 0L) { + return Math.min(1L, retryMaxBackoffMs); + } + long doubled = currentBackoffMs > Long.MAX_VALUE / 2L ? Long.MAX_VALUE : currentBackoffMs * 2L; + return Math.min(retryMaxBackoffMs, doubled); + } + + private void completeBatch(List batch) { + for (IoTDBTablePendingSave pending : batch) { + pending.future().set(pending.dataPointDays()); + unregisterAccepted(pending); + } + } + + private void failBatch(List batch, Throwable t) { + for (IoTDBTablePendingSave pending : batch) { + pending.future().setException(t); + unregisterAccepted(pending); + } + } + + private void registerAccepted(IoTDBTablePendingSave pending) { + acceptedSaves.put(pending.future(), pending); + } + + private void unregisterAccepted(IoTDBTablePendingSave pending) { + acceptedSaves.remove(pending.future(), pending); + } + + private void failUnfinishedPending(RuntimeException failure) { + List pending = new ArrayList<>(); + queue.drainTo(pending); + queueDepth.addAndGet(-pending.size()); + for (IoTDBTablePendingSave save : pending) { + acceptedSaves.putIfAbsent(save.future(), save); + } + long failed = 0L; + for (Map.Entry, IoTDBTablePendingSave> entry : + acceptedSaves.entrySet()) { + if (entry.getKey().setException(failure)) { + failed++; + } + acceptedSaves.remove(entry.getKey(), entry.getValue()); + } + shutdownFailedPending.addAndGet(failed); + } +} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriterStats.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriterStats.java new file mode 100644 index 00000000..b95acf0d --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesWriterStats.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +public record IoTDBTableTimeseriesWriterStats( + long enqueued, + long flushed, + long flushFailures, + long retries, + long rejectsFull, + long rejectsShutdown, + long shutdownFailedPending, + long queueDepth) {} diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/TypedKvValue.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/TypedKvValue.java new file mode 100644 index 00000000..284efd0e --- /dev/null +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/TypedKvValue.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import java.util.Objects; + +/** + * A single typed value mapped from one IoTDB Table Mode telemetry row's 5 typed FIELD columns + * (bool_v, long_v, double_v, str_v, json_v). Exactly one field is non-null per valid row. + */ +public record TypedKvValue( + Boolean booleanValue, + Long longValue, + Double doubleValue, + String stringValue, + String jsonValue) { + + public static TypedKvValue ofBoolean(boolean v) { + return new TypedKvValue(v, null, null, null, null); + } + + public static TypedKvValue ofLong(long v) { + return new TypedKvValue(null, v, null, null, null); + } + + public static TypedKvValue ofDouble(double v) { + return new TypedKvValue(null, null, v, null, null); + } + + public static TypedKvValue ofString(String v) { + return new TypedKvValue(null, null, null, Objects.requireNonNull(v, "stringValue"), null); + } + + public static TypedKvValue ofJson(String v) { + return new TypedKvValue(null, null, null, null, Objects.requireNonNull(v, "jsonValue")); + } + + public static TypedKvValue empty() { + return new TypedKvValue(null, null, null, null, null); + } + + public boolean hasValue() { + return valueCount() > 0; + } + + public Object value() { + int count = valueCount(); + if (count == 0) { + return null; + } + if (count > 1) { + throw new IllegalStateException("TypedKvValue contains multiple values"); + } + if (booleanValue != null) { + return booleanValue; + } + if (longValue != null) { + return longValue; + } + if (doubleValue != null) { + return doubleValue; + } + if (stringValue != null) { + return stringValue; + } + return jsonValue; + } + + private int valueCount() { + int count = 0; + count += booleanValue == null ? 0 : 1; + count += longValue == null ? 0 : 1; + count += doubleValue == null ? 0 : 1; + count += stringValue == null ? 0 : 1; + count += jsonValue == null ? 0 : 1; + return count; + } +} diff --git a/iotdb-thingsboard-table/src/main/resources/META-INF/spring.factories b/iotdb-thingsboard-table/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..8c040dd7 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/resources/META-INF/spring.factories @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.apache.iotdb.extras.thingsboard.table.IoTDBTableConfiguration diff --git a/iotdb-thingsboard-table/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/iotdb-thingsboard-table/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..aa6dd67f --- /dev/null +++ b/iotdb-thingsboard-table/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.iotdb.extras.thingsboard.table.IoTDBTableConfiguration diff --git a/iotdb-thingsboard-table/src/main/resources/schema-iotdb-table.sql b/iotdb-thingsboard-table/src/main/resources/schema-iotdb-table.sql new file mode 100644 index 00000000..b5257cf5 --- /dev/null +++ b/iotdb-thingsboard-table/src/main/resources/schema-iotdb-table.sql @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE DATABASE IF NOT EXISTS thingsboard; +USE thingsboard; + +CREATE TABLE telemetry ( + entity_type STRING TAG, -- DEVICE, ASSET, etc. + tenant_id STRING TAG, -- multi-tenant isolation + key STRING TAG, -- telemetry key name + entity_id STRING TAG, -- ThingsBoard entity UUID + bool_v BOOLEAN FIELD, + long_v INT64 FIELD, -- exactly one non-null per row, + double_v DOUBLE FIELD, -- mirroring AbstractTsKvEntity + str_v STRING FIELD, + json_v TEXT FIELD +) WITH (TTL=DEFAULT); + +CREATE TABLE entity_attributes ( + time TIMESTAMP TIME, + attribute_scope STRING TAG, -- CLIENT_SCOPE | SERVER_SCOPE | SHARED_SCOPE + entity_type STRING TAG, + tenant_id STRING TAG, + key STRING TAG, + entity_id STRING TAG, + bool_v BOOLEAN FIELD, long_v INT64 FIELD, double_v DOUBLE FIELD, + str_v STRING FIELD, json_v TEXT FIELD +) WITH (TTL='INF'); diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/EntityType.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/EntityType.java new file mode 100644 index 00000000..c197cabc --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/EntityType.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data; + +public enum EntityType { + DEVICE, + DEVICE_PROFILE, + ASSET, + TENANT, + CUSTOMER, + USER, + DASHBOARD, + RULE_CHAIN, + RULE_NODE, + EDGE, + ALARM, + OTA_PACKAGE, + QUEUE, + NOTIFICATION_REQUEST, + NOTIFICATION, + NOTIFICATION_RULE, + WIDGETS_BUNDLE, + WIDGET_TYPE, + API_USAGE_STATE, + TB_RESOURCE, + DOMAIN, + MOBILE_APP, + MOBILE_APP_BUNDLE +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/HasVersion.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/HasVersion.java new file mode 100644 index 00000000..8bc8e7f8 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/HasVersion.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data; + +public interface HasVersion { + Long getVersion(); + + default void setVersion(Long version) {} +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/EntityId.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/EntityId.java new file mode 100644 index 00000000..cb5e60dc --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/EntityId.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.id; + +import org.thingsboard.server.common.data.EntityType; + +import java.io.Serializable; +import java.util.UUID; + +public interface EntityId extends HasUUID, Serializable { + UUID NULL_UUID = UUID.fromString("13814000-1dd2-11b2-8080-808080808080"); + + @Override + UUID getId(); + + EntityType getEntityType(); + + default boolean isNullUid() { + return NULL_UUID.equals(getId()); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/HasUUID.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/HasUUID.java new file mode 100644 index 00000000..b7435c81 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/HasUUID.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.id; + +import java.util.UUID; + +public interface HasUUID { + UUID getId(); +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/TenantId.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/TenantId.java new file mode 100644 index 00000000..0663bbe5 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/TenantId.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.id; + +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public final class TenantId extends UUIDBased implements EntityId { + public static final TenantId SYS_TENANT_ID = TenantId.fromUUID(EntityId.NULL_UUID); + + public static TenantId fromUUID(UUID id) { + return new TenantId(id); + } + + public TenantId(UUID id) { + super(id); + } + + public boolean isSysTenantId() { + return this.equals(SYS_TENANT_ID); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/UUIDBased.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/UUIDBased.java new file mode 100644 index 00000000..32194cde --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/id/UUIDBased.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.id; + +import java.io.Serializable; +import java.util.UUID; + +public abstract class UUIDBased implements HasUUID, Serializable { + private final UUID id; + private transient int hash; + + protected UUIDBased() { + this(UUID.randomUUID()); + } + + public UUIDBased(UUID id) { + this.id = id; + } + + @Override + public UUID getId() { + return id; + } + + @Override + public int hashCode() { + if (hash == 0) { + hash = 31 + (id == null ? 0 : id.hashCode()); + } + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + UUIDBased other = (UUIDBased) obj; + if (id == null) { + return other.id == null; + } + return id.equals(other.id); + } + + @Override + public String toString() { + return String.valueOf(id); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/Aggregation.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/Aggregation.java new file mode 100644 index 00000000..e0627f66 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/Aggregation.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public enum Aggregation { + MIN, + MAX, + AVG, + SUM, + COUNT, + NONE +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/AggregationParams.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/AggregationParams.java new file mode 100644 index 00000000..93157e55 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/AggregationParams.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class AggregationParams { + private static final Map TZ_LINKS = + Map.of( + "EST", "America/New_York", + "GMT+0", "GMT", + "GMT-0", "GMT", + "HST", "US/Hawaii", + "MST", "America/Phoenix", + "ROC", "Asia/Taipei"); + + private final Aggregation aggregation; + private final IntervalType intervalType; + private final ZoneId tzId; + private final long interval; + + public AggregationParams( + Aggregation aggregation, IntervalType intervalType, ZoneId tzId, long interval) { + this.aggregation = Objects.requireNonNull(aggregation, "aggregation"); + this.intervalType = intervalType; + this.tzId = tzId; + this.interval = interval; + } + + public static AggregationParams none() { + return new AggregationParams(Aggregation.NONE, null, null, 0L); + } + + public static AggregationParams milliseconds( + Aggregation aggregationType, long aggregationIntervalMs) { + return new AggregationParams( + aggregationType, IntervalType.MILLISECONDS, null, aggregationIntervalMs); + } + + public static AggregationParams calendar( + Aggregation aggregationType, IntervalType intervalType, String tzIdStr) { + return calendar(aggregationType, intervalType, getZoneId(tzIdStr)); + } + + public static AggregationParams calendar( + Aggregation aggregationType, IntervalType intervalType, ZoneId tzId) { + return new AggregationParams(aggregationType, intervalType, tzId, 0L); + } + + public static AggregationParams of( + Aggregation aggregation, IntervalType intervalType, ZoneId tzId, long interval) { + return new AggregationParams(aggregation, intervalType, tzId, interval); + } + + public Aggregation getAggregation() { + return aggregation; + } + + public IntervalType getIntervalType() { + return intervalType; + } + + public ZoneId getTzId() { + return tzId; + } + + public long getInterval() { + // Matches real ThingsBoard v4.3.1.2 AggregationParams.getInterval(): a null IntervalType + // returns 0L. Real TB never pairs a null IntervalType with a non-NONE aggregation, so the + // DAO's MILLISECONDS-path positive-interval guard is never legitimately reached for a null + // type; the real MILLISECONDS path always carries IntervalType.MILLISECONDS with a positive + // interval (see AggregationParams.milliseconds). + if (intervalType == null) { + return 0L; + } + return switch (intervalType) { + case WEEK, WEEK_ISO -> TimeUnit.DAYS.toMillis(7); + case MONTH -> TimeUnit.DAYS.toMillis(30); + case QUARTER -> TimeUnit.DAYS.toMillis(90); + default -> interval; + }; + } + + private static ZoneId getZoneId(String tzIdStr) { + if (tzIdStr == null || tzIdStr.isEmpty()) { + return ZoneId.systemDefault(); + } + try { + return ZoneId.of(tzIdStr, TZ_LINKS); + } catch (DateTimeException e) { + return ZoneId.systemDefault(); + } + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseDeleteTsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseDeleteTsKvQuery.java new file mode 100644 index 00000000..df09672d --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseDeleteTsKvQuery.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public class BaseDeleteTsKvQuery extends BaseTsKvQuery implements DeleteTsKvQuery { + private final Boolean rewriteLatestIfDeleted; + private final Boolean deleteLatest; + + public BaseDeleteTsKvQuery( + String key, long startTs, long endTs, boolean rewriteLatestIfDeleted, boolean deleteLatest) { + super(key, startTs, endTs); + this.rewriteLatestIfDeleted = rewriteLatestIfDeleted; + this.deleteLatest = deleteLatest; + } + + public BaseDeleteTsKvQuery( + String key, long startTs, long endTs, boolean rewriteLatestIfDeleted) { + this(key, startTs, endTs, rewriteLatestIfDeleted, true); + } + + public BaseDeleteTsKvQuery(String key, long startTs, long endTs) { + this(key, startTs, endTs, false, true); + } + + @Override + public Boolean getRewriteLatestIfDeleted() { + return rewriteLatestIfDeleted; + } + + @Override + public Boolean getDeleteLatest() { + return deleteLatest; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java new file mode 100644 index 00000000..25f8feb1 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.time.ZoneId; + +public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery { + private final AggregationParams aggParameters; + private final int limit; + private final String order; + + public BaseReadTsKvQuery( + String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation) { + this(key, startTs, endTs, interval, limit, aggregation, "DESC"); + } + + public BaseReadTsKvQuery( + String key, + long startTs, + long endTs, + long interval, + int limit, + Aggregation aggregation, + String descOrder) { + this( + key, + startTs, + endTs, + AggregationParams.of( + aggregation, IntervalType.MILLISECONDS, ZoneId.systemDefault(), interval), + limit, + descOrder); + } + + public BaseReadTsKvQuery( + String key, long startTs, long endTs, AggregationParams parameters, int limit) { + this(key, startTs, endTs, parameters, limit, "DESC"); + } + + public BaseReadTsKvQuery( + String key, + long startTs, + long endTs, + AggregationParams parameters, + int limit, + String order) { + super(key, startTs, endTs); + this.aggParameters = parameters; + this.limit = limit; + this.order = order; + } + + public BaseReadTsKvQuery(String key, long startTs, long endTs) { + this(key, startTs, endTs, AggregationParams.milliseconds(Aggregation.AVG, endTs - startTs), 1); + } + + public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String order) { + this(key, startTs, endTs, AggregationParams.none(), limit, order); + } + + public BaseReadTsKvQuery(ReadTsKvQuery query, long startTs, long endTs) { + super(query.getId(), query.getKey(), startTs, endTs); + this.aggParameters = query.getAggParameters(); + this.limit = query.getLimit(); + this.order = query.getOrder(); + } + + @Override + public AggregationParams getAggParameters() { + return aggParameters; + } + + @Override + public int getLimit() { + return limit; + } + + @Override + public String getOrder() { + return order; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java new file mode 100644 index 00000000..c5b7d304 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public class BaseTsKvQuery implements TsKvQuery { + private static final ThreadLocal ID_SEQ = ThreadLocal.withInitial(() -> 0); + + private final int id; + private final String key; + private final long startTs; + private final long endTs; + + public BaseTsKvQuery(String key, long startTs, long endTs) { + this(ID_SEQ.get(), key, startTs, endTs); + ID_SEQ.set(id + 1); + } + + protected BaseTsKvQuery(int id, String key, long startTs, long endTs) { + this.id = id; + this.key = key; + this.startTs = startTs; + this.endTs = endTs; + } + + @Override + public int getId() { + return id; + } + + @Override + public String getKey() { + return key; + } + + @Override + public long getStartTs() { + return startTs; + } + + @Override + public long getEndTs() { + return endTs; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java new file mode 100644 index 00000000..7d19bc68 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public abstract class BasicKvEntry implements KvEntry { + private final String key; + + protected BasicKvEntry(String key) { + this.key = key; + } + + @Override + public String getKey() { + return key; + } + + @Override + public Optional getStrValue() { + return Optional.empty(); + } + + @Override + public Optional getLongValue() { + return Optional.empty(); + } + + @Override + public Optional getBooleanValue() { + return Optional.empty(); + } + + @Override + public Optional getDoubleValue() { + return Optional.empty(); + } + + @Override + public Optional getJsonValue() { + return Optional.empty(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof BasicKvEntry other)) { + return false; + } + return Objects.equals(key, other.key); + } + + @Override + public int hashCode() { + return Objects.hash(key); + } + + @Override + public String toString() { + return "BasicKvEntry{" + "key='" + key + '\'' + '}'; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java new file mode 100644 index 00000000..4bcf85e2 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Optional; + +public class BasicTsKvEntry implements TsKvEntry { + private static final int MAX_CHARS_PER_DATA_POINT = 512; + + protected final long ts; + private final KvEntry kv; + private final Long version; + + public BasicTsKvEntry(long ts, KvEntry kv) { + this.ts = ts; + this.kv = kv; + this.version = null; + } + + public BasicTsKvEntry(long ts, KvEntry kv, Long version) { + this.ts = ts; + this.kv = kv; + this.version = version; + } + + @Override + public long getTs() { + return ts; + } + + public KvEntry getKv() { + return kv; + } + + @Override + public Long getVersion() { + return version; + } + + @Override + public String getKey() { + return kv.getKey(); + } + + @Override + public DataType getDataType() { + return kv.getDataType(); + } + + @Override + public Optional getStrValue() { + return kv.getStrValue(); + } + + @Override + public Optional getLongValue() { + return kv.getLongValue(); + } + + @Override + public Optional getBooleanValue() { + return kv.getBooleanValue(); + } + + @Override + public Optional getDoubleValue() { + return kv.getDoubleValue(); + } + + @Override + public Optional getJsonValue() { + return kv.getJsonValue(); + } + + @Override + public Object getValue() { + return kv.getValue(); + } + + @Override + public String getValueAsString() { + return kv.getValueAsString(); + } + + @Override + public int getDataPoints() { + int length; + switch (getDataType()) { + case STRING: + length = getStrValue().orElse("").length(); + break; + case JSON: + length = getJsonValue().orElse("").length(); + break; + default: + return 1; + } + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java new file mode 100644 index 00000000..fc710deb --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public class BooleanDataEntry extends BasicKvEntry { + private final Boolean value; + + public BooleanDataEntry(String key, Boolean value) { + super(key); + this.value = value; + } + + @Override + public DataType getDataType() { + return DataType.BOOLEAN; + } + + @Override + public Optional getBooleanValue() { + return Optional.ofNullable(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String getValueAsString() { + return Boolean.toString(value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof BooleanDataEntry other) || !super.equals(obj)) { + return false; + } + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DataType.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DataType.java new file mode 100644 index 00000000..975248fc --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DataType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public enum DataType { + BOOLEAN, + LONG, + DOUBLE, + STRING, + JSON +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DeleteTsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DeleteTsKvQuery.java new file mode 100644 index 00000000..bbb2d269 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DeleteTsKvQuery.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public interface DeleteTsKvQuery extends TsKvQuery { + Boolean getRewriteLatestIfDeleted(); + + Boolean getDeleteLatest(); +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java new file mode 100644 index 00000000..2b36a316 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public class DoubleDataEntry extends BasicKvEntry { + private final Double value; + + public DoubleDataEntry(String key, Double value) { + super(key); + this.value = value; + } + + @Override + public DataType getDataType() { + return DataType.DOUBLE; + } + + @Override + public Optional getDoubleValue() { + return Optional.ofNullable(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String getValueAsString() { + return Double.toString(value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DoubleDataEntry other) || !super.equals(obj)) { + return false; + } + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/IntervalType.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/IntervalType.java new file mode 100644 index 00000000..67397537 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/IntervalType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public enum IntervalType { + MILLISECONDS, + WEEK, + WEEK_ISO, + MONTH, + QUARTER +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java new file mode 100644 index 00000000..044ae9c8 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public class JsonDataEntry extends BasicKvEntry { + private final String value; + + public JsonDataEntry(String key, String value) { + super(key); + this.value = value; + } + + @Override + public DataType getDataType() { + return DataType.JSON; + } + + @Override + public Optional getJsonValue() { + return Optional.ofNullable(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String getValueAsString() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof JsonDataEntry other) || !super.equals(obj)) { + return false; + } + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/KvEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/KvEntry.java new file mode 100644 index 00000000..7dc4c5c3 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/KvEntry.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.io.Serializable; +import java.util.Optional; + +public interface KvEntry extends Serializable { + String getKey(); + + DataType getDataType(); + + Optional getBooleanValue(); + + Optional getLongValue(); + + Optional getDoubleValue(); + + Optional getStrValue(); + + Optional getJsonValue(); + + String getValueAsString(); + + Object getValue(); +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/LongDataEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/LongDataEntry.java new file mode 100644 index 00000000..84beebf0 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/LongDataEntry.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public class LongDataEntry extends BasicKvEntry { + private final Long value; + + public LongDataEntry(String key, Long value) { + super(key); + this.value = value; + } + + @Override + public DataType getDataType() { + return DataType.LONG; + } + + @Override + public Optional getLongValue() { + return Optional.ofNullable(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String getValueAsString() { + return Long.toString(value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LongDataEntry other) || !super.equals(obj)) { + return false; + } + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java new file mode 100644 index 00000000..010cd46c --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public interface ReadTsKvQuery extends TsKvQuery { + AggregationParams getAggParameters(); + + default long getInterval() { + return getAggParameters().getInterval(); + } + + default Aggregation getAggregation() { + return getAggParameters().getAggregation(); + } + + int getLimit(); + + String getOrder(); +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java new file mode 100644 index 00000000..f20b0fda --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.ArrayList; +import java.util.List; + +public class ReadTsKvQueryResult { + private final int queryId; + private final List data; + private final long lastEntryTs; + + public ReadTsKvQueryResult(int queryId, List data, long lastEntryTs) { + this.queryId = queryId; + this.data = data; + this.lastEntryTs = lastEntryTs; + } + + public int getQueryId() { + return queryId; + } + + public List getData() { + return data; + } + + public long getLastEntryTs() { + return lastEntryTs; + } + + public TsValue[] toTsValues() { + if (data != null && !data.isEmpty()) { + List queryValues = new ArrayList<>(); + for (TsKvEntry entry : data) { + queryValues.add(entry.toTsValue()); + } + return queryValues.toArray(new TsValue[0]); + } + return new TsValue[0]; + } + + public TsValue toTsValue(ReadTsKvQuery query) { + if (data == null || data.isEmpty()) { + if (Aggregation.SUM.equals(query.getAggregation()) + || Aggregation.COUNT.equals(query.getAggregation())) { + long ts = query.getStartTs() + (query.getEndTs() - query.getStartTs()) / 2; + return new TsValue(ts, "0"); + } + return TsValue.EMPTY; + } + if (data.size() > 1) { + throw new RuntimeException("Query Result has multiple data points!"); + } + return data.get(0).toTsValue(); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/StringDataEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/StringDataEntry.java new file mode 100644 index 00000000..6f5d2132 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/StringDataEntry.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import java.util.Objects; +import java.util.Optional; + +public class StringDataEntry extends BasicKvEntry { + private final String value; + + public StringDataEntry(String key, String value) { + super(key); + this.value = value; + } + + @Override + public DataType getDataType() { + return DataType.STRING; + } + + @Override + public Optional getStrValue() { + return Optional.ofNullable(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String getValueAsString() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StringDataEntry other) || !super.equals(obj)) { + return false; + } + return Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), value); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvEntry.java new file mode 100644 index 00000000..f76c4fdc --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.query.TsValue; + +public interface TsKvEntry extends KvEntry, HasVersion { + long getTs(); + + int getDataPoints(); + + default TsValue toTsValue() { + return new TsValue(getTs(), getValueAsString()); + } + + default boolean isDeletedEntry() { + return getTs() == 0 && (getValue() == null || getValueAsString().isEmpty()); + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvQuery.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvQuery.java new file mode 100644 index 00000000..124778f5 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/kv/TsKvQuery.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.kv; + +public interface TsKvQuery { + int getId(); + + String getKey(); + + long getStartTs(); + + long getEndTs(); +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/query/TsValue.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/query/TsValue.java new file mode 100644 index 00000000..c18daf44 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/common/data/query/TsValue.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.common.data.query; + +public class TsValue { + public static final TsValue EMPTY = new TsValue(0, ""); + + private final long ts; + private final String value; + private final Long count; + + public TsValue(long ts, String value) { + this(ts, value, null); + } + + public TsValue(long ts, String value, Long count) { + this.ts = ts; + this.value = value; + this.count = count; + } + + public long getTs() { + return ts; + } + + public String getValue() { + return value; + } + + public Long getCount() { + return count; + } +} diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java new file mode 100644 index 00000000..81494b45 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (commit c37fb509) on 2026-06-12. +package org.thingsboard.server.dao.timeseries; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface TimeseriesDao { + ListenableFuture> findAllAsync( + TenantId tenantId, EntityId entityId, List queries); + + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); + + ListenableFuture savePartition( + TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key); + + ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); + + void cleanup(long systemTtl); +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAutoConfigurationTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAutoConfigurationTest.java new file mode 100644 index 00000000..22384cc0 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableAutoConfigurationTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Auto-configuration tests that drive the module through Spring Boot's real auto-configuration + * discovery path ({@link AutoConfigurations#of}, the mechanism a real ThingsBoard Spring Boot app + * uses via {@code META-INF/spring/...AutoConfiguration.imports}), rather than a plain + * {@code @Import}. This proves the META-INF auto-config registration actually activates the module + * the way a deployed application would, and that explicitly selecting the backend fails fast when + * the host already provides a conflicting {@link TimeseriesDao}. + */ +class IoTDBTableAutoConfigurationTest { + private static final String SESSION_POOL_BEAN_NAME = + IoTDBTableConfiguration.IOTDB_TABLE_SESSION_POOL_BEAN_NAME; + + private final ApplicationContextRunner contextRunner = + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(IoTDBTableConfiguration.class)); + + @Test + void autoConfigDiscovery_withSelector_createsPoolAndDao() { + contextRunner + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + // Offline test with no real IoTDB: skip the startup bootstrap so afterPropertiesSet() + // never opens a session. + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertTrue(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertTrue(context.getBean(SESSION_POOL_BEAN_NAME, ITableSessionPool.class) != null); + assertTrue(context.containsBean("ioTDBTableTimeseriesDao")); + assertTrue(context.getBean(IoTDBTableTimeseriesDao.class) != null); + }); + } + + @Test + void autoConfigDiscovery_withSelectorButNoExperimentalFlag_createsNoBeans() { + contextRunner + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("timeseriesWriter")); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + assertTrue(context.getBeansOfType(TimeseriesDao.class).isEmpty()); + }); + } + + @Test + void autoConfigDiscovery_withoutSelector_createsNoBeans() { + contextRunner.run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + }); + } + + @Test + void autoConfigDiscovery_withoutSelector_doesNotBindOrValidateIoTDBProperties() { + contextRunner + .withPropertyValues("iotdb.database=bad-name") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + }); + } + + @Test + void autoConfigDiscovery_withoutThingsBoardClasspath_createsNoBeans() { + contextRunner + .withClassLoader(new FilteredClassLoader("org.thingsboard")) + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("timeseriesWriter")); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + }); + } + + @Test + void hostProvidedTimeseriesDao_failsFastWhenIoTDBBackendEnabled() { + contextRunner + .withUserConfiguration(HostTimeseriesDaoConfiguration.class) + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + Throwable failure = context.getStartupFailure(); + assertTrue(failure != null, "context should fail on conflicting TimeseriesDao"); + assertTrue( + failureContains( + failure, + "database.ts.type=iotdb-table with iotdb.ts.experimental-raw-only=true, " + + "but a non-IoTDB TimeseriesDao bean 'hostTimeseriesDao' is present; " + + "remove it or unset the IoTDB selector"), + "startup failure should explain the conflicting TimeseriesDao bean: " + failure); + }); + } + + @Test + void hostProvidedTimeseriesDaoUsingModuleBeanName_failsFastWhenIoTDBBackendEnabled() { + contextRunner + .withUserConfiguration(HostNamedTimeseriesDaoConfiguration.class) + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + Throwable failure = context.getStartupFailure(); + assertTrue(failure != null, "context should fail on conflicting TimeseriesDao"); + assertTrue( + failureContains( + failure, + "database.ts.type=iotdb-table with iotdb.ts.experimental-raw-only=true, " + + "but a non-IoTDB TimeseriesDao bean 'ioTDBTableTimeseriesDao' is " + + "present; remove it or unset the IoTDB selector"), + "startup failure should explain the conflicting TimeseriesDao bean: " + failure); + }); + } + + private static boolean failureContains(Throwable failure, String expected) { + for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + String message = cause.getMessage(); + if (message != null && message.contains(expected)) { + return true; + } + } + return false; + } + + @Configuration + static class HostTimeseriesDaoConfiguration { + static final TimeseriesDao HOST_DAO = new NoopTimeseriesDao(); + + @Bean + TimeseriesDao hostTimeseriesDao() { + return HOST_DAO; + } + } + + @Configuration + static class HostNamedTimeseriesDaoConfiguration { + @Bean(name = IoTDBTableConfiguration.IOTDB_TABLE_TIMESERIES_DAO_BEAN_NAME) + TimeseriesDao ioTDBTableTimeseriesDao() { + return HostTimeseriesDaoConfiguration.HOST_DAO; + } + } + + /** Minimal host-supplied {@link TimeseriesDao} used only to trigger the back-off conditional. */ + private static final class NoopTimeseriesDao implements TimeseriesDao { + @Override + public ListenableFuture> findAllAsync( + TenantId tenantId, EntityId entityId, List queries) { + throw new UnsupportedOperationException(); + } + + @Override + public ListenableFuture save( + TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + throw new UnsupportedOperationException(); + } + + @Override + public ListenableFuture savePartition( + TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key) { + throw new UnsupportedOperationException(); + } + + @Override + public ListenableFuture remove( + TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + throw new UnsupportedOperationException(); + } + + @Override + public void cleanup(long systemTtl) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDaoTest.java new file mode 100644 index 00000000..b1ce967c --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableBaseDaoTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class IoTDBTableBaseDaoTest { + + @Test + void getEntry_mapsBooleanColumn() throws StatementExecutionException { + SessionDataSet.DataIterator row = mock(SessionDataSet.DataIterator.class); + when(row.isNull("bool_v")).thenReturn(false); + when(row.isNull("long_v")).thenReturn(true); + when(row.isNull("double_v")).thenReturn(true); + when(row.isNull("str_v")).thenReturn(true); + when(row.isNull("json_v")).thenReturn(true); + when(row.getBoolean("bool_v")).thenReturn(true); + + TypedKvValue result = baseDao().getEntry(row); + + assertTrue(result.hasValue()); + assertTrue(result.booleanValue()); + assertNull(result.longValue()); + assertNull(result.doubleValue()); + assertNull(result.stringValue()); + assertNull(result.jsonValue()); + } + + @Test + void getEntry_mapsLongColumn() throws StatementExecutionException { + SessionDataSet.DataIterator row = mock(SessionDataSet.DataIterator.class); + when(row.isNull("bool_v")).thenReturn(true); + when(row.isNull("long_v")).thenReturn(false); + when(row.isNull("double_v")).thenReturn(true); + when(row.isNull("str_v")).thenReturn(true); + when(row.isNull("json_v")).thenReturn(true); + when(row.getLong("long_v")).thenReturn(42L); + + TypedKvValue result = baseDao().getEntry(row); + + assertTrue(result.hasValue()); + assertNull(result.booleanValue()); + assertEquals(42L, result.longValue()); + assertNull(result.doubleValue()); + assertNull(result.stringValue()); + assertNull(result.jsonValue()); + } + + @Test + void getEntry_handlesAllNullRow() throws StatementExecutionException { + SessionDataSet.DataIterator row = mock(SessionDataSet.DataIterator.class); + when(row.isNull("bool_v")).thenReturn(true); + when(row.isNull("long_v")).thenReturn(true); + when(row.isNull("double_v")).thenReturn(true); + when(row.isNull("str_v")).thenReturn(true); + when(row.isNull("json_v")).thenReturn(true); + + TypedKvValue result = baseDao().getEntry(row); + + assertFalse(result.hasValue()); + } + + @Test + void getEntry_throwsOnMultipleNonNull() throws StatementExecutionException { + SessionDataSet.DataIterator row = mock(SessionDataSet.DataIterator.class); + when(row.isNull("bool_v")).thenReturn(false); + when(row.isNull("long_v")).thenReturn(false); + when(row.isNull("double_v")).thenReturn(true); + when(row.isNull("str_v")).thenReturn(true); + when(row.isNull("json_v")).thenReturn(true); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> baseDao().getEntry(row)); + + assertTrue(exception.getMessage().contains("telemetry schema stores exactly one typed value")); + } + + private IoTDBTableBaseDao baseDao() { + return new IoTDBTableBaseDao(mock(ITableSessionPool.class)); + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfigTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfigTest.java new file mode 100644 index 00000000..4daebc23 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableConfigTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import jakarta.validation.ValidatorFactory; +import org.hibernate.validator.messageinterpolation.ParameterMessageInterpolator; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.context.properties.source.MapConfigurationPropertySource; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class IoTDBTableConfigTest { + + @Test + void defaults_haveExpectedValues() { + IoTDBTableConfig config = new IoTDBTableConfig(); + + assertEquals("127.0.0.1", config.getHost()); + assertEquals(6667, config.getPort()); + assertEquals(8, config.getSessionPoolSize()); + assertEquals("thingsboard", config.getDatabase()); + assertEquals(-1L, config.getDefaultTtlMs()); + assertEquals("root", config.getUsername()); + assertEquals("root", config.getPassword()); + assertEquals(5000, config.getConnectionTimeoutMs()); + assertFalse(config.isEnableCompression()); + assertFalse(config.getTs().isExperimentalRawOnly()); + assertEquals(4, config.getTs().getRead().getThreads()); + assertEquals(500, config.getTs().getSave().getBatchSize()); + assertEquals(20L, config.getTs().getSave().getMaxLingerMs()); + assertEquals(50000, config.getTs().getSave().getQueueCapacity()); + assertEquals(5000L, config.getTs().getSave().getShutdownDrainTimeoutMs()); + assertEquals(1, config.getTs().getSave().getFlushThreads()); + assertEquals(3, config.getTs().getSave().getRetryMaxAttempts()); + assertEquals(50L, config.getTs().getSave().getRetryInitialBackoffMs()); + assertEquals(1000L, config.getTs().getSave().getRetryMaxBackoffMs()); + assertEquals(10000, config.getTs().getRead().getQueueCapacity()); + } + + @Test + void binding_fromProperties_overridesDefaults() { + MapConfigurationPropertySource source = + new MapConfigurationPropertySource( + Map.ofEntries( + Map.entry("iotdb.host", "10.0.0.5"), + Map.entry("iotdb.port", "6668"), + Map.entry("iotdb.session-pool-size", "16"), + Map.entry("iotdb.database", "tenant_a"), + Map.entry("iotdb.default-ttl-ms", "86400000"), + Map.entry("iotdb.username", "iot_user"), + Map.entry("iotdb.password", "iot_pass"), + Map.entry("iotdb.connection-timeout-ms", "9000"), + Map.entry("iotdb.enable-compression", "true"), + Map.entry("iotdb.ts.experimental-raw-only", "true"), + Map.entry("iotdb.ts.read.threads", "6"), + Map.entry("iotdb.ts.read.queue-capacity", "7000"), + Map.entry("iotdb.ts.save.batch-size", "250"), + Map.entry("iotdb.ts.save.max-linger-ms", "30"), + Map.entry("iotdb.ts.save.queue-capacity", "2000"), + Map.entry("iotdb.ts.save.shutdown-drain-timeout-ms", "3000"), + Map.entry("iotdb.ts.save.flush-threads", "1"), + Map.entry("iotdb.ts.save.retry-max-attempts", "4"), + Map.entry("iotdb.ts.save.retry-initial-backoff-ms", "10"), + Map.entry("iotdb.ts.save.retry-max-backoff-ms", "100"))); + + IoTDBTableConfig config = new Binder(source).bind("iotdb", IoTDBTableConfig.class).get(); + + assertEquals("10.0.0.5", config.getHost()); + assertEquals(6668, config.getPort()); + assertEquals(16, config.getSessionPoolSize()); + assertEquals("tenant_a", config.getDatabase()); + assertEquals(86400000L, config.getDefaultTtlMs()); + assertEquals("iot_user", config.getUsername()); + assertEquals("iot_pass", config.getPassword()); + assertEquals(9000, config.getConnectionTimeoutMs()); + assertEquals(true, config.isEnableCompression()); + assertTrue(config.getTs().isExperimentalRawOnly()); + assertEquals(6, config.getTs().getRead().getThreads()); + assertEquals(7000, config.getTs().getRead().getQueueCapacity()); + assertEquals(250, config.getTs().getSave().getBatchSize()); + assertEquals(30L, config.getTs().getSave().getMaxLingerMs()); + assertEquals(2000, config.getTs().getSave().getQueueCapacity()); + assertEquals(3000L, config.getTs().getSave().getShutdownDrainTimeoutMs()); + assertEquals(1, config.getTs().getSave().getFlushThreads()); + assertEquals(4, config.getTs().getSave().getRetryMaxAttempts()); + assertEquals(10L, config.getTs().getSave().getRetryInitialBackoffMs()); + assertEquals(100L, config.getTs().getSave().getRetryMaxBackoffMs()); + } + + @Test + void validation_rejectsInvalidJakartaConstraints() { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.setHost(" "); + config.setPort(70000); + config.setSessionPoolSize(0); + config.getTs().getRead().setThreads(0); + config.getTs().getRead().setQueueCapacity(0); + config.getTs().getSave().setShutdownDrainTimeoutMs(0); + + Set violationPaths = + validate(config).stream() + .map(v -> v.getPropertyPath().toString()) + .collect(Collectors.toSet()); + + assertTrue(violationPaths.contains("host")); + assertTrue(violationPaths.contains("port")); + assertTrue(violationPaths.contains("sessionPoolSize")); + assertTrue(violationPaths.contains("ts.read.threads")); + assertTrue(violationPaths.contains("ts.read.queueCapacity")); + assertTrue(violationPaths.contains("ts.save.shutdownDrainTimeoutMs")); + } + + @Test + void validation_rejectsMalformedDatabaseName() { + // The database name is spliced into CREATE DATABASE / USE DDL, so a name with a semicolon, a + // space or a leading digit must be rejected by the @Pattern (IoTDB identifier) constraint. + for (String bad : new String[] {"tb;drop", "tb db", "1tb", "tb-1", ""}) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.setDatabase(bad); + Set violationPaths = + validate(config).stream() + .map(v -> v.getPropertyPath().toString()) + .collect(Collectors.toSet()); + assertTrue( + violationPaths.contains("database"), + "database name '" + bad + "' must be rejected by validation"); + } + } + + @Test + void validation_acceptsValidDatabaseName() { + for (String good : new String[] {"thingsboard", "tb_custom", "_tb", "TB1", "tenant_a"}) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.setDatabase(good); + Set violationPaths = + validate(config).stream() + .map(v -> v.getPropertyPath().toString()) + .collect(Collectors.toSet()); + assertFalse( + violationPaths.contains("database"), "database name '" + good + "' must pass validation"); + } + } + + private Set> validate(IoTDBTableConfig config) { + try (ValidatorFactory factory = + Validation.byDefaultProvider() + .configure() + .messageInterpolator(new ParameterMessageInterpolator()) + .buildValidatorFactory()) { + Validator validator = factory.getValidator(); + return validator.validate(config); + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableContextStartupTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableContextStartupTest.java new file mode 100644 index 00000000..0acca26b --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableContextStartupTest.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.pool.ITableSessionPool; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +class IoTDBTableContextStartupTest { + private static final String SESSION_POOL_BEAN_NAME = + IoTDBTableConfiguration.IOTDB_TABLE_SESSION_POOL_BEAN_NAME; + + private final ApplicationContextRunner contextRunner = + new ApplicationContextRunner().withUserConfiguration(TableContextTestConfiguration.class); + + @Test + void noActivation_contextStartsWithoutIoTDBBeans() { + contextRunner.run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + assertFalse(context.containsBeanDefinition("ioTDBTableLatestDao")); + assertFalse(context.containsBeanDefinition("ioTDBTableLabelDao")); + }); + } + + @Test + void tsTypeActivationWithExperimentalFlag_createsPoolAndTimeseries() { + contextRunner + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + // Disable the startup schema bootstrap: this is an offline context test with no real + // IoTDB, so the bootstrap's afterPropertiesSet() must not try to open a session. + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertTrue(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertTrue(context.getBean(SESSION_POOL_BEAN_NAME, ITableSessionPool.class) != null); + assertTrue(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + assertTrue(context.getBean(IoTDBTableTimeseriesDao.class) != null); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableLatestDao")); + assertFalse(context.containsBeanDefinition("ioTDBTableLabelDao")); + }); + } + + @Test + void tsTypeActivationWithoutExperimentalFlag_createsNoIoTDBBeansOrTimeseriesDao() { + contextRunner + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("timeseriesWriter")); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + assertTrue(context.getBeansOfType(TimeseriesDao.class).isEmpty()); + }); + } + + // This module implements only the timeseries backend, so the latest-telemetry selector must NOT + // spin up a session pool or run the schema-bootstrap DDL before the latest DAO ships. + @Test + void tsLatestSelectorAlone_doesNotActivatePoolOrBootstrap() { + contextRunner + .withPropertyValues( + "database.ts_latest.type=iotdb-table", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + }); + } + + // Likewise the label selector must NOT activate the pool or schema bootstrap yet. + @Test + void labelsSelectorAlone_doesNotActivatePoolOrBootstrap() { + contextRunner + .withPropertyValues( + "iotdb.labels.enabled=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000") + .run( + context -> { + assertFalse(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertFalse(context.containsBeanDefinition("schemaBootstrap")); + assertFalse(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + }); + } + + @Test + void uppercaseSelector_stillActivatesPoolAndDao() { + contextRunner + .withPropertyValues( + "database.ts.type=IOTDB-TABLE", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + assertTrue(context.containsBean(SESSION_POOL_BEAN_NAME)); + assertTrue(context.containsBeanDefinition("ioTDBTableTimeseriesDao")); + }); + } + + @Test + void hostForeignSessionPoolDoesNotSuppressOrFeedModulePool() { + contextRunner + .withUserConfiguration(HostSessionPoolConfiguration.class) + .withPropertyValues( + "database.ts.type=iotdb-table", + "iotdb.ts.experimental-raw-only=true", + "iotdb.host=localhost", + "iotdb.port=6667", + "iotdb.username=root", + "iotdb.password=root", + "iotdb.session-pool-size=8", + "iotdb.connection-timeout-ms=5000", + "iotdb.schema.bootstrap=false") + .run( + context -> { + ITableSessionPool modulePool = + context.getBean(SESSION_POOL_BEAN_NAME, ITableSessionPool.class); + assertTrue(context.containsBean("hostSessionPool")); + assertNotSame(HostSessionPoolConfiguration.HOST_POOL, modulePool); + IoTDBTableTimeseriesDao dao = context.getBean(IoTDBTableTimeseriesDao.class); + IoTDBTableTimeseriesWriter writer = context.getBean(IoTDBTableTimeseriesWriter.class); + assertSame(modulePool, dao.tableSessionPool); + assertSame(modulePool, fieldValue(writer, "tableSessionPool")); + }); + } + + private static Object fieldValue(Object target, String fieldName) { + try { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + @Configuration + static class HostSessionPoolConfiguration { + static final ITableSessionPool HOST_POOL = mock(ITableSessionPool.class); + + @Bean + @Primary + ITableSessionPool hostSessionPool() { + return HOST_POOL; + } + } + + // IoTDBTableConfiguration registers the DAO bean itself via its @Bean methods, so importing the + // auto-configuration is enough -- no extra @ComponentScan is needed here. This mirrors how a real + // ThingsBoard deployment activates the module purely through the Spring Boot auto-configuration + // import, without component-scanning org.apache.iotdb.extras. + @Configuration + @Import(IoTDBTableConfiguration.class) + static class TableContextTestConfiguration {} +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrapTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrapTest.java new file mode 100644 index 00000000..658b25a5 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableSchemaBootstrapTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link IoTDBTableSchemaBootstrap}: it reads the packaged schema SQL, executes each + * statement through a table session, rewrites the database name when the configured database + * differs from the schema default, and tolerates "already exists" failures so a second run is + * idempotent. + */ +class IoTDBTableSchemaBootstrapTest { + + @Test + void appliesEveryStatementAndUsesConfiguredDatabaseName() throws Exception { + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + + IoTDBTableConfig config = new IoTDBTableConfig(); + config.setDatabase("tb_custom"); + + new IoTDBTableSchemaBootstrap(pool, config).afterPropertiesSet(); + + ArgumentCaptor statements = ArgumentCaptor.forClass(String.class); + verify(session, times(4)).executeNonQueryStatement(statements.capture()); + List executed = statements.getAllValues(); + // CREATE DATABASE, USE, CREATE TABLE telemetry, CREATE TABLE entity_attributes. + assertEquals(4, executed.size()); + assertTrue( + executed.get(0).contains("CREATE DATABASE IF NOT EXISTS tb_custom"), + "database name should be rewritten to the configured database: " + executed.get(0)); + assertTrue(executed.get(1).contains("USE tb_custom"), executed.get(1)); + assertTrue( + executed.stream().anyMatch(s -> s.contains("CREATE TABLE telemetry")), + "telemetry table DDL must be applied"); + assertTrue( + executed.stream().anyMatch(s -> s.contains("CREATE TABLE entity_attributes")), + "entity_attributes table DDL must be applied"); + assertContainsInOrder( + statementContaining(executed, "CREATE TABLE telemetry"), + "entity_type", + "tenant_id", + "key", + "entity_id", + "bool_v"); + assertContainsInOrder( + statementContaining(executed, "CREATE TABLE entity_attributes"), + "time", + "attribute_scope", + "entity_type", + "tenant_id", + "key", + "entity_id", + "bool_v"); + verify(session).close(); + } + + @Test + void keepsSchemaDefaultDatabaseWhenConfiguredDatabaseMatches() throws Exception { + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + + IoTDBTableConfig config = new IoTDBTableConfig(); // default database = thingsboard + + new IoTDBTableSchemaBootstrap(pool, config).afterPropertiesSet(); + + ArgumentCaptor statements = ArgumentCaptor.forClass(String.class); + verify(session, times(4)).executeNonQueryStatement(statements.capture()); + assertTrue( + statements.getAllValues().get(0).contains("CREATE DATABASE IF NOT EXISTS thingsboard"), + statements.getAllValues().get(0)); + } + + @Test + void toleratesAlreadyExistsSoReRunIsIdempotent() throws Exception { + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + // Simulate a second bootstrap run: the CREATE TABLE statements fail because the tables already + // exist. The bootstrap must swallow these and complete without throwing. + doThrow(new StatementExecutionException("Table 'telemetry' already exists")) + .when(session) + .executeNonQueryStatement(org.mockito.ArgumentMatchers.contains("CREATE TABLE")); + + IoTDBTableConfig config = new IoTDBTableConfig(); + + // Must not throw despite the already-exists failures. + new IoTDBTableSchemaBootstrap(pool, config).afterPropertiesSet(); + + verify(session, times(4)).executeNonQueryStatement(anyString()); + verify(session).close(); + } + + @Test + void propagatesNonAlreadyExistsFailures() throws Exception { + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + doThrow(new StatementExecutionException("permission denied")) + .when(session) + .executeNonQueryStatement(anyString()); + + IoTDBTableConfig config = new IoTDBTableConfig(); + + assertThrows( + StatementExecutionException.class, + () -> new IoTDBTableSchemaBootstrap(pool, config).afterPropertiesSet()); + } + + @Test + void rejectsMalformedDatabaseNameBeforeOpeningSession() throws Exception { + // Defense-in-depth: even constructed directly (bypassing @Pattern bean validation), the + // bootstrap must reject a database name that is not a valid IoTDB identifier before it splices + // the name into CREATE DATABASE / USE DDL, and must not open a session. + ITableSessionPool pool = mock(ITableSessionPool.class); + + for (String bad : new String[] {"tb;drop", "tb db", "1tb"}) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.setDatabase(bad); + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> new IoTDBTableSchemaBootstrap(pool, config).afterPropertiesSet()); + assertTrue( + ex.getMessage().contains(bad), + "rejection message should name the offending database: " + ex.getMessage()); + } + // No session was ever requested for any of the rejected names. + verify(pool, never()).getSession(); + } + + private static String statementContaining(List statements, String needle) { + return statements.stream() + .filter(statement -> statement.contains(needle)) + .findFirst() + .orElseThrow(() -> new AssertionError("missing statement containing " + needle)); + } + + private static void assertContainsInOrder(String text, String... needles) { + int previousIndex = -1; + for (String needle : needles) { + int nextIndex = text.indexOf(needle, previousIndex + 1); + assertTrue( + nextIndex > previousIndex, + "expected " + needle + " after index " + previousIndex + " in " + text); + previousIndex = nextIndex; + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java new file mode 100644 index 00000000..1f5dcb2f --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java @@ -0,0 +1,756 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Integration tests for the IoTDB Table Mode timeseries DAO against a real IoTDB 2.0.8 container: + * the WRITE path (verified by reading the telemetry table back through raw table-session SQL) plus + * the RAW (non-aggregated) READ path and the DELETE path exercised through the DAO. The + * time-bucketed aggregation read path is not implemented and is not exercised here. + */ +@Tag("integration") +@Testcontainers(disabledWithoutDocker = true) +class IoTDBTableTimeseriesDaoIT { + // Cold testcontainer first writes are slower than a warm production node, so the per-future + // assertion timeout is generous; production throughput is covered elsewhere, not here. + private static final int FUTURE_TIMEOUT_SECONDS = 30; + private static final Duration IOTDB_STARTUP_TIMEOUT = Duration.ofMinutes(3); + private static final Duration IOTDB_READY_TIMEOUT = Duration.ofSeconds(60); + private static final Duration IOTDB_READY_POLL_INTERVAL = Duration.ofMillis(500); + private static String originalDatabaseTsType; + private static String originalExperimentalRawOnly; + + @Container + static final GenericContainer IOTDB = + new GenericContainer<>(DockerImageName.parse("apache/iotdb:2.0.8-standalone")) + .withExposedPorts(6667) + // IoTDB binds its client RPC service to dn_rpc_address (default 127.0.0.1), so it would + // only listen on the container loopback and reject the Testcontainers port-mapped session + // handshake ("Fail to reconnect"). Bind to all interfaces so the mapped host port works. + .withEnv("dn_rpc_address", "0.0.0.0") + .waitingFor(Wait.forListeningPort().withStartupTimeout(IOTDB_STARTUP_TIMEOUT)); + + @BeforeAll + static void enableRawOnlyBackendProperties() { + originalDatabaseTsType = System.getProperty("database.ts.type"); + originalExperimentalRawOnly = System.getProperty("iotdb.ts.experimental-raw-only"); + System.setProperty("database.ts.type", "iotdb-table"); + System.setProperty("iotdb.ts.experimental-raw-only", "true"); + } + + @AfterAll + static void restoreRawOnlyBackendProperties() { + restoreProperty("database.ts.type", originalDatabaseTsType); + restoreProperty("iotdb.ts.experimental-raw-only", originalExperimentalRawOnly); + } + + /** + * Pins the IoTDB 2.0.8 engine behavior that a database-bound table-session pool can bootstrap a + * not-yet-existing database. If a future IoTDB image changes that behavior, this test fails + * before first boot breaks in production. + */ + @Test + void schemaBootstrapCreatesAFreshDatabaseThroughADatabaseBoundPool() throws Exception { + String database = uniqueDatabase("fresh_boot"); + try (ITableSessionPool pool = newPool(database)) { + try { + IoTDBTableConfig config = config(1); + config.setDatabase(database); + IoTDBTableSchemaBootstrap bootstrap = new IoTDBTableSchemaBootstrap(pool, config); + + bootstrap.afterPropertiesSet(); + + try (ITableSession session = pool.getSession()) { + session.executeNonQueryStatement("USE " + database); + try (SessionDataSet dataSet = session.executeQueryStatement("SHOW TABLES")) { + String tableNameColumn = dataSet.getColumnNames().get(0); + SessionDataSet.DataIterator rows = dataSet.iterator(); + List tableNames = new ArrayList<>(); + while (rows.next()) { + tableNames.add(rows.getString(tableNameColumn)); + } + assertTrue( + tableNames.containsAll(List.of("telemetry", "entity_attributes")), + "schema bootstrap should create telemetry and entity_attributes tables: " + + tableNames); + } + } + + bootstrap.afterPropertiesSet(); + } finally { + try (ITableSessionPool cleanupPool = newPool(null); + ITableSession cleanupSession = cleanupPool.getSession()) { + cleanupSession.executeNonQueryStatement("DROP DATABASE IF EXISTS " + database); + } catch (Exception e) { + // Keep cleanup failures from replacing the bootstrap failure under test. + } + } + } + } + + @Test + void schemaBootstrap_writesAllTypesAndReadsExactlyOneField() throws Exception { + TestScope scope = + scope( + "all_types", + "33333333-3333-3333-3333-333333333301", + "44444444-4444-4444-4444-444444444401"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(5); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + List.of( + dao.save( + scope.tenantId(), + scope.entityId(), + entry(1000L, "bool", DataType.BOOLEAN, true), + 0), + dao.save( + scope.tenantId(), + scope.entityId(), + entry(1001L, "long", DataType.LONG, 42L), + 0), + dao.save( + scope.tenantId(), + scope.entityId(), + entry(1002L, "double", DataType.DOUBLE, 4.2D), + 0), + dao.save( + scope.tenantId(), + scope.entityId(), + entry(1003L, "string", DataType.STRING, "value"), + 0), + dao.save( + scope.tenantId(), + scope.entityId(), + entry(1004L, "json", DataType.JSON, "{\"v\":1}"), + 0)) + .forEach( + future -> { + try { + assertEquals(1, future.get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + + int rowCount = 0; + for (String key : List.of("bool", "long", "double", "string", "json")) { + rowCount += assertTelemetryRows(pool, scope, key, 1, 1); + } + assertEquals(5, rowCount); + } finally { + writer.destroy(); + } + } + } + + @Test + void writesFiveHundredMixedEntriesInOneFlush() throws Exception { + TestScope scope = + scope( + "mixed_batch", + "33333333-3333-3333-3333-333333333302", + "44444444-4444-4444-4444-444444444402"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(500); + config.getTs().getSave().setMaxLingerMs(5000L); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 500; i++) { + futures.add( + dao.save( + scope.tenantId(), + scope.entityId(), + entry(2000L + i, "mixed-" + i, DataType.LONG, (long) i), + 0)); + } + for (ListenableFuture future : futures) { + assertEquals(1, future.get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(500, dao.stats().flushed()); + assertTelemetryRows(pool, scope, "mixed-499", 1, 1); + } finally { + writer.destroy(); + } + } + } + + @Test + void saveThenFindAllAsync_roundTripsAllFiveTypes() throws Exception { + TestScope scope = + scope( + "read_all_types", + "33333333-3333-3333-3333-333333333304", + "44444444-4444-4444-4444-444444444404"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(5); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(4000L, "bool", DataType.BOOLEAN, true), + entry(4001L, "long", DataType.LONG, 42L), + entry(4002L, "double", DataType.DOUBLE, 4.2D), + entry(4003L, "string", DataType.STRING, "value"), + entry(4004L, "json", DataType.JSON, "{\"v\":1}"))); + + List queries = + List.of( + new BaseReadTsKvQuery("bool", 3990L, 4010L, 1, "ASC"), + new BaseReadTsKvQuery("long", 3990L, 4010L, 1, "ASC"), + new BaseReadTsKvQuery("double", 3990L, 4010L, 1, "ASC"), + new BaseReadTsKvQuery("string", 3990L, 4010L, 1, "ASC"), + new BaseReadTsKvQuery("json", 3990L, 4010L, 1, "ASC")); + List results = + dao.findAllAsync(scope.tenantId(), scope.entityId(), queries) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertSingleEntry(results.get(0), 4000L, "bool", DataType.BOOLEAN, true); + assertSingleEntry(results.get(1), 4001L, "long", DataType.LONG, 42L); + assertSingleEntry(results.get(2), 4002L, "double", DataType.DOUBLE, 4.2D); + assertSingleEntry(results.get(3), 4003L, "string", DataType.STRING, "value"); + assertSingleEntry(results.get(4), 4004L, "json", DataType.JSON, "{\"v\":1}"); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void findAllAsync_honorsOrderAndLimit() throws Exception { + TestScope scope = + scope( + "read_order_limit", + "33333333-3333-3333-3333-333333333305", + "44444444-4444-4444-4444-444444444405"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(3); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(5000L, "ordered", DataType.LONG, 1L), + entry(5001L, "ordered", DataType.LONG, 2L), + entry(5002L, "ordered", DataType.LONG, 3L))); + + ReadTsKvQuery desc = new BaseReadTsKvQuery("ordered", 4990L, 5010L, 2, "DESC"); + ReadTsKvQuery asc = new BaseReadTsKvQuery("ordered", 4990L, 5010L, 2, "ASC"); + List results = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(desc, asc)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertTimestamps(results.get(0), 5002L, 5001L); + assertTimestamps(results.get(1), 5000L, 5001L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void findAllAsync_usesHalfOpenEnd() throws Exception { + TestScope scope = + scope( + "read_half_open", + "33333333-3333-3333-3333-333333333306", + "44444444-4444-4444-4444-444444444406"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(3); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(6000L, "window", DataType.LONG, 1L), + entry(6001L, "window", DataType.LONG, 2L), + entry(6002L, "window", DataType.LONG, 3L))); + + ReadTsKvQuery query = new BaseReadTsKvQuery("window", 6000L, 6002L, 10, "ASC"); + ReadTsKvQueryResult result = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + assertTimestamps(result, 6000L, 6001L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void remove_deletesOnlyIdentityKeyAndHalfOpenRange() throws Exception { + TestScope scope = + scope( + "remove_half_open", + "33333333-3333-3333-3333-333333333307", + "44444444-4444-4444-4444-444444444407"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(7000L, "target", DataType.LONG, 1L), + entry(7001L, "target", DataType.LONG, 2L), + entry(7002L, "target", DataType.LONG, 3L), + entry(7001L, "other", DataType.LONG, 4L))); + + dao.remove( + scope.tenantId(), scope.entityId(), new BaseDeleteTsKvQuery("target", 7000L, 7002L)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + ReadTsKvQuery target = new BaseReadTsKvQuery("target", 6990L, 7010L, 10, "ASC"); + ReadTsKvQuery other = new BaseReadTsKvQuery("other", 6990L, 7010L, 10, "ASC"); + List results = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(target, other)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertTimestamps(results.get(0), 7002L); + assertTimestamps(results.get(1), 7001L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + /** + * Pins the documented current-scope limitation (module README "Known limitations"): a save of the + * same (tenant, entity, key, timestamp) with a different value type in a separate flush + * can leave two typed columns on that one row, and a raw read of that point then fails fast with + * the single-typed-column invariant exception ({@link IllegalStateException} from {@link + * IoTDBTableBaseDao#getEntry}) rather than returning wrong data. {@code batchSize = 1} forces + * each save into its own flush so the LONG and the same-timestamp STRING are written by two + * distinct tablets. The delete-then-insert overwrite is outside the current scope; this test + * documents and locks the honest fail-fast behavior. + */ + @Test + void sameTimestampTypeChangeAcrossFlushes_readFailsFast() throws Exception { + TestScope scope = + scope( + "type_change", + "33333333-3333-3333-3333-333333333309", + "44444444-4444-4444-4444-444444444409"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + // batchSize = 1 => every save flushes on its own, so the two writes land in two separate + // flushes/tablets at the same (tenant, entity, key, timestamp). + IoTDBTableConfig config = config(1); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + long ts = 9000L; + String key = "type_change"; + + // Flush 1: LONG at ts. + assertEquals( + 1, + dao.save(scope.tenantId(), scope.entityId(), entry(ts, key, DataType.LONG, 7L), 0) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // Flush 2: STRING at the SAME ts. Lands in long_v vs str_v on the same row. + assertEquals( + 1, + dao.save(scope.tenantId(), scope.entityId(), entry(ts, key, DataType.STRING, "x"), 0) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + // Confirm the row now actually carries two typed columns (the poisoned state). + assertTelemetryRows(pool, scope, key, 1, 2); + + // A raw read of that point must fail fast with the single-typed-column invariant, not + // return a value. + ReadTsKvQuery query = new BaseReadTsKvQuery(key, ts - 10L, ts + 10L, 10, "ASC"); + ExecutionException failure = + assertThrows( + ExecutionException.class, + () -> + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Throwable cause = failure.getCause(); + assertTrue( + cause instanceof IllegalStateException, + "raw read of a poisoned point must fail with IllegalStateException, was: " + cause); + assertTrue( + cause.getMessage() != null && cause.getMessage().contains("typed value columns set"), + "fail-fast message should name the single-typed-column invariant: " + + cause.getMessage()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void keyEscaping_roundTrip() throws Exception { + TestScope scope = + scope( + "key_escape", + "33333333-3333-3333-3333-333333333308", + "44444444-4444-4444-4444-444444444408"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(1); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + String key = "a'b"; + saveAll(dao, scope, List.of(entry(8000L, key, DataType.LONG, 8L))); + + ReadTsKvQuery query = new BaseReadTsKvQuery(key, 7990L, 8010L, 10, "ASC"); + ReadTsKvQueryResult readBeforeDelete = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertTimestamps(readBeforeDelete, 8000L); + + dao.remove(scope.tenantId(), scope.entityId(), new BaseDeleteTsKvQuery(key, 7990L, 8010L)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + ReadTsKvQueryResult readAfterDelete = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertTrue(readAfterDelete.getData().isEmpty()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + private ITableSessionPool newPool(String database) { + TableSessionPoolBuilder builder = + new TableSessionPoolBuilder() + .nodeUrls(List.of("127.0.0.1:" + IOTDB.getMappedPort(6667))) + .user("root") + .password("root") + .maxSize(4); + if (database != null) { + builder.database(database); + } + return builder.build(); + } + + private void bootstrapSchema(String database) throws Exception { + awaitIoTDBReady(database); + + String schema; + try (InputStream stream = + IoTDBTableTimeseriesDaoIT.class + .getClassLoader() + .getResourceAsStream("schema-iotdb-table.sql")) { + schema = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + schema = + schema + .replace( + "CREATE DATABASE IF NOT EXISTS thingsboard;", + "CREATE DATABASE IF NOT EXISTS " + database + ";") + .replace("USE thingsboard;", "USE " + database + ";"); + schema = schema.replaceAll("(?s)/\\*.*?\\*/", "").replaceAll("(?m)--.*$", ""); + try (ITableSessionPool bootstrapPool = newPool(null)) { + try (ITableSession session = bootstrapPool.getSession()) { + for (String statement : schema.split(";")) { + String trimmed = statement.trim(); + if (!trimmed.isEmpty()) { + session.executeNonQueryStatement(trimmed); + } + } + } + } + } + + private void awaitIoTDBReady(String database) throws Exception { + long deadlineNanos = System.nanoTime() + IOTDB_READY_TIMEOUT.toNanos(); + Exception lastFailure = null; + while (System.nanoTime() < deadlineNanos) { + try (ITableSessionPool bootstrapPool = newPool(null); + ITableSession session = bootstrapPool.getSession()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS " + database); + return; + } catch (Exception e) { + lastFailure = e; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + if (remainingMillis <= 0) { + break; + } + Thread.sleep(Math.min(IOTDB_READY_POLL_INTERVAL.toMillis(), remainingMillis)); + } + } + throw new IllegalStateException( + "IoTDB did not accept table-session statements within " + IOTDB_READY_TIMEOUT, lastFailure); + } + + private IoTDBTableConfig config(int batchSize) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.getTs().setExperimentalRawOnly(true); + config.getTs().getSave().setBatchSize(batchSize); + config.getTs().getSave().setMaxLingerMs(20L); + config.getTs().getSave().setRetryInitialBackoffMs(1L); + config.getTs().getSave().setRetryMaxBackoffMs(1L); + return config; + } + + private static void restoreProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } + + private TestScope scope(String databasePrefix, String tenantId, String entityId) { + return new TestScope( + uniqueDatabase(databasePrefix), + new TenantId(UUID.fromString(tenantId)), + new TestEntityId(UUID.fromString(entityId), EntityType.DEVICE)); + } + + private String uniqueDatabase(String prefix) { + // IoTDB caps database names at 64 chars; keep the per-test prefix short and + // append a trimmed UUID so the total length stays well within the limit. + String shortPrefix = prefix.length() > 12 ? prefix.substring(0, 12) : prefix; + String shortUuid = UUID.randomUUID().toString().replace("-", "").substring(0, 16); + return "tb_it_" + shortPrefix + "_" + shortUuid; + } + + private int assertTelemetryRows( + ITableSessionPool pool, + TestScope scope, + String key, + int expectedRows, + int expectedTypedFields) + throws Exception { + try (ITableSession session = pool.getSession(); + SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT bool_v,long_v,double_v,str_v,json_v FROM telemetry " + + telemetryWhere(scope, key))) { + SessionDataSet.DataIterator rows = dataSet.iterator(); + int rowCount = 0; + while (rows.next()) { + assertEquals(expectedTypedFields, typedFieldCount(rows)); + rowCount++; + } + assertEquals(expectedRows, rowCount); + return rowCount; + } + } + + private String telemetryWhere(TestScope scope, String key) { + return "WHERE tenant_id='" + + scope.tenantId().getId() + + "' AND entity_type='DEVICE' AND entity_id='" + + scope.entityId().getId() + + "' AND key='" + + key + + "'"; + } + + private int typedFieldCount(SessionDataSet.DataIterator row) throws Exception { + int count = 0; + count += row.isNull("bool_v") ? 0 : 1; + count += row.isNull("long_v") ? 0 : 1; + count += row.isNull("double_v") ? 0 : 1; + count += row.isNull("str_v") ? 0 : 1; + count += row.isNull("json_v") ? 0 : 1; + return count; + } + + private TestTsKvEntry entry(long ts, String key, DataType dataType, Object value) { + return new TestTsKvEntry(ts, key, dataType, value); + } + + private void saveAll(IoTDBTableTimeseriesDao dao, TestScope scope, List entries) + throws Exception { + List> futures = new ArrayList<>(); + for (TestTsKvEntry entry : entries) { + futures.add(dao.save(scope.tenantId(), scope.entityId(), entry, 0)); + } + for (ListenableFuture future : futures) { + assertEquals(1, future.get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + } + + private void assertSingleEntry( + ReadTsKvQueryResult result, long ts, String key, DataType dataType, Object value) { + assertEquals(1, result.getData().size()); + TsKvEntry entry = result.getData().get(0); + assertEquals(ts, entry.getTs()); + assertEquals(key, entry.getKey()); + assertEquals(dataType, entry.getDataType()); + assertEquals(value, entry.getValue()); + assertEquals(String.valueOf(value), entry.getValueAsString()); + assertEquals(ts, result.getLastEntryTs()); + } + + private void assertTimestamps(ReadTsKvQueryResult result, long... expectedTs) { + assertEquals(expectedTs.length, result.getData().size()); + for (int i = 0; i < expectedTs.length; i++) { + assertEquals(expectedTs[i], result.getData().get(i).getTs()); + } + if (expectedTs.length > 0) { + long maxTs = expectedTs[0]; + for (long ts : expectedTs) { + maxTs = Math.max(maxTs, ts); + } + assertEquals(maxTs, result.getLastEntryTs()); + } + } + + private record TestScope(String database, TenantId tenantId, EntityId entityId) {} + + private record TestEntityId(UUID id, EntityType entityType) implements EntityId { + @Override + public UUID getId() { + return id; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + } + + private record TestTsKvEntry(long ts, String key, DataType dataType, Object value) + implements TsKvEntry { + @Override + public long getTs() { + return ts; + } + + @Override + public String getKey() { + return key; + } + + @Override + public DataType getDataType() { + return dataType; + } + + @Override + public Optional getBooleanValue() { + return dataType == DataType.BOOLEAN ? Optional.of((Boolean) value) : Optional.empty(); + } + + @Override + public Optional getLongValue() { + return dataType == DataType.LONG ? Optional.of((Long) value) : Optional.empty(); + } + + @Override + public Optional getDoubleValue() { + return dataType == DataType.DOUBLE ? Optional.of((Double) value) : Optional.empty(); + } + + @Override + public Optional getStrValue() { + return dataType == DataType.STRING ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public Optional getJsonValue() { + return dataType == DataType.JSON ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public String getValueAsString() { + return String.valueOf(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public Long getVersion() { + return null; + } + + @Override + public int getDataPoints() { + return 1; + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java new file mode 100644 index 00000000..94f2361c --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java @@ -0,0 +1,1449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the IoTDB Table Mode timeseries DAO: the WRITE path (multi-row Tablet mapping, + * batch flushing, connection retry, back-pressure rejection and graceful-shutdown drain) plus the + * RAW (non-aggregated) READ path, the DELETE path and the bounded read thread-pool. The + * time-bucketed aggregation read path is not implemented and is not exercised here. + */ +class IoTDBTableTimeseriesDaoTest { + private static final TenantId TENANT_ID = + new TenantId(UUID.fromString("11111111-1111-1111-1111-111111111111")); + private static final EntityId ENTITY_ID = + new TestEntityId(UUID.fromString("22222222-2222-2222-2222-222222222222"), EntityType.DEVICE); + + private IoTDBTableTimeseriesWriter writer; + private final List daos = new ArrayList<>(); + + @AfterEach + void tearDown() { + for (IoTDBTableTimeseriesDao dao : daos) { + dao.destroy(); + } + daos.clear(); + if (writer != null) { + writer.destroy(); + } + } + + @Test + void save_mapsAllDataTypesIntoSparseTablet() throws Exception { + TestContext context = newContext(config(5, 1000L, 100), true); + + List> futures = + List.of( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "bool", DataType.BOOLEAN, true), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "long", DataType.LONG, 42L), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(3L, "double", DataType.DOUBLE, 3.5D), 0), + context + .dao() + .save(TENANT_ID, ENTITY_ID, entry(4L, "string", DataType.STRING, "abc"), 0), + context + .dao() + .save(TENANT_ID, ENTITY_ID, entry(5L, "json", DataType.JSON, "{\"v\":1}"), 0)); + + for (ListenableFuture future : futures) { + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + } + + Tablet tablet = insertedTablet(context.session(), 1); + assertEquals("telemetry", tablet.getTableName()); + assertEquals(5, tablet.getRowSize()); + assertEquals(IoTDBTableTimeseriesWriter.COLUMN_NAMES, schemaNames(tablet)); + assertEquals(IoTDBTableTimeseriesWriter.DATA_TYPES, schemaTypes(tablet)); + assertEquals( + List.of( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD), + tablet.getColumnTypes()); + + assertRow(tablet, 0, 1L, "bool", 4); + assertRow(tablet, 1, 2L, "long", 5); + assertRow(tablet, 2, 3L, "double", 6); + assertRow(tablet, 3, 4L, "string", 7); + assertRow(tablet, 4, 5L, "json", 8); + } + + @Test + void save_returnsDataPointDaysWithEffectiveTtlAndEntryAmplification() throws Exception { + IoTDBTableConfig config = config(1, 1000L, 100); + config.setDefaultTtlMs(TimeUnit.DAYS.toMillis(2)); + TestContext context = newContext(config, true); + String largeString = "s".repeat(513); + String largeJson = "{\"v\":\"" + "j".repeat(1025) + "\"}"; + + ListenableFuture defaultTtlFuture = + context + .dao() + .save( + TENANT_ID, + ENTITY_ID, + entry(1L, "largeString", DataType.STRING, largeString, tbDataPoints(largeString)), + 0); + ListenableFuture perCallTtlFuture = + context + .dao() + .save( + TENANT_ID, + ENTITY_ID, + entry(2L, "largeJson", DataType.JSON, largeJson, tbDataPoints(largeJson)), + TimeUnit.DAYS.toSeconds(1)); + + assertEquals(2 * tbDataPoints(largeString), defaultTtlFuture.get(3, TimeUnit.SECONDS)); + assertEquals(tbDataPoints(largeJson), perCallTtlFuture.get(3, TimeUnit.SECONDS)); + } + + @Test + void save_returnsOneDayEquivalentWhenEffectiveTtlIsLessThanOneDay() throws Exception { + TestContext context = newContext(config(1, 1000L, 100), true); + + ListenableFuture future = + context + .dao() + .save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.DOUBLE, 21.5D), 3600L); + + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + } + + @Test + void savePartition_returnsImmediateZeroAndDoesNotWrite() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + assertEquals( + 0, + context + .dao() + .savePartition(TENANT_ID, ENTITY_ID, 1L, "temperature") + .get(3, TimeUnit.SECONDS)); + + assertEquals(0, context.dao().stats().enqueued()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + @Test + void save_rejectsBlankTelemetryKeyBeforeEnqueue() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + assertFutureFailsWith( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, " ", DataType.LONG, 1L), 0), + IllegalArgumentException.class); + + assertEquals(0, context.dao().stats().enqueued()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + @Test + void cleanup_isNoOp() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + context.dao().cleanup(TimeUnit.DAYS.toMillis(7)); + + verify(context.pool(), never()).getSession(); + } + + @Test + void save_flushesOneTabletWhenBatchThresholdIsReached() throws Exception { + TestContext context = newContext(config(500, 10000L, 1000), true); + List> futures = new ArrayList<>(); + for (int i = 0; i < 500; i++) { + futures.add( + context + .dao() + .save( + TENANT_ID, ENTITY_ID, entry(i, "temperature-" + i, DataType.LONG, (long) i), 0)); + } + + for (ListenableFuture future : futures) { + assertEquals(1, future.get(5, TimeUnit.SECONDS)); + } + + Tablet tablet = insertedTablet(context.session(), 1); + assertEquals(500, tablet.getRowSize()); + assertEquals(500, context.dao().stats().flushed()); + } + + @Test + void save_flushesAfterMaxLingerWhenBatchIsNotFull() throws Exception { + TestContext context = newContext(config(500, 20L, 1000), true); + + ListenableFuture future = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + Tablet tablet = insertedTablet(context.session(), 1); + assertEquals(1, tablet.getRowSize()); + } + + @Test + void save_rejectsImmediatelyWhenQueueIsFull() throws Exception { + TestContext context = newContext(config(500, 10000L, 1), false); + + ListenableFuture accepted = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "first", DataType.LONG, 1L), 0); + ListenableFuture rejected = + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "second", DataType.LONG, 2L), 0); + + assertFalse(accepted.isDone()); + assertFutureFailsWith(rejected, IoTDBTableSaveQueueFullException.class); + assertEquals(1, context.dao().stats().rejectsFull()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + @Test + void rejectWarningLimiterAllowsFirstRejectThenSuppressesUntilWindowExpires() { + AtomicLong clock = new AtomicLong(0L); + writer = newWriterWithClock(config(1, 1000L, 1), clock); + + assertTrue(writer.shouldLogRejectedSaveWarning()); + assertFalse(writer.shouldLogRejectedSaveWarning()); + + clock.set(TimeUnit.SECONDS.toNanos(10) - 1L); + assertFalse(writer.shouldLogRejectedSaveWarning()); + + clock.set(TimeUnit.SECONDS.toNanos(10) + 1L); + assertTrue(writer.shouldLogRejectedSaveWarning()); + } + + @Test + void queueFullAndShutdownRejectsShareWarningLimiter() throws Exception { + AtomicLong clock = new AtomicLong(0L); + writer = newWriterWithClock(config(1, 1000L, 1), clock); + + IoTDBTablePendingSave accepted = pendingSave(1L, "accepted"); + IoTDBTablePendingSave queueFull = pendingSave(2L, "queue-full"); + writer.enqueue(accepted); + writer.enqueue(queueFull); + + assertFalse(accepted.future().isDone()); + assertFutureFailsWith(queueFull.future(), IoTDBTableSaveQueueFullException.class); + assertEquals(1, writer.stats().rejectsFull()); + assertFalse(writer.shouldLogRejectedSaveWarning()); + + clock.set(TimeUnit.SECONDS.toNanos(10) + 1L); + writer.destroy(); + assertFutureFailsWith(accepted.future(), IoTDBTableDaoShuttingDownException.class); + + IoTDBTablePendingSave shutdown = pendingSave(3L, "shutdown"); + writer.enqueue(shutdown); + + assertFutureFailsWith(shutdown.future(), IoTDBTableDaoShuttingDownException.class); + assertEquals(1, writer.stats().rejectsShutdown()); + assertFalse(writer.shouldLogRejectedSaveWarning()); + } + + @Test + void initialBackoffMsCapsInitialBackoffOnlyWhenMaxIsPositive() { + assertEquals(10L, IoTDBTableTimeseriesWriter.initialBackoffMs(50L, 10L)); + assertEquals(10L, IoTDBTableTimeseriesWriter.initialBackoffMs(10L, 50L)); + assertEquals(50L, IoTDBTableTimeseriesWriter.initialBackoffMs(50L, 0L)); + assertEquals(50L, IoTDBTableTimeseriesWriter.initialBackoffMs(50L, -1L)); + } + + @Test + void save_retriesConnectionExceptionThenCompletesWholeBatch() throws Exception { + TestContext context = newContext(config(2, 1000L, 100), true); + AtomicInteger attempts = new AtomicInteger(); + doAnswer( + invocation -> { + if (attempts.getAndIncrement() == 0) { + throw new IoTDBConnectionException("temporary connection failure"); + } + return null; + }) + .when(context.session()) + .insert(any(Tablet.class)); + + ListenableFuture first = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "first", DataType.LONG, 1L), 0); + ListenableFuture second = + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "second", DataType.LONG, 2L), 0); + + assertEquals(1, first.get(3, TimeUnit.SECONDS)); + assertEquals(1, second.get(3, TimeUnit.SECONDS)); + verify(context.session(), timeout(3000).times(2)).insert(any(Tablet.class)); + assertEquals(1, context.dao().stats().retries()); + assertEquals(2, context.dao().stats().flushed()); + } + + @Test + void saveTreatsCloseAfterSuccessfulInsertAsSuccessWithoutReplay() throws Exception { + TestContext context = newContext(config(1, 1000L, 100), true); + doThrow(new IoTDBConnectionException("close failed")).when(context.session()).close(); + + ListenableFuture future = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 1L), 0); + + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + verify(context.session(), timeout(3000).times(1)).insert(any(Tablet.class)); + assertEquals(1, context.dao().stats().flushed()); + assertEquals(0, context.dao().stats().retries()); + assertEquals(0, context.dao().stats().flushFailures()); + } + + @Test + void save_doesNotRetryStatementExecutionExceptionAndFailsWholeBatch() throws Exception { + TestContext context = newContext(config(2, 1000L, 100), true); + doThrow(new StatementExecutionException("bad statement")) + .when(context.session()) + .insert(any(Tablet.class)); + + ListenableFuture first = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "first", DataType.LONG, 1L), 0); + ListenableFuture second = + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "second", DataType.LONG, 2L), 0); + + assertFutureFailsWith(first, StatementExecutionException.class); + assertFutureFailsWith(second, StatementExecutionException.class); + verify(context.session(), timeout(3000).times(1)).insert(any(Tablet.class)); + assertEquals(0, context.dao().stats().retries()); + assertEquals(1, context.dao().stats().flushFailures()); + } + + @Test + void destroyDrainsPendingWritesAndRejectsNewSaves() throws Exception { + TestContext context = newContext(config(10, 10000L, 100), true); + ListenableFuture pending = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + + writer.destroy(); + + assertEquals(1, pending.get(3, TimeUnit.SECONDS)); + verify(context.session(), timeout(3000).times(1)).insert(any(Tablet.class)); + + ListenableFuture rejected = + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "temperature", DataType.LONG, 43L), 0); + assertFutureFailsWith(rejected, IoTDBTableDaoShuttingDownException.class); + assertEquals(1, context.dao().stats().rejectsShutdown()); + } + + @Test + void destroyDrainsInFlightRetryAfterTransientErrorInsteadOfFailingTheBatch() throws Exception { + // A transient IoTDBConnectionException puts the worker into retry backoff; a concurrent + // destroy() + // must let the bounded drain window finish the retry rather than interrupt the backoff sleep + // and + // fail an already-accepted batch. The drain timeout is long and the backoff is long enough that + // destroy() runs while the worker is sleeping between attempts, then the retry completes. + IoTDBTableConfig config = config(1, 1000L, 100); + config.getTs().getSave().setRetryMaxAttempts(3); + config.getTs().getSave().setRetryInitialBackoffMs(300L); + config.getTs().getSave().setRetryMaxBackoffMs(300L); + config.getTs().getSave().setShutdownDrainTimeoutMs(5000L); + TestContext context = newContext(config, true); + + CountDownLatch firstInsertFailed = new CountDownLatch(1); + AtomicInteger inserts = new AtomicInteger(); + doAnswer( + invocation -> { + if (inserts.incrementAndGet() == 1) { + firstInsertFailed.countDown(); + throw new IoTDBConnectionException("transient blip"); + } + return null; // the retry succeeds + }) + .when(context.session()) + .insert(any(Tablet.class)); + + ListenableFuture save = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + assertTrue( + firstInsertFailed.await(3, TimeUnit.SECONDS), + "worker should hit the transient error and enter retry backoff"); + + writer.destroy(); // must NOT interrupt the in-flight retry backoff + + assertEquals( + 1, + save.get(5, TimeUnit.SECONDS), + "the accepted batch must drain (retry completes) rather than fail on shutdown"); + verify(context.session(), timeout(5000).times(2)).insert(any(Tablet.class)); + assertEquals(1, context.dao().stats().retries()); + assertEquals(0, context.dao().stats().flushFailures()); + } + + @Test + void destroyFlushesAPartialBatchPromptlyInsteadOfWaitingOutMaxLinger() throws Exception { + // A worker that has dequeued a PARTIAL batch and is in the linger poll must observe shutdown + // within one poll slice and flush promptly rather than wait out the full maxLingerMs. + // batchSize=2 + // with a single queued item keeps the worker lingering for a second item; a 30s linger makes an + // un-sliced poll fail the fast assertion below. + IoTDBTableConfig config = config(2, 30_000L, 100); // batchSize=2, maxLingerMs=30s + config.getTs().getSave().setShutdownDrainTimeoutMs(30_000L); + TestContext context = newContext(config, true); + + ListenableFuture save = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + + // destroy() on a separate thread so the test thread asserts the prompt flush regardless of + // join. + Thread closer = new Thread(writer::destroy, "closer"); + closer.start(); + + // With the slice fix the partial batch flushes within ~one slice of shutdown; without it the + // worker would wait out the 30s linger and this get() would time out. + assertEquals( + 1, + save.get(3, TimeUnit.SECONDS), + "partial batch must flush promptly on shutdown, not wait out maxLingerMs"); + verify(context.session(), timeout(3000).times(1)).insert(any(Tablet.class)); + closer.join(5_000L); + } + + @Test + void destroyForceStopsDequeuedLingeringBatchAndSettlesFuture() throws Exception { + IoTDBTableConfig config = config(2, 60_000L, 100); + config.getTs().getSave().setShutdownDrainTimeoutMs(1L); + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + DequeuedBatchLatchQueue queue = + new DequeuedBatchLatchQueue(config.getTs().getSave().getQueueCapacity()); + writer = new IoTDBTableTimeseriesWriter(pool, config, true, queue); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + daos.add(dao); + + ListenableFuture save = + dao.save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + assertTrue( + queue.awaitLingerPollStarted(3, TimeUnit.SECONDS), + "worker should dequeue the save and start waiting for the rest of the batch"); + + writer.destroy(); + + assertFutureFailsWith(save, IoTDBTableDaoShuttingDownException.class); + Thread workerThread = writer.workerThread(); + workerThread.join(1_000L); + assertFalse(workerThread.isAlive(), "forced-stop worker should terminate"); + verify(session, never()).insert(any(Tablet.class)); + } + + @Test + void destroyForceStopsSavesFrozenBetweenDrainToAndBatchAppend() throws Exception { + IoTDBTableConfig config = config(3, 60_000L, 100); + config.getTs().getSave().setShutdownDrainTimeoutMs(1L); + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + BlockingDrainToQueue queue = + new BlockingDrainToQueue(config.getTs().getSave().getQueueCapacity()); + writer = new IoTDBTableTimeseriesWriter(pool, config, false, queue); + IoTDBTablePendingSave first = pendingSave(1L, "first"); + IoTDBTablePendingSave duplicateA = pendingSave(2L, "duplicate"); + IoTDBTablePendingSave duplicateB = pendingSave(2L, "duplicate"); + assertEquals(duplicateA.identity(), duplicateB.identity()); + assertFalse(duplicateA.future() == duplicateB.future()); + + writer.enqueue(first); + writer.enqueue(duplicateA); + writer.enqueue(duplicateB); + Thread workerThread = writer.workerThread(); + try { + workerThread.start(); + assertTrue( + queue.awaitDrainToBlocked(3, TimeUnit.SECONDS), + "worker should block after drainTo removes saves and before batch append"); + + writer.destroy(); + + assertFutureFailsWith(first.future(), IoTDBTableDaoShuttingDownException.class); + assertFutureFailsWith(duplicateA.future(), IoTDBTableDaoShuttingDownException.class); + assertFutureFailsWith(duplicateB.future(), IoTDBTableDaoShuttingDownException.class); + } finally { + queue.releaseDrainTo(); + } + workerThread.join(1_000L); + assertFalse(workerThread.isAlive(), "forced-stop worker should terminate"); + verify(session, never()).insert(any(Tablet.class)); + } + + @Test + void destroyForceStopsRetryBackoffPromptlyWithoutReplayingInsert() throws Exception { + IoTDBTableConfig config = config(1, 1000L, 100); + config.getTs().getSave().setRetryMaxAttempts(3); + config.getTs().getSave().setRetryInitialBackoffMs(60_000L); + config.getTs().getSave().setRetryMaxBackoffMs(60_000L); + config.getTs().getSave().setShutdownDrainTimeoutMs(1L); + TestContext context = newContext(config, true); + + CountDownLatch firstInsertFailed = new CountDownLatch(1); + AtomicInteger inserts = new AtomicInteger(); + doAnswer( + invocation -> { + if (inserts.incrementAndGet() == 1) { + firstInsertFailed.countDown(); + throw new IoTDBConnectionException("transient blip"); + } + return null; + }) + .when(context.session()) + .insert(any(Tablet.class)); + + ListenableFuture save = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + assertTrue( + firstInsertFailed.await(3, TimeUnit.SECONDS), + "worker should enter retry handling after the first insert fails"); + + long startedNanos = System.nanoTime(); + writer.destroy(); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos); + + assertTrue(elapsedMs < 2000L, "destroy should not wait out the retry backoff"); + assertFutureFailsWith(save, IoTDBTableDaoShuttingDownException.class); + Thread workerThread = writer.workerThread(); + workerThread.join(1_000L); + assertFalse(workerThread.isAlive(), "forced-stop worker should terminate"); + verify(context.session(), times(1)).insert(any(Tablet.class)); + } + + @Test + void destroyForceStopPreventsInsertAfterBlockedSessionAcquisitionReleases() throws Exception { + IoTDBTableConfig config = config(1, 1000L, 100); + config.getTs().getSave().setShutdownDrainTimeoutMs(1L); + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + CountDownLatch getSessionStarted = new CountDownLatch(1); + CountDownLatch releaseGetSession = new CountDownLatch(1); + when(pool.getSession()) + .thenAnswer( + invocation -> { + getSessionStarted.countDown(); + boolean released = false; + while (!released) { + try { + released = releaseGetSession.await(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // Model session acquisition that does not react to forced-stop interruption. + } + } + return session; + }); + writer = new IoTDBTableTimeseriesWriter(pool, config, true); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + daos.add(dao); + + ListenableFuture save = + dao.save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 0); + assertTrue( + getSessionStarted.await(3, TimeUnit.SECONDS), + "worker should block while acquiring a table session"); + + writer.destroy(); + releaseGetSession.countDown(); + Thread workerThread = writer.workerThread(); + workerThread.join(1_000L); + + assertFalse( + workerThread.isAlive(), "forced-stop worker should terminate after getSession returns"); + assertFutureFailsWith(save, IoTDBTableDaoShuttingDownException.class); + verify(session, never()).insert(any(Tablet.class)); + } + + @Test + void destroyCompletesEveryFutureUnderConcurrentSavesAndDestroy() throws Exception { + int saverThreads = 32; + IoTDBTableConfig config = config(saverThreads, 10000L, saverThreads); + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + PausingOfferQueue queue = new PausingOfferQueue(saverThreads, saverThreads); + writer = new IoTDBTableTimeseriesWriter(pool, config, false, queue); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + daos.add(dao); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(saverThreads); + List>> submitted = new ArrayList<>(saverThreads); + try { + for (int i = 0; i < saverThreads; i++) { + final int index = i; + submitted.add( + executor.submit( + () -> { + start.await(); + return dao.save( + TENANT_ID, + ENTITY_ID, + entry(index, "race-" + index, DataType.LONG, (long) index), + 0); + })); + } + + start.countDown(); + assertTrue(queue.awaitPausedOffers(3, TimeUnit.SECONDS)); + writer.destroy(); + queue.releaseOffers(); + + for (Future> submittedFuture : submitted) { + ListenableFuture saveFuture = submittedFuture.get(3, TimeUnit.SECONDS); + assertTrue(saveFuture.isDone()); + assertFutureFailsWith(saveFuture, IoTDBTableDaoShuttingDownException.class); + } + + IoTDBTableTimeseriesWriterStats stats = dao.stats(); + assertEquals(saverThreads, stats.enqueued()); + assertEquals(0, stats.flushed()); + assertEquals(0, stats.rejectsFull()); + assertEquals(saverThreads, stats.rejectsShutdown()); + assertEquals(0, stats.shutdownFailedPending()); + assertEquals( + stats.enqueued(), + stats.flushed() + + stats.rejectsFull() + + stats.rejectsShutdown() + + stats.shutdownFailedPending()); + verify(session, never()).insert(any(Tablet.class)); + } finally { + queue.releaseOffers(); + executor.shutdownNow(); + } + } + + @Test + void destroyTimeoutFailsActiveBatchWhileWorkerIsMidFlush() throws Exception { + IoTDBTableConfig config = config(1, 1000L, 100); + config.getTs().getSave().setShutdownDrainTimeoutMs(50L); + TestContext context = newContext(config, true); + CountDownLatch insertStarted = new CountDownLatch(1); + CountDownLatch releaseInsert = new CountDownLatch(1); + CountDownLatch insertReturned = new CountDownLatch(1); + doAnswer( + invocation -> { + insertStarted.countDown(); + boolean released = false; + while (!released) { + try { + released = releaseInsert.await(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // Keep the mock insert in flight until the test releases it. + } + } + insertReturned.countDown(); + return null; + }) + .when(context.session()) + .insert(any(Tablet.class)); + + ListenableFuture active = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 1L), 0); + assertTrue(insertStarted.await(3, TimeUnit.SECONDS)); + + long startedNanos = System.nanoTime(); + writer.destroy(); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos); + + assertTrue(elapsedMs < 2000L, "destroy should return within bounded shutdown waits"); + assertFutureFailsWith(active, IoTDBTableDaoShuttingDownException.class); + assertTrue(context.dao().stats().shutdownFailedPending() > 0); + releaseInsert.countDown(); + assertTrue(insertReturned.await(3, TimeUnit.SECONDS)); + } + + @Test + void save_failsAllEntryFuturesWhenBatchInsertFails() throws Exception { + TestContext context = newContext(config(3, 1000L, 100), true); + doThrow(new StatementExecutionException("batch rejected")) + .when(context.session()) + .insert(any(Tablet.class)); + + List> futures = + List.of( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "first", DataType.LONG, 1L), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "second", DataType.LONG, 2L), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(3L, "third", DataType.LONG, 3L), 0)); + + for (ListenableFuture future : futures) { + assertFutureFailsWith(future, StatementExecutionException.class); + } + assertEquals(1, context.dao().stats().flushFailures()); + } + + @Test + void save_failsBatchFuturesAndKeepsWorkerAliveWhenInsertThrowsError() throws Exception { + TestContext context = newContext(config(2, 1000L, 100), true); + AtomicInteger insertAttempts = new AtomicInteger(); + doAnswer( + invocation -> { + if (insertAttempts.getAndIncrement() == 0) { + throw new NoSuchMethodError("simulated"); + } + return null; + }) + .when(context.session()) + .insert(any(Tablet.class)); + + List> failedBatch = + List.of( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "first", DataType.LONG, 1L), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(2L, "second", DataType.LONG, 2L), 0)); + + for (ListenableFuture future : failedBatch) { + assertFutureFailsWith(future, NoSuchMethodError.class); + } + assertEquals(1, context.dao().stats().flushFailures()); + + List> recoveredBatch = + List.of( + context.dao().save(TENANT_ID, ENTITY_ID, entry(3L, "third", DataType.LONG, 3L), 0), + context.dao().save(TENANT_ID, ENTITY_ID, entry(4L, "fourth", DataType.LONG, 4L), 0)); + + for (ListenableFuture future : recoveredBatch) { + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + } + verify(context.session(), timeout(3000).times(2)).insert(any(Tablet.class)); + assertEquals(1, context.dao().stats().flushFailures()); + assertEquals(2, context.dao().stats().flushed()); + } + + @Test + void save_doesNotIssueAlterTableForPerCallTtl() throws Exception { + TestContext context = newContext(config(1, 1000L, 100), true); + + ListenableFuture future = + context + .dao() + .save(TENANT_ID, ENTITY_ID, entry(1L, "temperature", DataType.LONG, 42L), 86400L); + + assertEquals(1, future.get(3, TimeUnit.SECONDS)); + verify(context.session(), never()).executeNonQueryStatement(any(String.class)); + } + + @Test + void save_resolvesInBatchSameTimestampTypeChangeWithLastWriterWins() throws Exception { + TestContext context = newContext(config(2, 1000L, 100), true); + + ListenableFuture first = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "status", DataType.LONG, 1L), 0); + ListenableFuture second = + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "status", DataType.STRING, "ok"), 0); + + assertEquals(1, first.get(3, TimeUnit.SECONDS)); + assertEquals(1, second.get(3, TimeUnit.SECONDS)); + + Tablet tablet = insertedTablet(context.session(), 1); + assertEquals(1, tablet.getRowSize()); + assertRow(tablet, 0, 1L, "status", 7); + assertEquals(1, context.dao().stats().flushed()); + } + + @Test + void findAllAsync_rawBuildsHalfOpenSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = dataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 100L, 200L, 17, "asc"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + assertEquals(query.getId(), result.getQueryId()); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertEquals( + "SELECT time, bool_v, long_v, double_v, str_v, json_v FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 100 AND time < 200 " + + "ORDER BY time ASC LIMIT 17", + sql.getValue()); + } + + @Test + void findAllAsync_mapsFiveTypesToBasicTsKvEntry() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet allTypesDataSet = + dataSet( + row(10L, "bool_v", true), + row(11L, "long_v", 42L), + row(12L, "double_v", 3.5D), + row(13L, "str_v", "abc"), + row(14L, "json_v", "{\"v\":1}")); + when(context.session().executeQueryStatement(anyString())).thenReturn(allTypesDataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("sensor", 0L, 20L, 10, "DESC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(5, data.size()); + assertMappedEntry(data.get(0), 10L, "sensor", DataType.BOOLEAN, true); + assertMappedEntry(data.get(1), 11L, "sensor", DataType.LONG, 42L); + assertMappedEntry(data.get(2), 12L, "sensor", DataType.DOUBLE, 3.5D); + assertMappedEntry(data.get(3), 13L, "sensor", DataType.STRING, "abc"); + assertMappedEntry(data.get(4), 14L, "sensor", DataType.JSON, "{\"v\":1}"); + } + + @Test + void findAllAsync_preservesOneResultPerQueryAndQueryId() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet firstDataSet = dataSet(); + SessionDataSet secondDataSet = dataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(firstDataSet, secondDataSet); + + ReadTsKvQuery first = new BaseReadTsKvQuery("first", 10L, 20L, 1, "DESC"); + ReadTsKvQuery second = new BaseReadTsKvQuery("second", 20L, 30L, 1, "DESC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(first, second)) + .get(3, TimeUnit.SECONDS); + + assertEquals(2, results.size()); + assertEquals(first.getId(), results.get(0).getQueryId()); + assertEquals(second.getId(), results.get(1).getQueryId()); + } + + @Test + void findAllAsync_lastEntryTsIsMaxReturnedTs() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet dataSet = dataSet(row(30L, "long_v", 3L), row(10L, "long_v", 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("counter", 0L, 40L, 10, "DESC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + assertEquals(30L, result.getLastEntryTs()); + } + + @Test + void findAllAsync_emptyResult() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = dataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery noRows = new BaseReadTsKvQuery("empty", 123L, 456L, 10, "DESC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(noRows)) + .get(3, TimeUnit.SECONDS) + .get(0); + + assertEquals(List.of(), result.getData()); + assertEquals(123L, result.getLastEntryTs()); + + ReadTsKvQuery zeroLimit = new BaseReadTsKvQuery("empty", 123L, 456L, 0, "DESC"); + ReadTsKvQueryResult zeroLimitResult = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(zeroLimit)) + .get(3, TimeUnit.SECONDS) + .get(0); + assertEquals(List.of(), zeroLimitResult.getData()); + assertEquals(123L, zeroLimitResult.getLastEntryTs()); + verify(context.session(), times(1)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_escapesKeyAndRejectsBadOrder() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = dataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery escaped = new BaseReadTsKvQuery("a'b", 1L, 2L, 1, "desc"); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(escaped)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertTrue(sql.getValue().contains("key='a''b'")); + + ReadTsKvQuery badOrder = new BaseReadTsKvQuery("key", 1L, 2L, 1, "sideways"); + assertFutureFailsWith( + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(badOrder)), + IllegalArgumentException.class); + verify(context.session(), times(1)).executeQueryStatement(anyString()); + } + + @Test + void readDeleteAndSaveRejectBlankTelemetryKeysBeforeSqlOrEnqueue() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + assertFutureFailsWith( + context + .dao() + .findAllAsync( + TENANT_ID, ENTITY_ID, List.of(new BaseReadTsKvQuery(" ", 1L, 2L, 1, "DESC"))), + IllegalArgumentException.class); + assertFutureFailsWith( + context.dao().remove(TENANT_ID, ENTITY_ID, new BaseDeleteTsKvQuery("\t", 1L, 2L)), + IllegalArgumentException.class); + assertFutureFailsWith( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, " ", DataType.LONG, 1L), 0), + IllegalArgumentException.class); + + assertEquals(0, context.dao().stats().enqueued()); + verify(context.session(), never()).executeQueryStatement(anyString()); + verify(context.session(), never()).executeNonQueryStatement(anyString()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + @Test + void remove_buildsHalfOpenDeleteSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + assertNull( + context + .dao() + .remove(TENANT_ID, ENTITY_ID, new BaseDeleteTsKvQuery("temperature", 100L, 200L)) + .get(3, TimeUnit.SECONDS)); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeNonQueryStatement(sql.capture()); + assertEquals( + "DELETE FROM telemetry WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 100 AND time < 200", + sql.getValue()); + } + + @Test + void readExecutorDoesNotUseWriter() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = dataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 1L, 2L, 1, "DESC"); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + context + .dao() + .remove(TENANT_ID, ENTITY_ID, new BaseDeleteTsKvQuery("temperature", 1L, 2L)) + .get(3, TimeUnit.SECONDS); + + assertEquals(0, context.dao().stats().enqueued()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + @Test + void destroyCompletesRunningAndQueuedReadFutures() throws Exception { + IoTDBTableConfig config = config(10, 1000L, 100); + config.getTs().getSave().setShutdownDrainTimeoutMs(100L); + TestContext context = newContext(config, false); + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch releaseRead = new CountDownLatch(1); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + readStarted.countDown(); + releaseRead.await(5, TimeUnit.SECONDS); + return dataSet(); + }); + + ListenableFuture> running = + context + .dao() + .findAllAsync( + TENANT_ID, ENTITY_ID, List.of(new BaseReadTsKvQuery("running", 1L, 2L, 1, "DESC"))); + assertTrue(readStarted.await(3, TimeUnit.SECONDS)); + ListenableFuture> queued = + context + .dao() + .findAllAsync( + TENANT_ID, ENTITY_ID, List.of(new BaseReadTsKvQuery("queued", 1L, 2L, 1, "DESC"))); + + context.dao().destroy(); + + assertFutureDoneWithin(running, 3, TimeUnit.SECONDS); + assertFutureDoneWithin(queued, 3, TimeUnit.SECONDS); + assertFutureFailsWith(running, InterruptedException.class); + assertFutureFailsWith(queued, IoTDBTableDaoShuttingDownException.class); + releaseRead.countDown(); + } + + @Test + void findAllAsyncReturnsFailedFutureWhenReadQueueIsFull() throws Exception { + IoTDBTableConfig config = config(10, 1000L, 100); + config.getTs().getRead().setQueueCapacity(1); + TestContext context = newContext(config, false); + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch releaseRead = new CountDownLatch(1); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + readStarted.countDown(); + releaseRead.await(5, TimeUnit.SECONDS); + return dataSet(); + }); + + ListenableFuture> running = + context + .dao() + .findAllAsync( + TENANT_ID, ENTITY_ID, List.of(new BaseReadTsKvQuery("running", 1L, 2L, 1, "DESC"))); + assertTrue(readStarted.await(3, TimeUnit.SECONDS)); + ListenableFuture> queued = + context + .dao() + .findAllAsync( + TENANT_ID, ENTITY_ID, List.of(new BaseReadTsKvQuery("queued", 1L, 2L, 1, "DESC"))); + ListenableFuture> rejected = + context + .dao() + .findAllAsync( + TENANT_ID, + ENTITY_ID, + List.of(new BaseReadTsKvQuery("rejected", 1L, 2L, 1, "DESC"))); + + assertFutureFailsWith(rejected, IoTDBTableReadQueueFullException.class); + releaseRead.countDown(); + assertEquals(1, running.get(3, TimeUnit.SECONDS).size()); + assertEquals(1, queued.get(3, TimeUnit.SECONDS).size()); + } + + @Test + void readAndDeleteReturnFailedFuturesAfterDestroy() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + + context.dao().destroy(); + + assertFutureFailsWith( + context + .dao() + .findAllAsync( + TENANT_ID, + ENTITY_ID, + List.of(new BaseReadTsKvQuery("after-destroy", 1L, 2L, 1, "DESC"))), + IoTDBTableDaoShuttingDownException.class); + assertFutureFailsWith( + context + .dao() + .remove(TENANT_ID, ENTITY_ID, new BaseDeleteTsKvQuery("after-destroy", 1L, 2L)), + IoTDBTableDaoShuttingDownException.class); + verify(context.session(), never()).executeQueryStatement(anyString()); + verify(context.session(), never()).executeNonQueryStatement(anyString()); + } + + @Test + void saveReturnsFailedFutureAfterDestroy() throws Exception { + // Mirror the read/delete-after-destroy contract: once the DAO has been destroyed it must stop + // accepting writes too, returning a failed future rather than enqueueing into a draining + // writer. + TestContext context = newContext(config(10, 1000L, 100), false); + + context.dao().destroy(); + + assertFutureFailsWith( + context.dao().save(TENANT_ID, ENTITY_ID, entry(1L, "after-destroy", DataType.LONG, 1L), 0), + IoTDBTableDaoShuttingDownException.class); + assertEquals(0, context.dao().stats().enqueued()); + verify(context.session(), never()).insert(any(Tablet.class)); + } + + private TestContext newContext(IoTDBTableConfig config, boolean startWorker) + throws IoTDBConnectionException { + ITableSessionPool pool = mock(ITableSessionPool.class); + ITableSession session = mock(ITableSession.class); + when(pool.getSession()).thenReturn(session); + writer = new IoTDBTableTimeseriesWriter(pool, config, startWorker); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + daos.add(dao); + return new TestContext(dao, pool, session); + } + + private IoTDBTableTimeseriesWriter newWriterWithClock(IoTDBTableConfig config, AtomicLong clock) { + return new IoTDBTableTimeseriesWriter( + mock(ITableSessionPool.class), + config, + false, + new ArrayBlockingQueue<>(config.getTs().getSave().getQueueCapacity()), + clock::get); + } + + private IoTDBTableConfig config(int batchSize, long maxLingerMs, int queueCapacity) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.getTs().getSave().setBatchSize(batchSize); + config.getTs().getSave().setMaxLingerMs(maxLingerMs); + config.getTs().getSave().setQueueCapacity(queueCapacity); + config.getTs().getSave().setRetryInitialBackoffMs(1L); + config.getTs().getSave().setRetryMaxBackoffMs(1L); + config.getTs().getRead().setThreads(1); + return config; + } + + private Tablet insertedTablet(ITableSession session, int expectedInserts) + throws StatementExecutionException, IoTDBConnectionException { + ArgumentCaptor captor = ArgumentCaptor.forClass(Tablet.class); + verify(session, timeout(3000).times(expectedInserts)).insert(captor.capture()); + return captor.getAllValues().get(expectedInserts - 1); + } + + private void assertRow(Tablet tablet, int row, long ts, String key, int activeFieldIndex) { + assertEquals(ts, tablet.getTimestamp(row)); + assertEquals(EntityType.DEVICE.name(), String.valueOf(tablet.getValue(row, 0))); + assertEquals(TENANT_ID.getId().toString(), String.valueOf(tablet.getValue(row, 1))); + assertEquals(key, String.valueOf(tablet.getValue(row, 2))); + assertEquals(ENTITY_ID.getId().toString(), String.valueOf(tablet.getValue(row, 3))); + for (int column = 4; column <= 8; column++) { + if (column == activeFieldIndex) { + assertFalse(tablet.isNull(row, column), "expected active field at column " + column); + } else { + assertTrue(tablet.isNull(row, column), "expected null inactive field at column " + column); + } + } + } + + private List schemaNames(Tablet tablet) { + return tablet.getSchemas().stream().map(schema -> schema.getMeasurementName()).toList(); + } + + private List schemaTypes(Tablet tablet) { + return tablet.getSchemas().stream().map(schema -> schema.getType()).toList(); + } + + private Throwable assertFutureFailsWith( + ListenableFuture future, Class expectedCause) throws Exception { + ExecutionException exception = + assertThrows(ExecutionException.class, () -> future.get(3, TimeUnit.SECONDS)); + assertInstanceOf(expectedCause, exception.getCause()); + return exception.getCause(); + } + + private void assertFutureDoneWithin(ListenableFuture future, long timeout, TimeUnit unit) + throws Exception { + long deadline = System.nanoTime() + unit.toNanos(timeout); + while (!future.isDone() && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + assertTrue(future.isDone(), "future did not complete within " + timeout + " " + unit); + } + + private void assertMappedEntry( + TsKvEntry entry, long ts, String key, DataType dataType, Object value) { + assertInstanceOf(BasicTsKvEntry.class, entry); + assertEquals(ts, entry.getTs()); + assertEquals(key, entry.getKey()); + assertEquals(dataType, entry.getDataType()); + assertEquals(value, entry.getValue()); + assertEquals(String.valueOf(value), entry.getValueAsString()); + } + + private SessionDataSet dataSet(MockTelemetryRow... rows) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = mock(SessionDataSet.class); + SessionDataSet.DataIterator iterator = mock(SessionDataSet.DataIterator.class); + AtomicInteger index = new AtomicInteger(-1); + when(dataSet.iterator()).thenReturn(iterator); + when(iterator.next()).thenAnswer(invocation -> index.incrementAndGet() < rows.length); + when(iterator.isNull(anyString())) + .thenAnswer(invocation -> rows[index.get()].isNull(invocation.getArgument(0))); + when(iterator.getBoolean(anyString())).thenAnswer(invocation -> rows[index.get()].value()); + when(iterator.getLong(anyString())).thenAnswer(invocation -> rows[index.get()].value()); + when(iterator.getDouble(anyString())).thenAnswer(invocation -> rows[index.get()].value()); + when(iterator.getString(anyString())) + .thenAnswer(invocation -> String.valueOf(rows[index.get()].value())); + when(iterator.getTimestamp("time")) + .thenAnswer(invocation -> new Timestamp(rows[index.get()].ts())); + return dataSet; + } + + private IoTDBTablePendingSave pendingSave(long ts, String key) { + return new IoTDBTablePendingSave( + TENANT_ID.getId().toString(), + EntityType.DEVICE.name(), + ENTITY_ID.getId().toString(), + key, + ts, + DataType.LONG, + 1L, + 1); + } + + private MockTelemetryRow row(long ts, String column, Object value) { + return new MockTelemetryRow(ts, column, value); + } + + private TestTsKvEntry entry(long ts, String key, DataType dataType, Object value) { + return entry(ts, key, dataType, value, 1); + } + + private TestTsKvEntry entry( + long ts, String key, DataType dataType, Object value, int dataPoints) { + return new TestTsKvEntry(ts, key, dataType, value, dataPoints); + } + + private int tbDataPoints(String value) { + return Math.max(1, (value.length() + 511) / 512); + } + + private record MockTelemetryRow(long ts, String valueColumn, Object value) { + private boolean isNull(String column) { + return !valueColumn.equals(column); + } + } + + private record TestContext( + IoTDBTableTimeseriesDao dao, ITableSessionPool pool, ITableSession session) {} + + private record TestEntityId(UUID id, EntityType entityType) implements EntityId { + @Override + public UUID getId() { + return id; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + } + + private record TestTsKvEntry(long ts, String key, DataType dataType, Object value, int dataPoints) + implements TsKvEntry { + @Override + public long getTs() { + return ts; + } + + @Override + public String getKey() { + return key; + } + + @Override + public DataType getDataType() { + return dataType; + } + + @Override + public Optional getBooleanValue() { + return dataType == DataType.BOOLEAN ? Optional.of((Boolean) value) : Optional.empty(); + } + + @Override + public Optional getLongValue() { + return dataType == DataType.LONG ? Optional.of((Long) value) : Optional.empty(); + } + + @Override + public Optional getDoubleValue() { + return dataType == DataType.DOUBLE ? Optional.of((Double) value) : Optional.empty(); + } + + @Override + public Optional getStrValue() { + return dataType == DataType.STRING ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public Optional getJsonValue() { + return dataType == DataType.JSON ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public String getValueAsString() { + return String.valueOf(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public Long getVersion() { + return null; + } + + @Override + public int getDataPoints() { + return dataPoints; + } + } + + private static final class PausingOfferQueue extends ArrayBlockingQueue { + private final CountDownLatch pausedOffers; + private final CountDownLatch releaseOffers = new CountDownLatch(1); + + private PausingOfferQueue(int capacity, int pauseCount) { + super(capacity); + this.pausedOffers = new CountDownLatch(pauseCount); + } + + @Override + public boolean offer(IoTDBTablePendingSave pending) { + pausedOffers.countDown(); + try { + if (!releaseOffers.await(5, TimeUnit.SECONDS)) { + return false; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + return super.offer(pending); + } + + private boolean awaitPausedOffers(long timeout, TimeUnit unit) throws InterruptedException { + return pausedOffers.await(timeout, unit); + } + + private void releaseOffers() { + releaseOffers.countDown(); + } + } + + private static final class DequeuedBatchLatchQueue + extends ArrayBlockingQueue { + private final CountDownLatch lingerPollStarted = new CountDownLatch(1); + private final AtomicBoolean dequeued = new AtomicBoolean(false); + + private DequeuedBatchLatchQueue(int capacity) { + super(capacity); + } + + @Override + public IoTDBTablePendingSave poll(long timeout, TimeUnit unit) throws InterruptedException { + if (dequeued.get()) { + lingerPollStarted.countDown(); + } + IoTDBTablePendingSave pending = super.poll(timeout, unit); + if (pending != null) { + dequeued.set(true); + } + return pending; + } + + private boolean awaitLingerPollStarted(long timeout, TimeUnit unit) + throws InterruptedException { + return lingerPollStarted.await(timeout, unit); + } + } + + private static final class BlockingDrainToQueue + extends ArrayBlockingQueue { + private final CountDownLatch drainToBlocked = new CountDownLatch(1); + private final CountDownLatch releaseDrainTo = new CountDownLatch(1); + + private BlockingDrainToQueue(int capacity) { + super(capacity); + } + + @Override + public int drainTo(Collection collection, int maxElements) { + int drained = super.drainTo(collection, maxElements); + if (drained == 0) { + return 0; + } + drainToBlocked.countDown(); + boolean interrupted = false; + while (true) { + try { + if (releaseDrainTo.await(100, TimeUnit.MILLISECONDS)) { + break; + } + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return drained; + } + + private boolean awaitDrainToBlocked(long timeout, TimeUnit unit) throws InterruptedException { + return drainToBlocked.await(timeout, unit); + } + + private void releaseDrainTo() { + releaseDrainTo.countDown(); + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/StrategyFContractTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/StrategyFContractTest.java new file mode 100644 index 00000000..8a1882d1 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/StrategyFContractTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Strategy-F guard tests. + * + *

Strategy F compiles against a copy of the ThingsBoard SPI surface under {@code + * src/provided/java} and excludes {@code org/thingsboard/**} from the built jar so the stub classes + * never ship and never shadow the real ThingsBoard types at runtime. These tests prove both + * invariants stay true and break the build if they drift: + * + *

    + *
  1. Packaging: the built jar (when present) contains no {@code org/thingsboard/**} (nor + * {@code org/apache/commons/**}); the module pom must also declare the jar exclude so the + * guard holds even during the unit ({@code test}) phase before the jar exists. + *
  2. SPI drift: the {@link TimeseriesDao} SPI methods the DAO depends on still exist with + * the exact parameter and return types this module consumes, so a silent change to the + * provided surface fails the build. + *
+ * + *

The compile-only surface under {@code src/provided/java} was manually verified against + * ThingsBoard {@code v4.3.1.2} (commit {@code c37fb509}). A fully-automated check against the + * upstream artifact is not possible because ThingsBoard's {@code dao}/{@code common-data} modules + * are not published to Maven Central (the reason for Strategy F), so {@code + * timeseriesDaoSpiMethodsMatchExpectedSignatures} below pins the exact SPI signatures as explicit + * expectations to catch any accidental drift in the local surface. + */ +class StrategyFContractTest { + + @Test + void builtJarDoesNotShipThingsBoardOrCommonsClasses() throws IOException { + Path target = Path.of("target"); + List jars = new ArrayList<>(); + if (Files.isDirectory(target)) { + try (var stream = Files.newDirectoryStream(target, "*.jar")) { + stream.forEach(jars::add); + } + } + // During `mvn test` the jar is not built yet; the pom-exclude assertion below is the guard for + // that phase. During `mvn package`/`verify` the jar exists and must be clean. + for (Path jar : jars) { + try (JarFile jarFile = new JarFile(jar.toFile())) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + assertFalse( + name.startsWith("org/thingsboard/"), + "built jar " + jar + " must not ship the compile-only ThingsBoard surface: " + name); + assertFalse( + name.startsWith("org/apache/commons/"), + "built jar " + jar + " must not ship shaded Apache Commons classes: " + name); + } + } + } + } + + @Test + void modulePomDeclaresJarExcludes() throws IOException { + String pom = Files.readString(Path.of("pom.xml"), StandardCharsets.UTF_8); + assertTrue( + pom.contains("org/thingsboard/**"), + "module pom must keep the maven-jar-plugin exclude for org/thingsboard/** (Strategy F)"); + assertTrue( + pom.contains("org/apache/commons/**"), + "module pom must keep the maven-jar-plugin exclude for org/apache/commons/**"); + } + + @Test + void timeseriesDaoSpiMethodsMatchExpectedSignatures() throws NoSuchMethodException { + // The DAO implements TimeseriesDao; if the provided SPI surface drifts, these reflective + // lookups throw NoSuchMethodException and fail the build. Each lookup pins the exact parameter + // and return types this module consumes from the ThingsBoard SPI as an explicit expectation, + // so an accidental edit to the local compile-only surface (src/provided) breaks the build. + // Verified against ThingsBoard v4.3.1.2 (commit c37fb509). + assertSpiMethod( + "findAllAsync", + ListenableFuture.class, + new Class[] {TenantId.class, EntityId.class, List.class}); + assertSpiMethod( + "save", + ListenableFuture.class, + new Class[] {TenantId.class, EntityId.class, TsKvEntry.class, long.class}); + assertSpiMethod( + "savePartition", + ListenableFuture.class, + new Class[] {TenantId.class, EntityId.class, long.class, String.class}); + assertSpiMethod( + "remove", + ListenableFuture.class, + new Class[] {TenantId.class, EntityId.class, DeleteTsKvQuery.class}); + assertSpiMethod("cleanup", void.class, new Class[] {long.class}); + + // The DAO is a genuine TimeseriesDao implementation. + assertTrue(TimeseriesDao.class.isAssignableFrom(IoTDBTableTimeseriesDao.class)); + } + + /** + * Asserts that {@link TimeseriesDao} declares a method with exactly the given name, return type + * and ordered parameter types. {@code getMethod} already throws {@link NoSuchMethodException} if + * no method matches the exact parameter types; the extra assertions on the return type and the + * resolved parameter-type array make the pinned expectation explicit (and the failure message + * actionable) when the surface drifts. + */ + private static void assertSpiMethod( + String name, Class expectedReturn, Class[] expectedParams) + throws NoSuchMethodException { + Method method = TimeseriesDao.class.getMethod(name, expectedParams); + assertEquals( + expectedReturn, + method.getReturnType(), + "TimeseriesDao." + name + " return type drifted from the pinned SPI expectation"); + assertArrayEquals( + expectedParams, + method.getParameterTypes(), + "TimeseriesDao." + name + " parameter types drifted from the pinned SPI expectation"); + } + + @Test + void readTsKvQueryResultSurfaceUsedByDaoExists() throws NoSuchMethodException { + // The raw read path constructs ReadTsKvQueryResult(queryId, entries, lastTs) and reads back the + // queryId/lastEntryTs; ReadTsKvQuery exposes the query shape the DAO reads. Lock those down. + ReadTsKvQueryResult.class.getMethod("getQueryId"); + ReadTsKvQueryResult.class.getMethod("getLastEntryTs"); + ReadTsKvQueryResult.class.getMethod("getData"); + + ReadTsKvQuery.class.getMethod("getKey"); + ReadTsKvQuery.class.getMethod("getStartTs"); + ReadTsKvQuery.class.getMethod("getEndTs"); + ReadTsKvQuery.class.getMethod("getLimit"); + ReadTsKvQuery.class.getMethod("getOrder"); + ReadTsKvQuery.class.getMethod("getInterval"); + } +} diff --git a/pom.xml b/pom.xml index ca88f395..3d03679a 100644 --- a/pom.xml +++ b/pom.xml @@ -2072,6 +2072,21 @@ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + + + iotdb-thingsboard-table-jdk17 + + [17,) + + + iotdb-thingsboard-table + + with-springboot