[repo] Cluster Tests

.github/workflows/analysis.yml

@@ -92,14 +92,14 @@ jobs:
       - name: Build and install
         run: |
           find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
-          mvn -q --no-transfer-progress --batch-mode -DclickhouseVersion=$PREFERRED_LTS_VERSION \
-            -DskipTests install
+          mvn -q --no-transfer-progress --batch-mode -DskipTests install
       - name: Generate coverage report
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.PREFERRED_LTS_VERSION }}
         run: |
-          mvn -fn --no-transfer-progress --batch-mode -DclickhouseVersion=$PREFERRED_LTS_VERSION \
+          mvn -fn --no-transfer-progress --batch-mode \
             -Pcoverage verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=ClickHouse_clickhouse-java
         continue-on-error: true
       - name: Generate and post coverage report

.github/workflows/build.yml

@@ -54,7 +54,7 @@ jobs:
           java-version: 21
           cache: "maven"
       - name: Build and install libraries
-        run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs -DclickhouseVersion=$PREFERRED_LTS_VERSION install
+        run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs install
       - name: Copy Artifacts to Build dir
         run: |
           mkdir clickhouse-jdbc-artifacts
@@ -193,8 +193,10 @@ jobs:
       - name: Build and install libraries
         run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipTests=true -Dmaven.javadoc.skip=true install
       - name: Test Java client
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ matrix.clickhouse }}
         run: |
-          mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dmaven.javadoc.skip=true verify
+          mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -Dmaven.javadoc.skip=true verify
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: failure()
@@ -210,14 +212,16 @@ jobs:
     needs: compile
     strategy:
       matrix:
-        # most recent LTS releases as well as latest stable builds
-        # https://github.com/ClickHouse/ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease
-        clickhouse: ["cloud"]
-#        project: ["clickhouse-http-client", "clickhouse-jdbc", "client-v2", "jdbc-v2"]
         project: ["client-v2", "jdbc-v2"]
       fail-fast: false
     timeout-minutes: 20
-    name: ${{ matrix.project }} + CH ${{ matrix.clickhouse }}
+    name: ${{ matrix.project }} + ClickHouse Cloud
+    env:
+      TEST_CLICKHOUSE_CONTAINER: none
+      TEST_CLICKHOUSE_SECURE: "true"
+      TEST_CLICKHOUSE_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT_PROD }}
+      TEST_CLICKHOUSE_USER: default
+      TEST_CLICKHOUSE_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT_PROD }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -261,12 +265,9 @@ jobs:
           echo "TEST_DB_NAME=$RANDOM_DB" >> $GITHUB_ENV
           echo "Generated database name: $RANDOM_DB"
       - name: Create Temporary Database
-        env:
-          CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT_PROD }}
-          CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT_PROD }}
         run: |
           echo "Creating database: \`$TEST_DB_NAME\`"
-          curl --fail-with-body -s -u "default:$CLICKHOUSE_CLOUD_PASSWORD" "https://$CLICKHOUSE_CLOUD_HOST:8443" --data-binary "CREATE DATABASE IF NOT EXISTS $TEST_DB_NAME"
+          curl --fail-with-body -s -u "$TEST_CLICKHOUSE_USER:$TEST_CLICKHOUSE_PASSWORD" "https://$TEST_CLICKHOUSE_HOST:8443" --data-binary "CREATE DATABASE IF NOT EXISTS $TEST_DB_NAME"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -275,38 +276,95 @@ jobs:
         run: python3 -m pip install -r .github/scripts/requirements.txt
       - name: Test http client
         env:
-          CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT_PROD }}
-          CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT_PROD }}
           JWT_PKEY: ${{ secrets.JWT_PKEY }}
         run: |
           python3 .github/scripts/generate_jwt.py jwt.token
           JWT_TOKEN="$(cat jwt.token)"
           rm -f jwt.token
           export JWT_TOKEN
-          mvn --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=http -Dmaven.javadoc.skip=true verify
+          mvn --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -Dprotocol=http -Dmaven.javadoc.skip=true verify
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: result ${{ github.job }}_${{ matrix.project }}_${{ matrix.clickhouse }}
+          name: result ${{ github.job }}_${{ matrix.project }}
           path: |
             **/target/failsafe-reports
             **/target/surefire-reports
       - name: Cleanup Database Unconditionally
         if: always() # CRITICAL: This ensures the step runs regardless of previous success/failure
-        env:
-          CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT_PROD }}
-          CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT_PROD }}
         run: |
-          # Verify we actually generated a DB name to avoid dropping something accidentally 
+          # Verify we actually generated a DB name to avoid dropping something accidentally
           if [ -n "$TEST_DB_NAME" ]; then
-            DROP_STMT="DROP DATABASE IF EXISTS \`$TEST_DB_NAME\`" 
+            DROP_STMT="DROP DATABASE IF EXISTS \`$TEST_DB_NAME\`"
             echo "Cleaning up database: \`$TEST_DB_NAME\` with statement '$DROP_STMT'";
-            curl --fail-with-body -s -u "default:$CLICKHOUSE_CLOUD_PASSWORD" "https://$CLICKHOUSE_CLOUD_HOST:8443" --data-binary "$DROP_STMT";
+            curl --fail-with-body -s -u "$TEST_CLICKHOUSE_USER:$TEST_CLICKHOUSE_PASSWORD" "https://$TEST_CLICKHOUSE_HOST:8443" --data-binary "$DROP_STMT";
           else
             echo "No database name was generated; skipping cleanup.";
           fi;
 
+  test-with-cluster:
+    runs-on: ubuntu-latest
+    needs: compile
+    strategy:
+      matrix:
+        clickhouse: ["25.8", "26.3", "latest"]
+        project: ["client-v2", "jdbc-v2"]
+      fail-fast: false
+    timeout-minutes: 40
+    name: ${{ matrix.project }} + CH cluster ${{ matrix.clickhouse }}
+    env:
+      TEST_CLICKHOUSE_CONTAINER: cluster
+      TEST_CLICKHOUSE_IMAGE_VERSION: ${{ matrix.clickhouse }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Check out PR
+        run: |
+          git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \
+            origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr
+        if: github.event.inputs.pr != ''
+      - name: Install JDK 17 and Maven
+        uses: actions/setup-java@v4
+        with:
+          distribution: "temurin"
+          java-version: |
+            8
+            17
+          cache: "maven"
+      - name: Setup Toolchain
+        shell: bash
+        run: |
+          mkdir -p $HOME/.m2 \
+          && cat << EOF > $HOME/.m2/toolchains.xml
+          <?xml version="1.0" encoding="UTF8"?>
+          <toolchains>
+            <toolchain>
+              <type>jdk</type>
+              <provides>
+                <version>17</version>
+              </provides>
+              <configuration>
+                <jdkHome>${{ env.JAVA_HOME }}</jdkHome>
+              </configuration>
+            </toolchain>
+          </toolchains>
+          EOF
+      - name: Install Java client
+        run: mvn --also-make --batch-mode --no-transfer-progress -DskipTests=true -Dmaven.javadoc.skip=true install
+      - name: Test ${{ matrix.project }}
+        run: |
+          mvn --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -Dmaven.javadoc.skip=true verify
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: result ${{ github.job }}_${{ matrix.project }}_${{ matrix.clickhouse }}
+          path: |
+            **/target/failsafe-reports
+            **/target/surefire-reports
+          retention-days: 5
+
   test-jdbc-driver:
     runs-on: ubuntu-latest
     strategy:
@@ -355,8 +413,10 @@ jobs:
       - name: Install Java client
         run: mvn --also-make --batch-mode --no-transfer-progress --projects clickhouse-http-client,client-v2 -DskipTests=true -Dmaven.javadoc.skip=true install
       - name: Test JDBC driver
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ matrix.clickhouse }}
         run: |
-          mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,jdbc-v2 -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true verify
+          mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,jdbc-v2 -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true verify
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: failure()
@@ -415,8 +475,10 @@ jobs:
       - name: Install Java client
         run: mvn --also-make --no-transfer-progress --batch-mode --projects clickhouse-jdbc -DskipTests=true -Dmaven.javadoc.skip=true install
       - name: Test R2DBC ${{ matrix.r2dbc }}
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ matrix.clickhouse }}
         run: |
-          mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc -DclickhouseVersion=${{ matrix.clickhouse }} \
+          mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc \
             -D'r2dbc-spi.version=${{ matrix.r2dbc }}' -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true verify
       - name: Upload test results
         uses: actions/upload-artifact@v4
@@ -468,9 +530,12 @@ jobs:
 #      - name: Install Java client
 #        run: mvn --also-make --batch-mode --no-transfer-progress --projects clickhouse-http-client -Dj8 -DskipTests install
 #      - name: Test JDBC and R2DBC drivers
+#        env:
+#          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.PREFERRED_LTS_VERSION }}
+#          TEST_CLICKHOUSE_TIMEZONE: ${{ matrix.serverTz }}
 #        run: |
-#          mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,clickhouse-r2dbc -DclickhouseVersion=$PREFERRED_LTS_VERSION \
-#            -DclickhouseTimezone=${{ matrix.serverTz }} -Duser.timezone=${{ matrix.clientTz }} \
+#          mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,clickhouse-r2dbc \
+#            -Duser.timezone=${{ matrix.clientTz }} \
 #            -Dj8 -DskipUTs verify
 #      - name: Upload test results
 #        uses: actions/upload-artifact@v4

.github/workflows/jvm-compatibility.yml

@@ -122,9 +122,11 @@ jobs:
           tar -xzf \
             "$RUNNER_TEMP/clickhouse-maven-repository/compiled-tests.tar.gz"
       - name: Test all modules
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.PREFERRED_LTS_VERSION }}
         run: |
           mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 \
-            -Dmaven.wagon.rto=30000 -DclickhouseVersion=$PREFERRED_LTS_VERSION -Dj8 \
+            -Dmaven.wagon.rto=30000 -Dj8 \
             -Pcompile-java11,compile-java17 \
             org.apache.maven.plugins:maven-surefire-plugin:3.5.3:test \
             org.apache.maven.plugins:maven-failsafe-plugin:3.1.2:integration-test \

.github/workflows/nightly.yml

@@ -72,10 +72,12 @@ jobs:
           echo "Published snapshot version: $VERSION"
           echo "jdbc_version=$VERSION" >> "$GITHUB_OUTPUT"
       - name: Release Snapshot
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }}
         uses: samuelmeuli/action-maven-publish@v1
         with:
           maven_profiles: release
-          maven_args: -q --batch-mode -DclickhouseVersion=${{ env.CH_VERSION }}
+          maven_args: -q --batch-mode
           server_id: central
           gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }}

.github/workflows/release.yml

@@ -61,7 +61,8 @@ jobs:
           MAVEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USER }}
           MAVEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN }}
           MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
-        run: mvn -q --batch-mode -P release -DclickhouseVersion=${{ env.CH_VERSION }} clean deploy
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }}
+        run: mvn -q --batch-mode -P release clean deploy
       - name: Create Pre-release on Github
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test_head.yml

@@ -37,8 +37,10 @@ jobs:
           distribution: 'temurin'
           architecture: x64
       - name: Test Java client
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }}
         run: |
-          mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true clean install
+          mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -Dmaven.javadoc.skip=true clean install
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: failure()
@@ -68,10 +70,12 @@ jobs:
           architecture: x64
       - name: Build
         run: |
-          mvn --also-make --batch-mode --no-transfer-progress -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true -DskipTests=true clean install
+          mvn --also-make --batch-mode --no-transfer-progress -Dmaven.javadoc.skip=true -DskipTests=true clean install
       - name: Test R2DBC ${{ matrix.r2dbc }}
+        env:
+          TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }}
         run: |
-          mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc -DclickhouseVersion=$CH_VERSION \
+          mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc \
             -D'r2dbc-spi.version=${{ matrix.r2dbc }}' -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true clean install
       - name: Upload test results
         uses: actions/upload-artifact@v4

.gitignore

@@ -22,6 +22,9 @@
 
 # Eclipse TestNG plugin
 test-output/
+**/.apt_generated/
+**/.apt_generated_tests/
+
 
 # Intellij
 .idea/

CONTRIBUTING.md

@@ -139,36 +139,35 @@ Compile examples or packaging modules when your change affects examples, packagi
 
 Unit tests do not require a running ClickHouse server. Relevant unit tests should pass locally before a PR is submitted.
 
-Integration tests usually require [Docker](https://docs.docker.com/engine/install/). The Docker image defaults to `clickhouse/clickhouse-server`, and containers are created automatically by [testcontainers](https://www.testcontainers.org/). To test against a specific ClickHouse version, pass a Maven parameter such as:
+Integration tests usually require [Docker](https://docs.docker.com/engine/install/). The Docker image defaults to `clickhouse/clickhouse-server`, and containers are created automatically by [testcontainers](https://www.testcontainers.org/). Configuration is read from environment variables prefixed with `TEST_`:
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `TEST_CLICKHOUSE_CONTAINER` | `single` | `single` starts one server, `cluster` starts three servers behind nginx, `none` starts nothing |
+| `TEST_CLICKHOUSE_IMAGE_VERSION` | | Image tag used when a container is started, for example `24.8` (`TEST_CLICKHOUSE_VERSION` is also accepted) |
+| `TEST_CLICKHOUSE_IMAGE` | `clickhouse/clickhouse-server` | Image repository |
+| `TEST_CLICKHOUSE_HOST` | `localhost` | Host used when no container is started |
+| `TEST_CLICKHOUSE_USER` | `default` | User name |
+| `TEST_CLICKHOUSE_PASSWORD` | `test_default_password` | Password |
+| `TEST_CLICKHOUSE_SECURE` | `false` | `true` uses HTTPS on port 8443 (ClickHouse Cloud) |
+| `TEST_CLICKHOUSE_TIMEZONE` | `UTC` | Timezone inside a started container |
+| `TEST_CLICKHOUSE_ADDITIONAL_PACKAGES` | | Extra apt packages installed into the image |
+| `TEST_DB_NAME` | generated | Database name. An external name must start with `clickhouse_java_test_` |
+| `TEST_PROXY_ADDRESS` | | External proxy as `host:port` |
+| `TEST_PROXY_IMAGE` | `ghcr.io/shopify/toxiproxy:2.5.0` | Toxiproxy image |
+
+To test against a specific ClickHouse version:
 
 ```bash
-mvn -pl <module> test -DclickhouseVersion=23.3
+TEST_CLICKHOUSE_IMAGE_VERSION=24.8 mvn -pl <module> test
 ```
 
-If you do not want to use Docker, or you prefer to test against an existing server:
+To use an existing server instead of Docker, set `TEST_CLICKHOUSE_CONTAINER=none` and point `TEST_CLICKHOUSE_HOST` at that server (defaults to port `8123`). The server must accept `TEST_CLICKHOUSE_USER` / `TEST_CLICKHOUSE_PASSWORD` (defaults: `default` / `test_default_password`) with DDL and DML privileges. When containers are started by tests, ClickHouse HTTP port is `18123` to avoid colliding with any locally running ClickHouse instance. For ClickHouse Cloud also set `TEST_CLICKHOUSE_SECURE=true`.
 
-- make sure the server can be accessed with the default account, user `default` and no password, with both DDL and DML privileges
-- add the test server configuration files and expose all default ports:
-  - [ports.xml](clickhouse-client/src/test/resources/containers/clickhouse-server/config.d/ports.xml)
-  - [users.xml](clickhouse-client/src/test/resources/containers/clickhouse-server/users.d/users.xml)
-- make sure the ClickHouse binary, usually `/usr/bin/clickhouse`, is available in `PATH` for `clickhouse-cli-client` tests
-- put `test.properties` under either `~/.clickhouse` or the module's `src/test/resources`
-
-Example `test.properties`:
-
-```properties
-# ClickHouse server for integration tests
-clickhouseServer=x.x.x.x
-
-# Custom HTTP proxy for integration tests
-proxyAddress=<host>:<port>
-
-# Properties below are only useful for testcontainers
-#clickhouseVersion=latest
-#clickhouseTimezone=UTC
-#clickhouseImage=clickhouse/clickhouse-server
-#additionalPackages=
-#proxyImage=ghcr.io/shopify/toxiproxy:2.5.0
+```bash
+TEST_CLICKHOUSE_CONTAINER=none \
+TEST_CLICKHOUSE_HOST=127.0.0.1 \
+mvn -pl <module> test
 ```
 
 TBD: document a dockerized development environment for running the standard local test suite.