From eece162cc39f789d61e944f32935fbf275caa6f7 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Fri, 14 Aug 2026 14:51:30 +0000 Subject: [PATCH 01/33] test pipeline --- .../builds/linux-bq-driver-benchmark.sh | 588 ++++++++++++++++++ .../benchmark-bq-driver-unixODBC-ci.yaml | 35 ++ .../triggers/integration-bq-driver-dm-ci.yaml | 2 +- .../triggers/integration-bq-driver-dm-pr.yaml | 2 +- 4 files changed, 625 insertions(+), 2 deletions(-) create mode 100644 ci/cloudbuild/builds/linux-bq-driver-benchmark.sh create mode 100644 ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh new file mode 100644 index 0000000000..4b50524b28 --- /dev/null +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -0,0 +1,588 @@ +#!/bin/bash +# +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/bazel.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/cloudbuild/builds/lib/unit-tests.sh +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +# ------------------------------------------------------------ +# Vcpkg +# ------------------------------------------------------------ + +VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +export VCPKG_VERSION + +echo "Using VCPKG_VERSION=${VCPKG_VERSION}" + +export VCPKG_ROOT=/vcpkg + +git clone --branch "${VCPKG_VERSION}" \ + https://github.com/microsoft/vcpkg.git \ + "${VCPKG_ROOT}" + +cd "${VCPKG_ROOT}" +git checkout "${VCPKG_VERSION}" + +./bootstrap-vcpkg.sh -disableMetrics + +cd "${WORKSPACE_DIR}" + +# ------------------------------------------------------------ +# Unit tests +# ------------------------------------------------------------ + +mapfile -t args < <(bazel::common_args) +mapfile -t unit_tests_args < <(unit_tests::bazel_args) +mapfile -t secrets_bazel < <(secrets::bazel_args) + +io::run bazel test \ + "${args[@]}" \ + "${secrets_bazel[@]}" \ + "${unit_tests_args[@]}" \ + --test_tag_filters=unit-tests \ + ... + +# ------------------------------------------------------------ +# Common CMake arguments +# ------------------------------------------------------------ + +mapfile -t cmake_args < <(cmake::common_args) + +# ------------------------------------------------------------ +# Benchmark configuration +# +# This build is ONLY for performance_test. +# +# We intentionally use a separate cmake-out directory so that +# this benchmark build does not interfere with the normal build. +# ------------------------------------------------------------ + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" + +# ------------------------------------------------------------ +# Check unixODBC +# ------------------------------------------------------------ + +if command -v odbcinst &>/dev/null; then + export UNIXODBC_INSTALLED=true + echo "unixODBC is installed." +else + export UNIXODBC_INSTALLED=false + export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini + echo "unixODBC is not installed." +fi + +# ------------------------------------------------------------ +# Configure Google Driver + performance_test +# ------------------------------------------------------------ + +echo "============================================================" +echo "Configuring Google ODBC performance benchmark build" +echo "Build directory: ${BUILD_DIR}" +echo "============================================================" + +io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ + "${cmake_args[@]}" \ + -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON + +# ------------------------------------------------------------ +# Build ONLY performance_test +# ------------------------------------------------------------ + +echo "============================================================" +echo "Building Google Driver performance_test" +echo "============================================================" + +io::run cmake --build "${BUILD_DIR}" \ + --target performance_test \ + --config Release \ + --parallel "$(nproc)" + +# ------------------------------------------------------------ +# Locate performance_test +# ------------------------------------------------------------ + +PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + +if [[ ! -f "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found:" + echo " ${PERFORMANCE_TEST}" + echo + echo "Build directory contents:" + find "${BUILD_DIR}" -type f -name "performance_test*" -print + exit 1 +fi + +echo "Google performance_test:" +echo " ${PERFORMANCE_TEST}" + +# ------------------------------------------------------------ +# Copy certificates required by Google Driver +# ------------------------------------------------------------ + +GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" + +if [[ -f /opt/odbc-driver/roots.pem ]]; then + io::run cp \ + /opt/odbc-driver/roots.pem \ + "${GOOGLE_DRIVER_DIR}/roots.pem" +else + echo "ERROR: /opt/odbc-driver/roots.pem not found" + exit 1 +fi + +# ------------------------------------------------------------ +# Google Driver benchmark DSN +# ------------------------------------------------------------ + +export ODBCINI=/opt/odbc-driver/odbc.ini + +echo "Using ODBCINI=${ODBCINI}" +echo "Using Google DSN=${ODBC_TESTS_DSN}" + +# Verify Google driver exists. +GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "${GOOGLE_DRIVER}" ]]; then + echo "ERROR: Google driver not found:" + echo " ${GOOGLE_DRIVER}" + exit 1 +fi + +echo "Google driver:" +echo " ${GOOGLE_DRIVER}" + +# ------------------------------------------------------------ +# Run Google Driver benchmark +# +# One complete performance_test suite run. +# +# BENCHMARK_ITERATIONS can be supplied by the CI trigger. +# Default = 1. +# +# Every test in performance_test is compared, not just one test. +# ------------------------------------------------------------ + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" + +GOOGLE_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_bq.txt" + +echo "============================================================" +echo "Running Google Driver performance suite" +echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "============================================================" + +: > "${GOOGLE_RESULTS}" + +GOOGLE_EXIT_CODE=0 + +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== Google Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >> "${GOOGLE_RESULTS}" + + set +e + "${PERFORMANCE_TEST}" >> "${GOOGLE_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e + + if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then + echo "WARNING: Google Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} + fi +done + +if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then + echo "ERROR: Google Driver performance_test failed." + exit "${GOOGLE_EXIT_CODE}" +fi + +# ------------------------------------------------------------ +# Run Simba Driver performance suite +# +# Simba is already installed/configured by the Simba pipeline. +# +# We do NOT rebuild or download Simba here. +# ------------------------------------------------------------ + +export ODBC_TESTS_DSN="SampleDSN" + +SIMBA_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_core.txt" + +echo "============================================================" +echo "Running Simba Driver performance suite" +echo "DSN: ${ODBC_TESTS_DSN}" +echo "============================================================" + +: > "${SIMBA_RESULTS}" + +SIMBA_EXIT_CODE=0 + +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== Simba Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >> "${SIMBA_RESULTS}" + + set +e + "${PERFORMANCE_TEST}" >> "${SIMBA_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e + + if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then + echo "WARNING: Simba Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + SIMBA_EXIT_CODE=${RUN_EXIT_CODE} + fi +done + +if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then + echo "ERROR: Simba Driver performance_test failed." + exit "${SIMBA_EXIT_CODE}" +fi + +# ------------------------------------------------------------ +# Main branch Google benchmark +# +# Main is optional. If it has already been uploaded, download it +# and include it in the comparison. +# ------------------------------------------------------------ + +mkdir -p "${WORKSPACE_DIR}/benchmark_results" + +CURRENT_BRANCH="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ + sed 's/[^a-zA-Z0-9._-]/_/g') + +MAIN_RESULTS="${WORKSPACE_DIR}/benchmark_results/main_bq.txt" + +echo "============================================================" +echo "Downloading main branch Google benchmark baseline" +echo "============================================================" + +set +e + +gcloud storage cp \ + "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ + "${MAIN_RESULTS}" + +MAIN_DOWNLOAD_EXIT=$? + +set -e + +if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then + echo "WARNING: Main branch benchmark result not available." + rm -f "${MAIN_RESULTS}" +fi + +# ------------------------------------------------------------ +# Parse results and generate comparison table +# ------------------------------------------------------------ + +SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" + +python3 <<'PYTHON' +import os +import re + +SIMBA_FILE = "benchmark_results_current_core.txt" +GOOGLE_FILE = "benchmark_results_current_bq.txt" +MAIN_FILE = "benchmark_results/main_bq.txt" +OUTPUT_FILE = "benchmark_summary_table.txt" + + +def clean_test_name(name): + # GTest: + # + # [Instantiation/]TestSuite.TestCase[/Param] + # + # Keep TestCase so that the same test can be compared + # between Simba and Google drivers. + + if "." in name: + name = name.split(".", 1)[1] + + # Remove old HTAPI parameter suffix if present. + name = re.sub(r"/(?:With|Without)HTAPI$", "", name) + + return name + + +def parse_time_to_ms(value): + if not value: + return None + + value = value.strip() + + if value == "N/A": + return None + + match = re.match(r"^([\d.]+)\s*(\w+)$", value) + + if not match: + return None + + number = float(match.group(1)) + unit = match.group(2).lower() + + if unit == "ms": + return number + + if unit == "s": + return number * 1000 + + if unit == "us": + return number / 1000 + + if unit == "ns": + return number / 1_000_000 + + return number + + +def parse_gtest_output(filename): + samples = {} + + if not os.path.exists(filename): + return {} + + pattern = re.compile( + r"\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)" + ) + + with open(filename, "r", errors="replace") as file: + for line in file: + match = pattern.search(line) + + if not match: + continue + + test_name = clean_test_name(match.group(1)) + duration = parse_time_to_ms(match.group(2)) + + if duration is None: + continue + + samples.setdefault(test_name, []).append(duration) + + # Median per test across iterations. + results = {} + + for test_name, values in samples.items(): + values.sort() + + n = len(values) + + if n % 2: + median = values[n // 2] + else: + median = ( + values[n // 2 - 1] + + values[n // 2] + ) / 2.0 + + results[test_name] = median + + return results + + +def percentage(value, reference): + if value is None or reference is None or reference == 0: + return "N/A" + + pct = ((value - reference) / reference) * 100 + + return f"{pct:+.0f}%" + + +simba = parse_gtest_output(SIMBA_FILE) +google = parse_gtest_output(GOOGLE_FILE) +main_google = parse_gtest_output(MAIN_FILE) + +all_tests = sorted( + set(simba.keys()) | + set(google.keys()) | + set(main_google.keys()) +) + +rows = [] + +for test in all_tests: + + simba_ms = simba.get(test) + google_ms = google.get(test) + main_ms = main_google.get(test) + + google_vs_simba = percentage( + google_ms, + simba_ms + ) + + main_vs_google = percentage( + main_ms, + google_ms + ) + + simba_value = ( + f"{simba_ms:.0f}ms" + if simba_ms is not None + else "N/A" + ) + + google_value = ( + f"{google_ms:.0f}ms ({google_vs_simba})" + if google_ms is not None + else "N/A" + ) + + main_value = ( + f"{main_ms:.0f}ms ({main_vs_google})" + if main_ms is not None + else "N/A" + ) + + rows.append( + ( + test, + simba_value, + google_value, + main_value, + ) + ) + + +headers = [ + "Test Case", + "Simba Driver", + "Google Driver (Current)", + "Google Driver (Main)", +] + +widths = [] + +for index, header in enumerate(headers): + width = len(header) + + for row in rows: + width = max(width, len(row[index])) + + widths.append(width) + + +table = [] + +table.append( + "| " + + " | ".join( + header.ljust(widths[index]) + for index, header in enumerate(headers) + ) + + " |" +) + +table.append( + "| " + + " | ".join( + "-" * widths[index] + for index in range(len(headers)) + ) + + " |" +) + +for row in rows: + table.append( + "| " + + " | ".join( + row[index].ljust(widths[index]) + for index in range(len(headers)) + ) + + " |" + ) + + +description = ( + "Percentages in **Google Driver (Current)** are relative to " + "**Simba Driver**. Percentages in **Google Driver (Main)** " + "are relative to **Google Driver (Current)**. " + "Negative values mean the Google Driver is faster; " + "positive values mean it is slower." +) + +output = description + "\n\n" + "\n".join(table) + "\n" + +with open(OUTPUT_FILE, "w") as file: + file.write(output) + +print(output) +print(f"Compared {len(all_tests)} performance tests.") + +PYTHON + +# ------------------------------------------------------------ +# Upload current results +# ------------------------------------------------------------ + +RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" + +echo "============================================================" +echo "Uploading benchmark results" +echo "Destination: ${RESULTS_BUCKET}" +echo "============================================================" + +io::run gcloud storage cp \ + "${GOOGLE_RESULTS}" \ + "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" + +io::run gcloud storage cp \ + "${SIMBA_RESULTS}" \ + "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" + +io::run gcloud storage cp \ + "${SUMMARY_FILE}" \ + "${RESULTS_BUCKET}/benchmark_summary_table.txt" + +echo +echo "============================================================" +echo "Benchmark completed" +echo "============================================================" +echo +echo "Simba results:" +echo " ${SIMBA_RESULTS}" +echo +echo "Google results:" +echo " ${GOOGLE_RESULTS}" +echo +echo "Comparison:" +echo " ${SUMMARY_FILE}" +echo +echo "GCS:" +echo " ${RESULTS_BUCKET}/" +echo "============================================================" \ No newline at end of file diff --git a/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml b/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml new file mode 100644 index 0000000000..50d7eae851 --- /dev/null +++ b/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml + +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + branch: ^main$ + +name: benchmark-bq-driver-unixODBC-ci + +substitutions: + _BUILD_NAME: linux-bq-driver-benchmark + _DEPENDENCIES: 'BAZEL,unixODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DISTRO: ubuntu-20.04-release + _TRIGGER_TYPE: ci + +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS + +tags: +- ci +- benchmark \ No newline at end of file diff --git a/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml b/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml index ca18e08bc5..90f6919e3b 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml @@ -20,7 +20,7 @@ github: branch: ^main$ name: integration-bq-driver-dm-ci substitutions: - _BUILD_NAME: integration-production-bq-driver-dm + _BUILD_NAME: linux-bq-driver-benchmark _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci diff --git a/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml b/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml index d6b0f3c5e1..aef721458c 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml @@ -21,7 +21,7 @@ github: commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY name: integration-bq-driver-dm-pr substitutions: - _BUILD_NAME: integration-production-bq-driver-dm + _BUILD_NAME: linux-bq-driver-benchmark _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: pr From ecbc3942b682ba771c79ceb7c398add5c6ac99a1 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Fri, 14 Aug 2026 16:56:08 +0000 Subject: [PATCH 02/33] test --- .../integration-production-bq-driver-dm.sh | 563 +++++++++++++++++- .../triggers/integration-bq-driver-dm-ci.yaml | 2 +- .../triggers/integration-bq-driver-dm-pr.yaml | 2 +- 3 files changed, 534 insertions(+), 33 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..4b50524b28 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -27,61 +27,562 @@ source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) -# Export as env variable +# ------------------------------------------------------------ +# Vcpkg +# ------------------------------------------------------------ + VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION -echo "Using VCPKG_VERSION=$VCPKG_VERSION" -# Vcpkg install and configure +echo "Using VCPKG_VERSION=${VCPKG_VERSION}" + export VCPKG_ROOT=/vcpkg -git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" -cd "$VCPKG_ROOT" -git checkout "$VCPKG_VERSION" -# Bootstrap +git clone --branch "${VCPKG_VERSION}" \ + https://github.com/microsoft/vcpkg.git \ + "${VCPKG_ROOT}" + +cd "${VCPKG_ROOT}" +git checkout "${VCPKG_VERSION}" + ./bootstrap-vcpkg.sh -disableMetrics -cd "$WORKSPACE_DIR" -# This runs all the unit tests +cd "${WORKSPACE_DIR}" + +# ------------------------------------------------------------ +# Unit tests +# ------------------------------------------------------------ + mapfile -t args < <(bazel::common_args) mapfile -t unit_tests_args < <(unit_tests::bazel_args) mapfile -t secrets_bazel < <(secrets::bazel_args) -io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... +io::run bazel test \ + "${args[@]}" \ + "${secrets_bazel[@]}" \ + "${unit_tests_args[@]}" \ + --test_tag_filters=unit-tests \ + ... + +# ------------------------------------------------------------ +# Common CMake arguments +# ------------------------------------------------------------ -# Run the integration tests mapfile -t cmake_args < <(cmake::common_args) -BUILD_DIR="/opt/odbc-driver" -# This is the name of DSN set in odbc.ini -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +# ------------------------------------------------------------ +# Benchmark configuration +# +# This build is ONLY for performance_test. +# +# We intentionally use a separate cmake-out directory so that +# this benchmark build does not interfere with the normal build. +# ------------------------------------------------------------ + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" + +# ------------------------------------------------------------ +# Check unixODBC +# ------------------------------------------------------------ -# Check if unixODBC is installed if command -v odbcinst &>/dev/null; then - # unixODBC is installed, export environment variable export UNIXODBC_INSTALLED=true echo "unixODBC is installed." else - # unixODBC is not installed export UNIXODBC_INSTALLED=false export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini echo "unixODBC is not installed." fi -io::run cmake -B "$BUILD_DIR" \ +# ------------------------------------------------------------ +# Configure Google Driver + performance_test +# ------------------------------------------------------------ + +echo "============================================================" +echo "Configuring Google ODBC performance benchmark build" +echo "Build directory: ${BUILD_DIR}" +echo "============================================================" + +io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=ON \ - -DODBC_EXAMPLES=ON \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -io::run cmake --build cmake-out - -# Copy the roots.pem file to the .so directory to run test cases. -cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON + +# ------------------------------------------------------------ +# Build ONLY performance_test +# ------------------------------------------------------------ + +echo "============================================================" +echo "Building Google Driver performance_test" +echo "============================================================" + +io::run cmake --build "${BUILD_DIR}" \ + --target performance_test \ + --config Release \ + --parallel "$(nproc)" + +# ------------------------------------------------------------ +# Locate performance_test +# ------------------------------------------------------------ + +PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + +if [[ ! -f "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found:" + echo " ${PERFORMANCE_TEST}" + echo + echo "Build directory contents:" + find "${BUILD_DIR}" -type f -name "performance_test*" -print + exit 1 +fi + +echo "Google performance_test:" +echo " ${PERFORMANCE_TEST}" + +# ------------------------------------------------------------ +# Copy certificates required by Google Driver +# ------------------------------------------------------------ + +GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" + +if [[ -f /opt/odbc-driver/roots.pem ]]; then + io::run cp \ + /opt/odbc-driver/roots.pem \ + "${GOOGLE_DRIVER_DIR}/roots.pem" +else + echo "ERROR: /opt/odbc-driver/roots.pem not found" + exit 1 +fi + +# ------------------------------------------------------------ +# Google Driver benchmark DSN +# ------------------------------------------------------------ + +export ODBCINI=/opt/odbc-driver/odbc.ini + +echo "Using ODBCINI=${ODBCINI}" +echo "Using Google DSN=${ODBC_TESTS_DSN}" + +# Verify Google driver exists. +GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "${GOOGLE_DRIVER}" ]]; then + echo "ERROR: Google driver not found:" + echo " ${GOOGLE_DRIVER}" + exit 1 +fi + +echo "Google driver:" +echo " ${GOOGLE_DRIVER}" + +# ------------------------------------------------------------ +# Run Google Driver benchmark +# +# One complete performance_test suite run. +# +# BENCHMARK_ITERATIONS can be supplied by the CI trigger. +# Default = 1. +# +# Every test in performance_test is compared, not just one test. +# ------------------------------------------------------------ + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" + +GOOGLE_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_bq.txt" + +echo "============================================================" +echo "Running Google Driver performance suite" +echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "============================================================" + +: > "${GOOGLE_RESULTS}" + +GOOGLE_EXIT_CODE=0 + +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== Google Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >> "${GOOGLE_RESULTS}" + + set +e + "${PERFORMANCE_TEST}" >> "${GOOGLE_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e + + if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then + echo "WARNING: Google Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} + fi +done + +if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then + echo "ERROR: Google Driver performance_test failed." + exit "${GOOGLE_EXIT_CODE}" +fi + +# ------------------------------------------------------------ +# Run Simba Driver performance suite +# +# Simba is already installed/configured by the Simba pipeline. +# +# We do NOT rebuild or download Simba here. +# ------------------------------------------------------------ + +export ODBC_TESTS_DSN="SampleDSN" + +SIMBA_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_core.txt" + +echo "============================================================" +echo "Running Simba Driver performance suite" +echo "DSN: ${ODBC_TESTS_DSN}" +echo "============================================================" + +: > "${SIMBA_RESULTS}" + +SIMBA_EXIT_CODE=0 + +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== Simba Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >> "${SIMBA_RESULTS}" + + set +e + "${PERFORMANCE_TEST}" >> "${SIMBA_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e + + if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then + echo "WARNING: Simba Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + SIMBA_EXIT_CODE=${RUN_EXIT_CODE} + fi +done + +if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then + echo "ERROR: Simba Driver performance_test failed." + exit "${SIMBA_EXIT_CODE}" +fi + +# ------------------------------------------------------------ +# Main branch Google benchmark +# +# Main is optional. If it has already been uploaded, download it +# and include it in the comparison. +# ------------------------------------------------------------ + +mkdir -p "${WORKSPACE_DIR}/benchmark_results" + +CURRENT_BRANCH="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ + sed 's/[^a-zA-Z0-9._-]/_/g') + +MAIN_RESULTS="${WORKSPACE_DIR}/benchmark_results/main_bq.txt" + +echo "============================================================" +echo "Downloading main branch Google benchmark baseline" +echo "============================================================" + +set +e + +gcloud storage cp \ + "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ + "${MAIN_RESULTS}" + +MAIN_DOWNLOAD_EXIT=$? + +set -e + +if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then + echo "WARNING: Main branch benchmark result not available." + rm -f "${MAIN_RESULTS}" +fi + +# ------------------------------------------------------------ +# Parse results and generate comparison table +# ------------------------------------------------------------ + +SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" + +python3 <<'PYTHON' +import os +import re + +SIMBA_FILE = "benchmark_results_current_core.txt" +GOOGLE_FILE = "benchmark_results_current_bq.txt" +MAIN_FILE = "benchmark_results/main_bq.txt" +OUTPUT_FILE = "benchmark_summary_table.txt" + + +def clean_test_name(name): + # GTest: + # + # [Instantiation/]TestSuite.TestCase[/Param] + # + # Keep TestCase so that the same test can be compared + # between Simba and Google drivers. + + if "." in name: + name = name.split(".", 1)[1] + + # Remove old HTAPI parameter suffix if present. + name = re.sub(r"/(?:With|Without)HTAPI$", "", name) + + return name + + +def parse_time_to_ms(value): + if not value: + return None + + value = value.strip() + + if value == "N/A": + return None + + match = re.match(r"^([\d.]+)\s*(\w+)$", value) + + if not match: + return None + + number = float(match.group(1)) + unit = match.group(2).lower() + + if unit == "ms": + return number + + if unit == "s": + return number * 1000 + + if unit == "us": + return number / 1000 + + if unit == "ns": + return number / 1_000_000 + + return number + + +def parse_gtest_output(filename): + samples = {} + + if not os.path.exists(filename): + return {} + + pattern = re.compile( + r"\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)" + ) + + with open(filename, "r", errors="replace") as file: + for line in file: + match = pattern.search(line) + + if not match: + continue + + test_name = clean_test_name(match.group(1)) + duration = parse_time_to_ms(match.group(2)) + + if duration is None: + continue + + samples.setdefault(test_name, []).append(duration) + + # Median per test across iterations. + results = {} + + for test_name, values in samples.items(): + values.sort() + + n = len(values) + + if n % 2: + median = values[n // 2] + else: + median = ( + values[n // 2 - 1] + + values[n // 2] + ) / 2.0 + + results[test_name] = median + + return results + + +def percentage(value, reference): + if value is None or reference is None or reference == 0: + return "N/A" + + pct = ((value - reference) / reference) * 100 + + return f"{pct:+.0f}%" + + +simba = parse_gtest_output(SIMBA_FILE) +google = parse_gtest_output(GOOGLE_FILE) +main_google = parse_gtest_output(MAIN_FILE) + +all_tests = sorted( + set(simba.keys()) | + set(google.keys()) | + set(main_google.keys()) +) + +rows = [] + +for test in all_tests: + + simba_ms = simba.get(test) + google_ms = google.get(test) + main_ms = main_google.get(test) + + google_vs_simba = percentage( + google_ms, + simba_ms + ) + + main_vs_google = percentage( + main_ms, + google_ms + ) + + simba_value = ( + f"{simba_ms:.0f}ms" + if simba_ms is not None + else "N/A" + ) + + google_value = ( + f"{google_ms:.0f}ms ({google_vs_simba})" + if google_ms is not None + else "N/A" + ) + + main_value = ( + f"{main_ms:.0f}ms ({main_vs_google})" + if main_ms is not None + else "N/A" + ) + + rows.append( + ( + test, + simba_value, + google_value, + main_value, + ) + ) + + +headers = [ + "Test Case", + "Simba Driver", + "Google Driver (Current)", + "Google Driver (Main)", +] + +widths = [] + +for index, header in enumerate(headers): + width = len(header) + + for row in rows: + width = max(width, len(row[index])) + + widths.append(width) + + +table = [] + +table.append( + "| " + + " | ".join( + header.ljust(widths[index]) + for index, header in enumerate(headers) + ) + + " |" +) + +table.append( + "| " + + " | ".join( + "-" * widths[index] + for index in range(len(headers)) + ) + + " |" +) + +for row in rows: + table.append( + "| " + + " | ".join( + row[index].ljust(widths[index]) + for index in range(len(headers)) + ) + + " |" + ) + + +description = ( + "Percentages in **Google Driver (Current)** are relative to " + "**Simba Driver**. Percentages in **Google Driver (Main)** " + "are relative to **Google Driver (Current)**. " + "Negative values mean the Google Driver is faster; " + "positive values mean it is slower." +) + +output = description + "\n\n" + "\n".join(table) + "\n" + +with open(OUTPUT_FILE, "w") as file: + file.write(output) + +print(output) +print(f"Compared {len(all_tests)} performance tests.") + +PYTHON + +# ------------------------------------------------------------ +# Upload current results +# ------------------------------------------------------------ + +RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" + +echo "============================================================" +echo "Uploading benchmark results" +echo "Destination: ${RESULTS_BUCKET}" +echo "============================================================" + +io::run gcloud storage cp \ + "${GOOGLE_RESULTS}" \ + "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" + +io::run gcloud storage cp \ + "${SIMBA_RESULTS}" \ + "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" + +io::run gcloud storage cp \ + "${SUMMARY_FILE}" \ + "${RESULTS_BUCKET}/benchmark_summary_table.txt" + +echo +echo "============================================================" +echo "Benchmark completed" +echo "============================================================" +echo +echo "Simba results:" +echo " ${SIMBA_RESULTS}" +echo +echo "Google results:" +echo " ${GOOGLE_RESULTS}" +echo +echo "Comparison:" +echo " ${SUMMARY_FILE}" +echo +echo "GCS:" +echo " ${RESULTS_BUCKET}/" +echo "============================================================" \ No newline at end of file diff --git a/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml b/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml index 90f6919e3b..ca18e08bc5 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-dm-ci.yaml @@ -20,7 +20,7 @@ github: branch: ^main$ name: integration-bq-driver-dm-ci substitutions: - _BUILD_NAME: linux-bq-driver-benchmark + _BUILD_NAME: integration-production-bq-driver-dm _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci diff --git a/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml b/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml index aef721458c..d6b0f3c5e1 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-dm-pr.yaml @@ -21,7 +21,7 @@ github: commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY name: integration-bq-driver-dm-pr substitutions: - _BUILD_NAME: linux-bq-driver-benchmark + _BUILD_NAME: integration-production-bq-driver-dm _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: pr From 6cb2e39d574813a586bc4e4a1eb23bff1cef6e39 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Fri, 14 Aug 2026 17:23:37 +0000 Subject: [PATCH 03/33] test --- .../builds/integration-production-bq-driver-dm.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 4b50524b28..787147a5c9 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -171,18 +171,6 @@ export ODBCINI=/opt/odbc-driver/odbc.ini echo "Using ODBCINI=${ODBCINI}" echo "Using Google DSN=${ODBC_TESTS_DSN}" -# Verify Google driver exists. -GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -if [[ ! -f "${GOOGLE_DRIVER}" ]]; then - echo "ERROR: Google driver not found:" - echo " ${GOOGLE_DRIVER}" - exit 1 -fi - -echo "Google driver:" -echo " ${GOOGLE_DRIVER}" - # ------------------------------------------------------------ # Run Google Driver benchmark # From f10ecc162efddfb6bdfa96674e888fb3c9626a5c Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Mon, 17 Aug 2026 08:02:34 +0000 Subject: [PATCH 04/33] test --- .../integration-production-bq-driver-dm.sh | 543 ++++++++++++------ 1 file changed, 381 insertions(+), 162 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 787147a5c9..5c6fea7500 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -20,75 +20,70 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/bazel.sh -source module ci/cloudbuild/builds/lib/secrets.sh -source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh -WORKSPACE_DIR=$(pwd) +WORKSPACE_DIR="$(pwd)" -# ------------------------------------------------------------ -# Vcpkg -# ------------------------------------------------------------ +# ============================================================ +# VCPKG +# ============================================================ -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +VCPKG_VERSION="$(cat /tmp/vcpkg-version.txt)" export VCPKG_VERSION +echo "============================================================" +echo "VCPKG" +echo "============================================================" echo "Using VCPKG_VERSION=${VCPKG_VERSION}" export VCPKG_ROOT=/vcpkg -git clone --branch "${VCPKG_VERSION}" \ - https://github.com/microsoft/vcpkg.git \ - "${VCPKG_ROOT}" +if [[ ! -d "${VCPKG_ROOT}/.git" ]]; then + git clone \ + --branch "${VCPKG_VERSION}" \ + https://github.com/microsoft/vcpkg.git \ + "${VCPKG_ROOT}" +fi cd "${VCPKG_ROOT}" + git checkout "${VCPKG_VERSION}" ./bootstrap-vcpkg.sh -disableMetrics cd "${WORKSPACE_DIR}" -# ------------------------------------------------------------ -# Unit tests -# ------------------------------------------------------------ - -mapfile -t args < <(bazel::common_args) -mapfile -t unit_tests_args < <(unit_tests::bazel_args) -mapfile -t secrets_bazel < <(secrets::bazel_args) - -io::run bazel test \ - "${args[@]}" \ - "${secrets_bazel[@]}" \ - "${unit_tests_args[@]}" \ - --test_tag_filters=unit-tests \ - ... - -# ------------------------------------------------------------ -# Common CMake arguments -# ------------------------------------------------------------ +# ============================================================ +# COMMON CMAKE ARGUMENTS +# ============================================================ mapfile -t cmake_args < <(cmake::common_args) -# ------------------------------------------------------------ -# Benchmark configuration -# -# This build is ONLY for performance_test. -# -# We intentionally use a separate cmake-out directory so that -# this benchmark build does not interfere with the normal build. -# ------------------------------------------------------------ +# ============================================================ +# BENCHMARK CONFIGURATION +# ============================================================ BUILD_DIR="${WORKSPACE_DIR}/cmake-out" -export ODBC_TESTS_DSN="SampleDSN" +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" -# ------------------------------------------------------------ -# Check unixODBC -# ------------------------------------------------------------ +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" + +echo "============================================================" +echo "Benchmark configuration" +echo "============================================================" +echo "Workspace : ${WORKSPACE_DIR}" +echo "Build directory : ${BUILD_DIR}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Google DSN : ${ODBC_TESTS_DSN}" +echo "============================================================" + +# ============================================================ +# UNIXODBC +# ============================================================ if command -v odbcinst &>/dev/null; then export UNIXODBC_INSTALLED=true @@ -97,18 +92,27 @@ else export UNIXODBC_INSTALLED=false export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini echo "unixODBC is not installed." + echo "Using ODBCINSTINI=${ODBCINSTINI}" fi -# ------------------------------------------------------------ -# Configure Google Driver + performance_test -# ------------------------------------------------------------ +# ============================================================ +# CONFIGURE CMAKE +# +# IMPORTANT: +# This is a separate benchmark build. +# +# BUILD_PERFORMANCE_TEST_ONLY=ON means we are configuring the +# performance-test build rather than configuring/running the +# complete integration-test suite. +# ============================================================ echo "============================================================" -echo "Configuring Google ODBC performance benchmark build" -echo "Build directory: ${BUILD_DIR}" +echo "Configuring performance_test" echo "============================================================" -io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ +io::run cmake \ + -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ @@ -116,22 +120,23 @@ io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ -DBUILD_PERFORMANCE_TEST_ONLY=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON -# ------------------------------------------------------------ -# Build ONLY performance_test -# ------------------------------------------------------------ +# ============================================================ +# BUILD ONLY performance_test +# ============================================================ echo "============================================================" -echo "Building Google Driver performance_test" +echo "Building ONLY performance_test" echo "============================================================" -io::run cmake --build "${BUILD_DIR}" \ +io::run cmake \ + --build "${BUILD_DIR}" \ --target performance_test \ --config Release \ --parallel "$(nproc)" -# ------------------------------------------------------------ -# Locate performance_test -# ------------------------------------------------------------ +# ============================================================ +# LOCATE PERFORMANCE TEST +# ============================================================ PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" @@ -139,186 +144,333 @@ if [[ ! -f "${PERFORMANCE_TEST}" ]]; then echo "ERROR: performance_test was not found:" echo " ${PERFORMANCE_TEST}" echo - echo "Build directory contents:" + echo "Searching build directory:" find "${BUILD_DIR}" -type f -name "performance_test*" -print exit 1 fi -echo "Google performance_test:" -echo " ${PERFORMANCE_TEST}" +echo "============================================================" +echo "performance_test found" +echo "============================================================" +echo "${PERFORMANCE_TEST}" -# ------------------------------------------------------------ -# Copy certificates required by Google Driver -# ------------------------------------------------------------ +# ============================================================ +# GOOGLE DRIVER +# +# The Google driver is generated by the performance_test build. +# We do NOT download an existing Google .so. +# ============================================================ -GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" +GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" -if [[ -f /opt/odbc-driver/roots.pem ]]; then - io::run cp \ - /opt/odbc-driver/roots.pem \ - "${GOOGLE_DRIVER_DIR}/roots.pem" -else - echo "ERROR: /opt/odbc-driver/roots.pem not found" +if [[ ! -f "${GOOGLE_DRIVER}" ]]; then + echo "ERROR: Google driver was not generated by the performance build:" + echo " ${GOOGLE_DRIVER}" + echo + echo "Generated shared libraries:" + find "${BUILD_DIR}" -type f -name "*.so" -print exit 1 fi -# ------------------------------------------------------------ -# Google Driver benchmark DSN -# ------------------------------------------------------------ +echo "============================================================" +echo "Google driver generated" +echo "============================================================" +echo "${GOOGLE_DRIVER}" + +# ============================================================ +# CERTIFICATE +# ============================================================ -export ODBCINI=/opt/odbc-driver/odbc.ini +GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" + +if [[ ! -f /opt/odbc-driver/roots.pem ]]; then + echo "ERROR: /opt/odbc-driver/roots.pem was not found." + exit 1 +fi -echo "Using ODBCINI=${ODBCINI}" -echo "Using Google DSN=${ODBC_TESTS_DSN}" +io::run cp \ + /opt/odbc-driver/roots.pem \ + "${GOOGLE_DRIVER_DIR}/roots.pem" -# ------------------------------------------------------------ -# Run Google Driver benchmark +# ============================================================ +# ODBC CONFIGURATION +# +# The existing /opt/odbc-driver/odbc.ini must contain the DSNs +# used below. # -# One complete performance_test suite run. +# Google: +# SampleDSNGoogleDriver # -# BENCHMARK_ITERATIONS can be supplied by the CI trigger. -# Default = 1. +# Simba: +# SampleDSN # -# Every test in performance_test is compared, not just one test. -# ------------------------------------------------------------ +# If SampleDSN does not exist, Simba cannot be benchmarked. +# ============================================================ -BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" +ODBC_INI="/opt/odbc-driver/odbc.ini" + +if [[ ! -f "${ODBC_INI}" ]]; then + echo "ERROR: ODBC configuration was not found:" + echo " ${ODBC_INI}" + exit 1 +fi -GOOGLE_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_bq.txt" +export ODBCINI="${ODBC_INI}" echo "============================================================" -echo "Running Google Driver performance suite" -echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "ODBC configuration" echo "============================================================" +echo "ODBCINI=${ODBCINI}" -: > "${GOOGLE_RESULTS}" +echo +echo "Available DSNs:" +grep -E '^\[[^]]+\]$' "${ODBCINI}" || true + +# ============================================================ +# VERIFY GOOGLE DSN +# ============================================================ + +if ! grep -q '^\[SampleDSNGoogleDriver\]$' "${ODBC_INI}"; then + echo + echo "ERROR: Google DSN [SampleDSNGoogleDriver] was not found in:" + echo " ${ODBC_INI}" + exit 1 +fi + +# ============================================================ +# VERIFY SIMBA DSN +# ============================================================ + +if ! grep -q '^\[SampleDSN\]$' "${ODBC_INI}"; then + echo + echo "ERROR: Simba DSN [SampleDSN] was not found in:" + echo " ${ODBC_INI}" + echo + echo "The performance executable can be reused for Simba, but" + echo "the Simba DSN must be available in ODBCINI." + exit 1 +fi + +# ============================================================ +# RESULT FILES +# ============================================================ + +RESULT_DIR="${WORKSPACE_DIR}/benchmark_results" + +mkdir -p "${RESULT_DIR}" + +GOOGLE_RESULTS="${RESULT_DIR}/current_bq.txt" +SIMBA_RESULTS="${RESULT_DIR}/current_core.txt" + +MAIN_GOOGLE_RESULTS="${RESULT_DIR}/main_bq.txt" + +SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" + +rm -f \ + "${GOOGLE_RESULTS}" \ + "${SIMBA_RESULTS}" \ + "${MAIN_GOOGLE_RESULTS}" \ + "${SUMMARY_FILE}" + +# ============================================================ +# RUN GOOGLE DRIVER +# +# IMPORTANT: +# +# The executable is invoked directly. +# +# This executes the COMPLETE performance_test suite. +# +# There is NO ctest here. +# +# There is NO individual test filter. +# +# With BENCHMARK_ITERATIONS=1: +# +# performance_test +# +# runs exactly once, and every test registered in that executable +# is executed once. +# ============================================================ + +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + +echo "============================================================" +echo "Running Google Driver performance_test" +echo "============================================================" +echo "DSN : ${ODBC_TESTS_DSN}" +echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "Executable: ${PERFORMANCE_TEST}" +echo "============================================================" GOOGLE_EXIT_CODE=0 -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - echo "=== Google Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ +for ((i=1; i<=BENCHMARK_ITERATIONS; i++)); do + + echo "=== Google Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ >> "${GOOGLE_RESULTS}" set +e + "${PERFORMANCE_TEST}" >> "${GOOGLE_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Google Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + echo "WARNING: Google Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} fi + done if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then + echo echo "ERROR: Google Driver performance_test failed." + echo + echo "Last Google benchmark output:" + tail -100 "${GOOGLE_RESULTS}" || true exit "${GOOGLE_EXIT_CODE}" fi -# ------------------------------------------------------------ -# Run Simba Driver performance suite +echo "Google Driver performance suite completed successfully." + +# ============================================================ +# RUN SIMBA DRIVER +# +# SAME performance_test executable. # -# Simba is already installed/configured by the Simba pipeline. +# Only the DSN changes. # -# We do NOT rebuild or download Simba here. -# ------------------------------------------------------------ +# Therefore the exact same performance test cases are executed +# against Simba. +# ============================================================ export ODBC_TESTS_DSN="SampleDSN" -SIMBA_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_core.txt" - echo "============================================================" -echo "Running Simba Driver performance suite" -echo "DSN: ${ODBC_TESTS_DSN}" +echo "Running Simba Driver performance_test" +echo "============================================================" +echo "DSN : ${ODBC_TESTS_DSN}" +echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "Executable: ${PERFORMANCE_TEST}" echo "============================================================" - -: > "${SIMBA_RESULTS}" SIMBA_EXIT_CODE=0 -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - echo "=== Simba Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ +for ((i=1; i<=BENCHMARK_ITERATIONS; i++)); do + + echo "=== Simba Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ >> "${SIMBA_RESULTS}" set +e + "${PERFORMANCE_TEST}" >> "${SIMBA_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Simba Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + echo "WARNING: Simba Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" SIMBA_EXIT_CODE=${RUN_EXIT_CODE} fi + done if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then + echo echo "ERROR: Simba Driver performance_test failed." + echo + echo "Last Simba benchmark output:" + tail -100 "${SIMBA_RESULTS}" || true exit "${SIMBA_EXIT_CODE}" fi -# ------------------------------------------------------------ -# Main branch Google benchmark -# -# Main is optional. If it has already been uploaded, download it -# and include it in the comparison. -# ------------------------------------------------------------ +echo "Simba Driver performance suite completed successfully." -mkdir -p "${WORKSPACE_DIR}/benchmark_results" +# ============================================================ +# DOWNLOAD MAIN GOOGLE DRIVER RESULTS +# ============================================================ CURRENT_BRANCH="${BRANCH_NAME:-main}" -SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ - sed 's/[^a-zA-Z0-9._-]/_/g') - -MAIN_RESULTS="${WORKSPACE_DIR}/benchmark_results/main_bq.txt" +SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | \ + sed 's/[^a-zA-Z0-9._-]/_/g')" echo "============================================================" -echo "Downloading main branch Google benchmark baseline" +echo "Downloading Google Driver main baseline" echo "============================================================" set +e gcloud storage cp \ "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ - "${MAIN_RESULTS}" + "${MAIN_GOOGLE_RESULTS}" MAIN_DOWNLOAD_EXIT=$? set -e if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then - echo "WARNING: Main branch benchmark result not available." - rm -f "${MAIN_RESULTS}" + echo "WARNING: Main Google benchmark result is not available." + rm -f "${MAIN_GOOGLE_RESULTS}" +else + echo "Main Google benchmark downloaded successfully." fi -# ------------------------------------------------------------ -# Parse results and generate comparison table -# ------------------------------------------------------------ +# ============================================================ +# PARSE AND COMPARE +# +# Comparison: +# +# Simba Driver +# | +# | Google Current vs Simba +# v +# Google Driver Current +# | +# | Google Main vs Google Current +# v +# Google Driver Main +# +# Every test appearing in any of the three result files is +# included in the table. +# ============================================================ -SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" +echo "============================================================" +echo "Generating performance comparison" +echo "============================================================" python3 <<'PYTHON' import os import re -SIMBA_FILE = "benchmark_results_current_core.txt" -GOOGLE_FILE = "benchmark_results_current_bq.txt" + +SIMBA_FILE = "benchmark_results/current_core.txt" +GOOGLE_FILE = "benchmark_results/current_bq.txt" MAIN_FILE = "benchmark_results/main_bq.txt" + OUTPUT_FILE = "benchmark_summary_table.txt" def clean_test_name(name): - # GTest: - # - # [Instantiation/]TestSuite.TestCase[/Param] - # - # Keep TestCase so that the same test can be compared - # between Simba and Google drivers. + """ + GTest names can look like: + + TestSuite.TestCase + Instantiation/TestSuite.TestCase + TestSuite.TestCase/parameter + + Keep the TestCase portion so the same test can be compared + between the two drivers. + """ if "." in name: name = name.split(".", 1)[1] - # Remove old HTAPI parameter suffix if present. + # Remove old HTAPI parameter suffixes. name = re.sub(r"/(?:With|Without)HTAPI$", "", name) return name @@ -353,10 +505,20 @@ def parse_time_to_ms(value): if unit == "ns": return number / 1_000_000 - return number + return None def parse_gtest_output(filename): + """ + Parse: + + [ OK ] TestSuite.TestCase (123 ms) + + Multiple iterations are supported. + + The median is used if BENCHMARK_ITERATIONS > 1. + """ + samples = {} if not os.path.exists(filename): @@ -367,13 +529,16 @@ def parse_gtest_output(filename): ) with open(filename, "r", errors="replace") as file: + for line in file: + match = pattern.search(line) if not match: continue test_name = clean_test_name(match.group(1)) + duration = parse_time_to_ms(match.group(2)) if duration is None: @@ -381,15 +546,15 @@ def parse_gtest_output(filename): samples.setdefault(test_name, []).append(duration) - # Median per test across iterations. results = {} for test_name, values in samples.items(): + values.sort() n = len(values) - if n % 2: + if n % 2 == 1: median = values[n // 2] else: median = ( @@ -403,6 +568,7 @@ def parse_gtest_output(filename): def percentage(value, reference): + if value is None or reference is None or reference == 0: return "N/A" @@ -412,21 +578,28 @@ def percentage(value, reference): simba = parse_gtest_output(SIMBA_FILE) + google = parse_gtest_output(GOOGLE_FILE) + main_google = parse_gtest_output(MAIN_FILE) + all_tests = sorted( set(simba.keys()) | set(google.keys()) | set(main_google.keys()) ) + rows = [] + for test in all_tests: simba_ms = simba.get(test) + google_ms = google.get(test) + main_ms = main_google.get(test) google_vs_simba = percentage( @@ -439,23 +612,26 @@ for test in all_tests: google_ms ) - simba_value = ( - f"{simba_ms:.0f}ms" - if simba_ms is not None - else "N/A" - ) - - google_value = ( - f"{google_ms:.0f}ms ({google_vs_simba})" - if google_ms is not None - else "N/A" - ) + if simba_ms is None: + simba_value = "N/A" + else: + simba_value = f"{simba_ms:.0f}ms" + + if google_ms is None: + google_value = "N/A" + else: + google_value = ( + f"{google_ms:.0f}ms " + f"({google_vs_simba})" + ) - main_value = ( - f"{main_ms:.0f}ms ({main_vs_google})" - if main_ms is not None - else "N/A" - ) + if main_ms is None: + main_value = "N/A" + else: + main_value = ( + f"{main_ms:.0f}ms " + f"({main_vs_google})" + ) rows.append( ( @@ -474,19 +650,25 @@ headers = [ "Google Driver (Main)", ] + widths = [] for index, header in enumerate(headers): + width = len(header) for row in rows: - width = max(width, len(row[index])) + width = max( + width, + len(row[index]) + ) widths.append(width) table = [] + table.append( "| " + " | ".join( @@ -496,6 +678,7 @@ table.append( + " |" ) + table.append( "| " + " | ".join( @@ -505,7 +688,9 @@ table.append( + " |" ) + for row in rows: + table.append( "| " + " | ".join( @@ -520,29 +705,42 @@ description = ( "Percentages in **Google Driver (Current)** are relative to " "**Simba Driver**. Percentages in **Google Driver (Main)** " "are relative to **Google Driver (Current)**. " - "Negative values mean the Google Driver is faster; " - "positive values mean it is slower." + "Negative values indicate that the Google Driver is faster; " + "positive values indicate that it is slower." +) + + +output = ( + description + + "\n\n" + + "\n".join(table) + + "\n" ) -output = description + "\n\n" + "\n".join(table) + "\n" with open(OUTPUT_FILE, "w") as file: file.write(output) + print(output) -print(f"Compared {len(all_tests)} performance tests.") + +print( + f"Compared {len(all_tests)} performance test cases." +) PYTHON -# ------------------------------------------------------------ -# Upload current results -# ------------------------------------------------------------ +# ============================================================ +# UPLOAD RESULTS +# ============================================================ RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" echo "============================================================" echo "Uploading benchmark results" -echo "Destination: ${RESULTS_BUCKET}" +echo "============================================================" +echo "Destination:" +echo " ${RESULTS_BUCKET}" echo "============================================================" io::run gcloud storage cp \ @@ -557,20 +755,41 @@ io::run gcloud storage cp \ "${SUMMARY_FILE}" \ "${RESULTS_BUCKET}/benchmark_summary_table.txt" +# ============================================================ +# FINAL SUMMARY +# ============================================================ + echo echo "============================================================" -echo "Benchmark completed" +echo "BENCHMARK COMPLETED SUCCESSFULLY" echo "============================================================" + echo -echo "Simba results:" -echo " ${SIMBA_RESULTS}" +echo "Performance executable:" +echo " ${PERFORMANCE_TEST}" + +echo +echo "Google Driver:" +echo " ${GOOGLE_DRIVER}" + echo echo "Google results:" echo " ${GOOGLE_RESULTS}" + +echo +echo "Simba results:" +echo " ${SIMBA_RESULTS}" + echo echo "Comparison:" echo " ${SUMMARY_FILE}" + echo echo "GCS:" echo " ${RESULTS_BUCKET}/" + +echo +echo "All tests registered in performance_test were executed." +echo "Benchmark iterations: ${BENCHMARK_ITERATIONS}" + echo "============================================================" \ No newline at end of file From 44c8e13efb0ff4f64256e8e9c9c78f2361304d03 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Mon, 17 Aug 2026 10:59:12 +0000 Subject: [PATCH 05/33] test --- .../integration-production-bq-driver-dm.sh | 36 +++++++++++-------- .../builds/linux-bq-driver-benchmark.sh | 16 ++++----- .../benchmark-bq-driver-unixODBC-ci.yaml | 2 +- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 5c6fea7500..278a1b792f 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -110,15 +110,17 @@ echo "============================================================" echo "Configuring performance_test" echo "============================================================" -io::run cmake \ - -S "${WORKSPACE_DIR}" \ - -B "${BUILD_DIR}" \ +io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_PERFORMANCE_TEST_ONLY=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON + -DODBC_INTEGRATION_TESTING=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON \ + -DODBC_DEMO_TESTING=OFF \ + -DODBC_EXAMPLES=OFF \ + -DODBC_UNIT_TESTING=OFF \ + -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF # ============================================================ # BUILD ONLY performance_test @@ -128,8 +130,12 @@ echo "============================================================" echo "Building ONLY performance_test" echo "============================================================" -io::run cmake \ - --build "${BUILD_DIR}" \ +io::run cmake --build "${BUILD_DIR}" \ + --target google_cloud_odbc_bq_driver \ + --config Release \ + --parallel "$(nproc)" + +io::run cmake --build "${BUILD_DIR}" \ --target performance_test \ --config Release \ --parallel "$(nproc)" @@ -305,14 +311,14 @@ echo "============================================================" GOOGLE_EXIT_CODE=0 -for ((i=1; i<=BENCHMARK_ITERATIONS; i++)); do +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo "=== Google Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${GOOGLE_RESULTS}" + >>"${GOOGLE_RESULTS}" set +e - "${PERFORMANCE_TEST}" >> "${GOOGLE_RESULTS}" 2>&1 + "${PERFORMANCE_TEST}" >>"${GOOGLE_RESULTS}" 2>&1 RUN_EXIT_CODE=$? @@ -359,14 +365,14 @@ echo "============================================================" SIMBA_EXIT_CODE=0 -for ((i=1; i<=BENCHMARK_ITERATIONS; i++)); do +for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo "=== Simba Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${SIMBA_RESULTS}" + >>"${SIMBA_RESULTS}" set +e - "${PERFORMANCE_TEST}" >> "${SIMBA_RESULTS}" 2>&1 + "${PERFORMANCE_TEST}" >>"${SIMBA_RESULTS}" 2>&1 RUN_EXIT_CODE=$? @@ -396,7 +402,7 @@ echo "Simba Driver performance suite completed successfully." CURRENT_BRANCH="${BRANCH_NAME:-main}" -SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | \ +SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | sed 's/[^a-zA-Z0-9._-]/_/g')" echo "============================================================" @@ -792,4 +798,4 @@ echo echo "All tests registered in performance_test were executed." echo "Benchmark iterations: ${BENCHMARK_ITERATIONS}" -echo "============================================================" \ No newline at end of file +echo "============================================================" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 4b50524b28..9e212009b8 100644 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -203,16 +203,16 @@ echo "Running Google Driver performance suite" echo "Iterations: ${BENCHMARK_ITERATIONS}" echo "============================================================" -: > "${GOOGLE_RESULTS}" +: >"${GOOGLE_RESULTS}" GOOGLE_EXIT_CODE=0 for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo "=== Google Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${GOOGLE_RESULTS}" + >>"${GOOGLE_RESULTS}" set +e - "${PERFORMANCE_TEST}" >> "${GOOGLE_RESULTS}" 2>&1 + "${PERFORMANCE_TEST}" >>"${GOOGLE_RESULTS}" 2>&1 RUN_EXIT_CODE=$? set -e @@ -244,16 +244,16 @@ echo "Running Simba Driver performance suite" echo "DSN: ${ODBC_TESTS_DSN}" echo "============================================================" -: > "${SIMBA_RESULTS}" +: >"${SIMBA_RESULTS}" SIMBA_EXIT_CODE=0 for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo "=== Simba Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${SIMBA_RESULTS}" + >>"${SIMBA_RESULTS}" set +e - "${PERFORMANCE_TEST}" >> "${SIMBA_RESULTS}" 2>&1 + "${PERFORMANCE_TEST}" >>"${SIMBA_RESULTS}" 2>&1 RUN_EXIT_CODE=$? set -e @@ -279,7 +279,7 @@ mkdir -p "${WORKSPACE_DIR}/benchmark_results" CURRENT_BRANCH="${BRANCH_NAME:-main}" -SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ +SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | sed 's/[^a-zA-Z0-9._-]/_/g') MAIN_RESULTS="${WORKSPACE_DIR}/benchmark_results/main_bq.txt" @@ -585,4 +585,4 @@ echo " ${SUMMARY_FILE}" echo echo "GCS:" echo " ${RESULTS_BUCKET}/" -echo "============================================================" \ No newline at end of file +echo "============================================================" diff --git a/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml b/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml index 50d7eae851..582906248d 100644 --- a/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml +++ b/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml @@ -32,4 +32,4 @@ includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS tags: - ci -- benchmark \ No newline at end of file +- benchmark From f08dccfbdd08e2261531d3d2252a40e47fd35967 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Mon, 17 Aug 2026 11:35:20 +0000 Subject: [PATCH 06/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 278a1b792f..77f4ce82eb 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -213,7 +213,7 @@ io::run cp \ # If SampleDSN does not exist, Simba cannot be benchmarked. # ============================================================ -ODBC_INI="/opt/odbc-driver/odbc.ini" +ODBC_INI="opt/odbc-driver/googlebigqueryodbc/odbc.ini" if [[ ! -f "${ODBC_INI}" ]]; then echo "ERROR: ODBC configuration was not found:" From ad1755b579c8b743be5852c32d8523f0497fc5ca Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Mon, 17 Aug 2026 12:15:14 +0000 Subject: [PATCH 07/33] test --- .../integration-production-bq-driver-dm.sh | 2 +- ci/gha/builds/lib/odbc.ini | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 77f4ce82eb..278a1b792f 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -213,7 +213,7 @@ io::run cp \ # If SampleDSN does not exist, Simba cannot be benchmarked. # ============================================================ -ODBC_INI="opt/odbc-driver/googlebigqueryodbc/odbc.ini" +ODBC_INI="/opt/odbc-driver/odbc.ini" if [[ ! -f "${ODBC_INI}" ]]; then echo "ERROR: ODBC configuration was not found:" diff --git a/ci/gha/builds/lib/odbc.ini b/ci/gha/builds/lib/odbc.ini index 879d7ced8b..428bb3b316 100644 --- a/ci/gha/builds/lib/odbc.ini +++ b/ci/gha/builds/lib/odbc.ini @@ -8,6 +8,7 @@ TraceFile=/tmp/odbc.log [ODBC Data Sources] SampleDSNGoogleDriver=ODBC Driver for BigQuery ODBCTransactionsTestsDSN=ODBC Driver for BigQuery +SampleDSN = ODBC Driver for BigQuery [SampleDSNGoogleDriver] @@ -156,3 +157,81 @@ LargeResultsTempTableExpirationTime=3600000 # given. Result pages have at most as many rows as are specified by the RowsFetchedPerBlock # connection property. #HTAPI_MinActivationRatio=3 + +[SampleDSN] + +PropertyWithoutValue + +Value with equals=I=am=a=value + +# Description: DSN Description. +# This key is not necessary and is only to give a description of the data source. + Description = ODBC Driver for BigQuery + +# Driver: The location where the ODBC driver is installed to. + Driver = /opt/odbc-driver/googlebigqueryodbc/lib/libgooglebigqueryodbc_sb64.so + +# These values can be set here, or on the connection string. +# Catalog: The catalog to connect to. This is a required setting. +Catalog=bigquery-devtools-drivers + +# SQLDialect: The SQL Dialect to use. There are two SQL dialects: +# 0 = BigQuery Legacy SQL +# 1 = BigQuery Standard SQL (SQL 11) +SQLDialect=1 + +# OAuth Mechanism: The OAuth mechanism to use. There are two choices: +# 0 = Service Authentication +# 1 = User Authentication +# +# This is a required setting. +OAuthMechanism=0 + +# RefreshToken: The Refresh Token used. This can be generated from the Windows connection dialog. +# It can also be generated by executing the following steps: +# 1. Get an Authentication by logging into Google from the following URL: +# https://accounts.google.com/o/oauth2/auth?scope=https://www.googleapis.com/auth/bigquery&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=977385342095.apps.googleusercontent.com&hl=en&from_login=1&as=76356ac9e8ce640b&pli=1&authuser=0 +# 2. Run the get_refresh_token.sh shell script and pass in the Authentication Token received in step 1. +# 3. Copy the Refresh Token (the text on the right-side of the colon, without the trailing or leading spaces) from the output of the script. +# This is a required setting. +# RefreshToken= + +# Email: For Service Authentication, this is a required setting. It is your GENERATED service account email (not a typical Gmail account). +# It is unique and associated with at least one public/private key pair. +Email=bq-devtools-drivers-test@bigquery-devtools-drivers.iam.gserviceaccount.com + +# KeyFile Path: For Service Authentication, this is a required setting. This is the path to the stored keyfile (.p12). +KeyFilePath=/opt/odbc-driver/connection/key.json + +# Used to specify the full path of the PEM formatted file containing trusted SSL CA certificates. +# If an empty string is passed in for the configuration, the driver expects the trusted SSL CA +# certificates can be found in the file named cacerts.pem located in the same directory as the +# driver's shared library. +#TrustedCerts= + +# AllowLargeResults: When set to 1, the driver allows for result sets in responses to be larger than 128 MB. +AllowLargeResults=0 + +# LargeResultsDataSetId: DatasetId to store temporary tables created. This is a required setting if AllowLargeResults is set to 1. +LargeResultsDataSetId=_bqodbc_temp_tables + +# LargeResultsTempTableExpirationTime: Time in milliseconds before the temporary tables created expire. This is a required setting if AllowLargeResults is set to 1. +LargeResultsTempTableExpirationTime=3600000 + +# EnableHTAPI: This set to 1 by default. +#EnableHTAPI=1 + +# HTAPI_MinResultsSize: An integer representing the minimum required number of rows in the results of +# a query to enable the High-Throughput API. This means that when the results of a query are +# gathered, they must have at least this many rows or the High-Throughput API will not be used to +# retrieve them. +#HTAPI_MinResultsSize=1000 + +# HTAPI_MinActivationRatio: An integer representing the minimum required ratio of rows to block size +# to enable the High-Throughput API. This means that when the results from a query are gathered, +# if there are not at least N times as many rows in the results as there are in the first page of +# results, then the High-Throughput API will not be used to retrieve them, where N is the ratio +# given. Result pages have at most as many rows as are specified by the RowsFetchedPerBlock +# connection property. +#HTAPI_MinActivationRatio=3 + From 699bc47ccccf4343e28d605ff54ce5aa64fde953 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Mon, 17 Aug 2026 13:56:25 +0000 Subject: [PATCH 08/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 278a1b792f..abd984df0c 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -114,11 +114,10 @@ io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=Release \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=OFF \ - -DODBC_EXAMPLES=OFF \ + -DODBC_DEMO_TESTING=ON \ + -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF From 60e90b417b377dfef5dccd8495717ba4f56cc1eb Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 05:25:20 +0000 Subject: [PATCH 09/33] test --- .../builds/linux-bq-driver-benchmark.sh | 548 ++++++++++++------ ci/gha/builds/lib/odbc.ini | 1 - 2 files changed, 380 insertions(+), 169 deletions(-) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 9e212009b8..abd984df0c 100644 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -20,75 +20,70 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/bazel.sh -source module ci/cloudbuild/builds/lib/secrets.sh -source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh -WORKSPACE_DIR=$(pwd) +WORKSPACE_DIR="$(pwd)" -# ------------------------------------------------------------ -# Vcpkg -# ------------------------------------------------------------ +# ============================================================ +# VCPKG +# ============================================================ -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +VCPKG_VERSION="$(cat /tmp/vcpkg-version.txt)" export VCPKG_VERSION +echo "============================================================" +echo "VCPKG" +echo "============================================================" echo "Using VCPKG_VERSION=${VCPKG_VERSION}" export VCPKG_ROOT=/vcpkg -git clone --branch "${VCPKG_VERSION}" \ - https://github.com/microsoft/vcpkg.git \ - "${VCPKG_ROOT}" +if [[ ! -d "${VCPKG_ROOT}/.git" ]]; then + git clone \ + --branch "${VCPKG_VERSION}" \ + https://github.com/microsoft/vcpkg.git \ + "${VCPKG_ROOT}" +fi cd "${VCPKG_ROOT}" + git checkout "${VCPKG_VERSION}" ./bootstrap-vcpkg.sh -disableMetrics cd "${WORKSPACE_DIR}" -# ------------------------------------------------------------ -# Unit tests -# ------------------------------------------------------------ - -mapfile -t args < <(bazel::common_args) -mapfile -t unit_tests_args < <(unit_tests::bazel_args) -mapfile -t secrets_bazel < <(secrets::bazel_args) - -io::run bazel test \ - "${args[@]}" \ - "${secrets_bazel[@]}" \ - "${unit_tests_args[@]}" \ - --test_tag_filters=unit-tests \ - ... - -# ------------------------------------------------------------ -# Common CMake arguments -# ------------------------------------------------------------ +# ============================================================ +# COMMON CMAKE ARGUMENTS +# ============================================================ mapfile -t cmake_args < <(cmake::common_args) -# ------------------------------------------------------------ -# Benchmark configuration -# -# This build is ONLY for performance_test. -# -# We intentionally use a separate cmake-out directory so that -# this benchmark build does not interfere with the normal build. -# ------------------------------------------------------------ +# ============================================================ +# BENCHMARK CONFIGURATION +# ============================================================ BUILD_DIR="${WORKSPACE_DIR}/cmake-out" -export ODBC_TESTS_DSN="SampleDSN" +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" -# ------------------------------------------------------------ -# Check unixODBC -# ------------------------------------------------------------ +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" + +echo "============================================================" +echo "Benchmark configuration" +echo "============================================================" +echo "Workspace : ${WORKSPACE_DIR}" +echo "Build directory : ${BUILD_DIR}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Google DSN : ${ODBC_TESTS_DSN}" +echo "============================================================" + +# ============================================================ +# UNIXODBC +# ============================================================ if command -v odbcinst &>/dev/null; then export UNIXODBC_INSTALLED=true @@ -97,41 +92,56 @@ else export UNIXODBC_INSTALLED=false export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini echo "unixODBC is not installed." + echo "Using ODBCINSTINI=${ODBCINSTINI}" fi -# ------------------------------------------------------------ -# Configure Google Driver + performance_test -# ------------------------------------------------------------ +# ============================================================ +# CONFIGURE CMAKE +# +# IMPORTANT: +# This is a separate benchmark build. +# +# BUILD_PERFORMANCE_TEST_ONLY=ON means we are configuring the +# performance-test build rather than configuring/running the +# complete integration-test suite. +# ============================================================ echo "============================================================" -echo "Configuring Google ODBC performance benchmark build" -echo "Build directory: ${BUILD_DIR}" +echo "Configuring performance_test" echo "============================================================" io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_PERFORMANCE_TEST_ONLY=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON + -DODBC_INTEGRATION_TESTING=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON \ + -DODBC_DEMO_TESTING=ON \ + -DODBC_EXAMPLES=ON \ + -DODBC_UNIT_TESTING=OFF \ + -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -# ------------------------------------------------------------ -# Build ONLY performance_test -# ------------------------------------------------------------ +# ============================================================ +# BUILD ONLY performance_test +# ============================================================ echo "============================================================" -echo "Building Google Driver performance_test" +echo "Building ONLY performance_test" echo "============================================================" +io::run cmake --build "${BUILD_DIR}" \ + --target google_cloud_odbc_bq_driver \ + --config Release \ + --parallel "$(nproc)" + io::run cmake --build "${BUILD_DIR}" \ --target performance_test \ --config Release \ --parallel "$(nproc)" -# ------------------------------------------------------------ -# Locate performance_test -# ------------------------------------------------------------ +# ============================================================ +# LOCATE PERFORMANCE TEST +# ============================================================ PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" @@ -139,198 +149,333 @@ if [[ ! -f "${PERFORMANCE_TEST}" ]]; then echo "ERROR: performance_test was not found:" echo " ${PERFORMANCE_TEST}" echo - echo "Build directory contents:" + echo "Searching build directory:" find "${BUILD_DIR}" -type f -name "performance_test*" -print exit 1 fi -echo "Google performance_test:" -echo " ${PERFORMANCE_TEST}" +echo "============================================================" +echo "performance_test found" +echo "============================================================" +echo "${PERFORMANCE_TEST}" -# ------------------------------------------------------------ -# Copy certificates required by Google Driver -# ------------------------------------------------------------ +# ============================================================ +# GOOGLE DRIVER +# +# The Google driver is generated by the performance_test build. +# We do NOT download an existing Google .so. +# ============================================================ + +GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "${GOOGLE_DRIVER}" ]]; then + echo "ERROR: Google driver was not generated by the performance build:" + echo " ${GOOGLE_DRIVER}" + echo + echo "Generated shared libraries:" + find "${BUILD_DIR}" -type f -name "*.so" -print + exit 1 +fi + +echo "============================================================" +echo "Google driver generated" +echo "============================================================" +echo "${GOOGLE_DRIVER}" + +# ============================================================ +# CERTIFICATE +# ============================================================ GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" -if [[ -f /opt/odbc-driver/roots.pem ]]; then - io::run cp \ - /opt/odbc-driver/roots.pem \ - "${GOOGLE_DRIVER_DIR}/roots.pem" -else - echo "ERROR: /opt/odbc-driver/roots.pem not found" +if [[ ! -f /opt/odbc-driver/roots.pem ]]; then + echo "ERROR: /opt/odbc-driver/roots.pem was not found." + exit 1 +fi + +io::run cp \ + /opt/odbc-driver/roots.pem \ + "${GOOGLE_DRIVER_DIR}/roots.pem" + +# ============================================================ +# ODBC CONFIGURATION +# +# The existing /opt/odbc-driver/odbc.ini must contain the DSNs +# used below. +# +# Google: +# SampleDSNGoogleDriver +# +# Simba: +# SampleDSN +# +# If SampleDSN does not exist, Simba cannot be benchmarked. +# ============================================================ + +ODBC_INI="/opt/odbc-driver/odbc.ini" + +if [[ ! -f "${ODBC_INI}" ]]; then + echo "ERROR: ODBC configuration was not found:" + echo " ${ODBC_INI}" exit 1 fi -# ------------------------------------------------------------ -# Google Driver benchmark DSN -# ------------------------------------------------------------ +export ODBCINI="${ODBC_INI}" -export ODBCINI=/opt/odbc-driver/odbc.ini +echo "============================================================" +echo "ODBC configuration" +echo "============================================================" +echo "ODBCINI=${ODBCINI}" -echo "Using ODBCINI=${ODBCINI}" -echo "Using Google DSN=${ODBC_TESTS_DSN}" +echo +echo "Available DSNs:" +grep -E '^\[[^]]+\]$' "${ODBCINI}" || true -# Verify Google driver exists. -GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +# ============================================================ +# VERIFY GOOGLE DSN +# ============================================================ -if [[ ! -f "${GOOGLE_DRIVER}" ]]; then - echo "ERROR: Google driver not found:" - echo " ${GOOGLE_DRIVER}" +if ! grep -q '^\[SampleDSNGoogleDriver\]$' "${ODBC_INI}"; then + echo + echo "ERROR: Google DSN [SampleDSNGoogleDriver] was not found in:" + echo " ${ODBC_INI}" exit 1 fi -echo "Google driver:" -echo " ${GOOGLE_DRIVER}" +# ============================================================ +# VERIFY SIMBA DSN +# ============================================================ + +if ! grep -q '^\[SampleDSN\]$' "${ODBC_INI}"; then + echo + echo "ERROR: Simba DSN [SampleDSN] was not found in:" + echo " ${ODBC_INI}" + echo + echo "The performance executable can be reused for Simba, but" + echo "the Simba DSN must be available in ODBCINI." + exit 1 +fi + +# ============================================================ +# RESULT FILES +# ============================================================ -# ------------------------------------------------------------ -# Run Google Driver benchmark +RESULT_DIR="${WORKSPACE_DIR}/benchmark_results" + +mkdir -p "${RESULT_DIR}" + +GOOGLE_RESULTS="${RESULT_DIR}/current_bq.txt" +SIMBA_RESULTS="${RESULT_DIR}/current_core.txt" + +MAIN_GOOGLE_RESULTS="${RESULT_DIR}/main_bq.txt" + +SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" + +rm -f \ + "${GOOGLE_RESULTS}" \ + "${SIMBA_RESULTS}" \ + "${MAIN_GOOGLE_RESULTS}" \ + "${SUMMARY_FILE}" + +# ============================================================ +# RUN GOOGLE DRIVER # -# One complete performance_test suite run. +# IMPORTANT: # -# BENCHMARK_ITERATIONS can be supplied by the CI trigger. -# Default = 1. +# The executable is invoked directly. # -# Every test in performance_test is compared, not just one test. -# ------------------------------------------------------------ - -BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" +# This executes the COMPLETE performance_test suite. +# +# There is NO ctest here. +# +# There is NO individual test filter. +# +# With BENCHMARK_ITERATIONS=1: +# +# performance_test +# +# runs exactly once, and every test registered in that executable +# is executed once. +# ============================================================ -GOOGLE_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_bq.txt" +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" echo "============================================================" -echo "Running Google Driver performance suite" +echo "Running Google Driver performance_test" +echo "============================================================" +echo "DSN : ${ODBC_TESTS_DSN}" echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "Executable: ${PERFORMANCE_TEST}" echo "============================================================" -: >"${GOOGLE_RESULTS}" - GOOGLE_EXIT_CODE=0 for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - echo "=== Google Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + + echo "=== Google Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ >>"${GOOGLE_RESULTS}" set +e + "${PERFORMANCE_TEST}" >>"${GOOGLE_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Google Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + echo "WARNING: Google Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} fi + done if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then + echo echo "ERROR: Google Driver performance_test failed." + echo + echo "Last Google benchmark output:" + tail -100 "${GOOGLE_RESULTS}" || true exit "${GOOGLE_EXIT_CODE}" fi -# ------------------------------------------------------------ -# Run Simba Driver performance suite +echo "Google Driver performance suite completed successfully." + +# ============================================================ +# RUN SIMBA DRIVER +# +# SAME performance_test executable. # -# Simba is already installed/configured by the Simba pipeline. +# Only the DSN changes. # -# We do NOT rebuild or download Simba here. -# ------------------------------------------------------------ +# Therefore the exact same performance test cases are executed +# against Simba. +# ============================================================ export ODBC_TESTS_DSN="SampleDSN" -SIMBA_RESULTS="${WORKSPACE_DIR}/benchmark_results_current_core.txt" - echo "============================================================" -echo "Running Simba Driver performance suite" -echo "DSN: ${ODBC_TESTS_DSN}" +echo "Running Simba Driver performance_test" +echo "============================================================" +echo "DSN : ${ODBC_TESTS_DSN}" +echo "Iterations: ${BENCHMARK_ITERATIONS}" +echo "Executable: ${PERFORMANCE_TEST}" echo "============================================================" - -: >"${SIMBA_RESULTS}" SIMBA_EXIT_CODE=0 for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - echo "=== Simba Driver benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + + echo "=== Simba Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ >>"${SIMBA_RESULTS}" set +e + "${PERFORMANCE_TEST}" >>"${SIMBA_RESULTS}" 2>&1 + RUN_EXIT_CODE=$? + set -e if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Simba Driver iteration ${i} failed with ${RUN_EXIT_CODE}" + echo "WARNING: Simba Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" SIMBA_EXIT_CODE=${RUN_EXIT_CODE} fi + done if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then + echo echo "ERROR: Simba Driver performance_test failed." + echo + echo "Last Simba benchmark output:" + tail -100 "${SIMBA_RESULTS}" || true exit "${SIMBA_EXIT_CODE}" fi -# ------------------------------------------------------------ -# Main branch Google benchmark -# -# Main is optional. If it has already been uploaded, download it -# and include it in the comparison. -# ------------------------------------------------------------ +echo "Simba Driver performance suite completed successfully." -mkdir -p "${WORKSPACE_DIR}/benchmark_results" +# ============================================================ +# DOWNLOAD MAIN GOOGLE DRIVER RESULTS +# ============================================================ CURRENT_BRANCH="${BRANCH_NAME:-main}" -SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | - sed 's/[^a-zA-Z0-9._-]/_/g') - -MAIN_RESULTS="${WORKSPACE_DIR}/benchmark_results/main_bq.txt" +SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | + sed 's/[^a-zA-Z0-9._-]/_/g')" echo "============================================================" -echo "Downloading main branch Google benchmark baseline" +echo "Downloading Google Driver main baseline" echo "============================================================" set +e gcloud storage cp \ "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ - "${MAIN_RESULTS}" + "${MAIN_GOOGLE_RESULTS}" MAIN_DOWNLOAD_EXIT=$? set -e if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then - echo "WARNING: Main branch benchmark result not available." - rm -f "${MAIN_RESULTS}" + echo "WARNING: Main Google benchmark result is not available." + rm -f "${MAIN_GOOGLE_RESULTS}" +else + echo "Main Google benchmark downloaded successfully." fi -# ------------------------------------------------------------ -# Parse results and generate comparison table -# ------------------------------------------------------------ +# ============================================================ +# PARSE AND COMPARE +# +# Comparison: +# +# Simba Driver +# | +# | Google Current vs Simba +# v +# Google Driver Current +# | +# | Google Main vs Google Current +# v +# Google Driver Main +# +# Every test appearing in any of the three result files is +# included in the table. +# ============================================================ -SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" +echo "============================================================" +echo "Generating performance comparison" +echo "============================================================" python3 <<'PYTHON' import os import re -SIMBA_FILE = "benchmark_results_current_core.txt" -GOOGLE_FILE = "benchmark_results_current_bq.txt" + +SIMBA_FILE = "benchmark_results/current_core.txt" +GOOGLE_FILE = "benchmark_results/current_bq.txt" MAIN_FILE = "benchmark_results/main_bq.txt" + OUTPUT_FILE = "benchmark_summary_table.txt" def clean_test_name(name): - # GTest: - # - # [Instantiation/]TestSuite.TestCase[/Param] - # - # Keep TestCase so that the same test can be compared - # between Simba and Google drivers. + """ + GTest names can look like: + + TestSuite.TestCase + Instantiation/TestSuite.TestCase + TestSuite.TestCase/parameter + + Keep the TestCase portion so the same test can be compared + between the two drivers. + """ if "." in name: name = name.split(".", 1)[1] - # Remove old HTAPI parameter suffix if present. + # Remove old HTAPI parameter suffixes. name = re.sub(r"/(?:With|Without)HTAPI$", "", name) return name @@ -365,10 +510,20 @@ def parse_time_to_ms(value): if unit == "ns": return number / 1_000_000 - return number + return None def parse_gtest_output(filename): + """ + Parse: + + [ OK ] TestSuite.TestCase (123 ms) + + Multiple iterations are supported. + + The median is used if BENCHMARK_ITERATIONS > 1. + """ + samples = {} if not os.path.exists(filename): @@ -379,13 +534,16 @@ def parse_gtest_output(filename): ) with open(filename, "r", errors="replace") as file: + for line in file: + match = pattern.search(line) if not match: continue test_name = clean_test_name(match.group(1)) + duration = parse_time_to_ms(match.group(2)) if duration is None: @@ -393,15 +551,15 @@ def parse_gtest_output(filename): samples.setdefault(test_name, []).append(duration) - # Median per test across iterations. results = {} for test_name, values in samples.items(): + values.sort() n = len(values) - if n % 2: + if n % 2 == 1: median = values[n // 2] else: median = ( @@ -415,6 +573,7 @@ def parse_gtest_output(filename): def percentage(value, reference): + if value is None or reference is None or reference == 0: return "N/A" @@ -424,21 +583,28 @@ def percentage(value, reference): simba = parse_gtest_output(SIMBA_FILE) + google = parse_gtest_output(GOOGLE_FILE) + main_google = parse_gtest_output(MAIN_FILE) + all_tests = sorted( set(simba.keys()) | set(google.keys()) | set(main_google.keys()) ) + rows = [] + for test in all_tests: simba_ms = simba.get(test) + google_ms = google.get(test) + main_ms = main_google.get(test) google_vs_simba = percentage( @@ -451,23 +617,26 @@ for test in all_tests: google_ms ) - simba_value = ( - f"{simba_ms:.0f}ms" - if simba_ms is not None - else "N/A" - ) - - google_value = ( - f"{google_ms:.0f}ms ({google_vs_simba})" - if google_ms is not None - else "N/A" - ) + if simba_ms is None: + simba_value = "N/A" + else: + simba_value = f"{simba_ms:.0f}ms" + + if google_ms is None: + google_value = "N/A" + else: + google_value = ( + f"{google_ms:.0f}ms " + f"({google_vs_simba})" + ) - main_value = ( - f"{main_ms:.0f}ms ({main_vs_google})" - if main_ms is not None - else "N/A" - ) + if main_ms is None: + main_value = "N/A" + else: + main_value = ( + f"{main_ms:.0f}ms " + f"({main_vs_google})" + ) rows.append( ( @@ -486,19 +655,25 @@ headers = [ "Google Driver (Main)", ] + widths = [] for index, header in enumerate(headers): + width = len(header) for row in rows: - width = max(width, len(row[index])) + width = max( + width, + len(row[index]) + ) widths.append(width) table = [] + table.append( "| " + " | ".join( @@ -508,6 +683,7 @@ table.append( + " |" ) + table.append( "| " + " | ".join( @@ -517,7 +693,9 @@ table.append( + " |" ) + for row in rows: + table.append( "| " + " | ".join( @@ -532,29 +710,42 @@ description = ( "Percentages in **Google Driver (Current)** are relative to " "**Simba Driver**. Percentages in **Google Driver (Main)** " "are relative to **Google Driver (Current)**. " - "Negative values mean the Google Driver is faster; " - "positive values mean it is slower." + "Negative values indicate that the Google Driver is faster; " + "positive values indicate that it is slower." +) + + +output = ( + description + + "\n\n" + + "\n".join(table) + + "\n" ) -output = description + "\n\n" + "\n".join(table) + "\n" with open(OUTPUT_FILE, "w") as file: file.write(output) + print(output) -print(f"Compared {len(all_tests)} performance tests.") + +print( + f"Compared {len(all_tests)} performance test cases." +) PYTHON -# ------------------------------------------------------------ -# Upload current results -# ------------------------------------------------------------ +# ============================================================ +# UPLOAD RESULTS +# ============================================================ RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" echo "============================================================" echo "Uploading benchmark results" -echo "Destination: ${RESULTS_BUCKET}" +echo "============================================================" +echo "Destination:" +echo " ${RESULTS_BUCKET}" echo "============================================================" io::run gcloud storage cp \ @@ -569,20 +760,41 @@ io::run gcloud storage cp \ "${SUMMARY_FILE}" \ "${RESULTS_BUCKET}/benchmark_summary_table.txt" +# ============================================================ +# FINAL SUMMARY +# ============================================================ + echo echo "============================================================" -echo "Benchmark completed" +echo "BENCHMARK COMPLETED SUCCESSFULLY" echo "============================================================" + echo -echo "Simba results:" -echo " ${SIMBA_RESULTS}" +echo "Performance executable:" +echo " ${PERFORMANCE_TEST}" + +echo +echo "Google Driver:" +echo " ${GOOGLE_DRIVER}" + echo echo "Google results:" echo " ${GOOGLE_RESULTS}" + +echo +echo "Simba results:" +echo " ${SIMBA_RESULTS}" + echo echo "Comparison:" echo " ${SUMMARY_FILE}" + echo echo "GCS:" echo " ${RESULTS_BUCKET}/" + +echo +echo "All tests registered in performance_test were executed." +echo "Benchmark iterations: ${BENCHMARK_ITERATIONS}" + echo "============================================================" diff --git a/ci/gha/builds/lib/odbc.ini b/ci/gha/builds/lib/odbc.ini index 428bb3b316..5092a47b40 100644 --- a/ci/gha/builds/lib/odbc.ini +++ b/ci/gha/builds/lib/odbc.ini @@ -234,4 +234,3 @@ LargeResultsTempTableExpirationTime=3600000 # given. Result pages have at most as many rows as are specified by the RowsFetchedPerBlock # connection property. #HTAPI_MinActivationRatio=3 - From 766bc0ca24c3ca33636122e3e0238b8a0488df49 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 07:02:55 +0000 Subject: [PATCH 10/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index abd984df0c..c7c8dbe8b7 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -352,6 +352,7 @@ echo "Google Driver performance suite completed successfully." # against Simba. # ============================================================ +export ODBCINI="opt/odbc-driver/googlebigqueryodbc/odbc.ini" export ODBC_TESTS_DSN="SampleDSN" echo "============================================================" From 8280abd64d8473a416c67cd39f677d2bfc84c22e Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 09:25:23 +0000 Subject: [PATCH 11/33] test --- .../integration-production-bq-driver-dm.sh | 798 ++++++++---------- 1 file changed, 339 insertions(+), 459 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index c7c8dbe8b7..b23dc6a2f5 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -1,3 +1,6 @@ +# linux-bq-driver-benchmark.sh + +```bash #!/bin/bash # # Copyright 2025 Google LLC @@ -22,18 +25,56 @@ source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh source module ci/lib/io.sh -WORKSPACE_DIR="$(pwd)" +WORKSPACE_DIR=$(pwd) +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" -# ============================================================ -# VCPKG -# ============================================================ +# Number of times to run the COMPLETE performance_test suite. +# Default 1 means every performance test runs exactly once. +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" -VCPKG_VERSION="$(cat /tmp/vcpkg-version.txt)" -export VCPKG_VERSION +# Branch used for storing current benchmark results. +CURRENT_BRANCH="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ + sed 's/[^a-zA-Z0-9._-]/_/g') + +# ------------------------------------------------------------ +# Benchmark result locations +# ------------------------------------------------------------ + +RESULTS_DIR="${WORKSPACE_DIR}/benchmark_results" +mkdir -p "${RESULTS_DIR}" + +SIMBA_RESULTS="${RESULTS_DIR}/performance_benchmark_results_Core.txt" +GOOGLE_RESULTS="${RESULTS_DIR}/performance_benchmark_results_BqDriver.txt" +MAIN_GOOGLE_RESULTS="${RESULTS_DIR}/main_bq.txt" + +SUMMARY_FILE="${RESULTS_DIR}/benchmark_summary_table.txt" + +# ------------------------------------------------------------ +# Header +# ------------------------------------------------------------ echo "============================================================" -echo "VCPKG" +echo "Linux ODBC Performance Benchmark Comparison" +echo "============================================================" +echo "Current branch : ${CURRENT_BRANCH}" +echo "Benchmark iterations : ${BENCHMARK_ITERATIONS}" +echo "Build directory : ${BUILD_DIR}" +echo +echo "Comparison:" +echo " 1. Simba Driver - Current Branch" +echo " 2. Google Driver - Current Branch" +echo " 3. Google Driver - Main Branch" echo "============================================================" + +# ------------------------------------------------------------ +# Vcpkg +# ------------------------------------------------------------ + +VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +export VCPKG_VERSION + echo "Using VCPKG_VERSION=${VCPKG_VERSION}" export VCPKG_ROOT=/vcpkg @@ -46,73 +87,42 @@ if [[ ! -d "${VCPKG_ROOT}/.git" ]]; then fi cd "${VCPKG_ROOT}" - git checkout "${VCPKG_VERSION}" -./bootstrap-vcpkg.sh -disableMetrics +if [[ ! -f "${VCPKG_ROOT}/vcpkg" ]]; then + ./bootstrap-vcpkg.sh -disableMetrics +fi cd "${WORKSPACE_DIR}" -# ============================================================ -# COMMON CMAKE ARGUMENTS -# ============================================================ +# ------------------------------------------------------------ +# CMake arguments +# ------------------------------------------------------------ mapfile -t cmake_args < <(cmake::common_args) -# ============================================================ -# BENCHMARK CONFIGURATION -# ============================================================ - -BUILD_DIR="${WORKSPACE_DIR}/cmake-out" - -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" - -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" - -BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" - -echo "============================================================" -echo "Benchmark configuration" -echo "============================================================" -echo "Workspace : ${WORKSPACE_DIR}" -echo "Build directory : ${BUILD_DIR}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Google DSN : ${ODBC_TESTS_DSN}" -echo "============================================================" - -# ============================================================ -# UNIXODBC -# ============================================================ - -if command -v odbcinst &>/dev/null; then - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." -else - export UNIXODBC_INSTALLED=false - export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini - echo "unixODBC is not installed." - echo "Using ODBCINSTINI=${ODBCINSTINI}" -fi - -# ============================================================ -# CONFIGURE CMAKE +# ------------------------------------------------------------ +# Configure the normal Google Driver build # -# IMPORTANT: -# This is a separate benchmark build. +# DO NOT use BUILD_PERFORMANCE_TEST_ONLY here. # -# BUILD_PERFORMANCE_TEST_ONLY=ON means we are configuring the -# performance-test build rather than configuring/running the -# complete integration-test suite. -# ============================================================ +# The benchmark executable must be able to test the current +# branch Google driver, so the normal driver build is required. +# ------------------------------------------------------------ +echo echo "============================================================" -echo "Configuring performance_test" +echo "Configuring current branch build" echo "============================================================" -io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ +rm -rf "${BUILD_DIR}" + +io::run cmake \ + -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ + -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_STANDARD=17 \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ @@ -121,292 +131,246 @@ io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ -DODBC_UNIT_TESTING=OFF \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -# ============================================================ -# BUILD ONLY performance_test -# ============================================================ +# ------------------------------------------------------------ +# Build ONLY performance_test +# +# This does not run ctest and does not explicitly build every +# project target. CMake builds performance_test and its required +# dependencies. +# ------------------------------------------------------------ +echo echo "============================================================" -echo "Building ONLY performance_test" +echo "Building performance_test" echo "============================================================" -io::run cmake --build "${BUILD_DIR}" \ - --target google_cloud_odbc_bq_driver \ - --config Release \ - --parallel "$(nproc)" - -io::run cmake --build "${BUILD_DIR}" \ +io::run cmake \ + --build "${BUILD_DIR}" \ --target performance_test \ - --config Release \ --parallel "$(nproc)" -# ============================================================ -# LOCATE PERFORMANCE TEST -# ============================================================ +# ------------------------------------------------------------ +# Locate performance_test +# ------------------------------------------------------------ PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" if [[ ! -f "${PERFORMANCE_TEST}" ]]; then + echo echo "ERROR: performance_test was not found:" echo " ${PERFORMANCE_TEST}" echo - echo "Searching build directory:" - find "${BUILD_DIR}" -type f -name "performance_test*" -print + echo "Matching files:" + find "${BUILD_DIR}" -type f -name "performance_test*" -print || true exit 1 fi -echo "============================================================" -echo "performance_test found" -echo "============================================================" -echo "${PERFORMANCE_TEST}" +echo +echo "Performance test executable:" +echo " ${PERFORMANCE_TEST}" -# ============================================================ -# GOOGLE DRIVER -# -# The Google driver is generated by the performance_test build. -# We do NOT download an existing Google .so. -# ============================================================ +# ------------------------------------------------------------ +# Verify current branch Google Driver +# ------------------------------------------------------------ GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" if [[ ! -f "${GOOGLE_DRIVER}" ]]; then - echo "ERROR: Google driver was not generated by the performance build:" + echo + echo "ERROR: Current branch Google Driver was not generated:" echo " ${GOOGLE_DRIVER}" echo - echo "Generated shared libraries:" - find "${BUILD_DIR}" -type f -name "*.so" -print + echo "Matching driver files:" + find "${BUILD_DIR}" \ + -type f \ + \( -name "*google_cloud_odbc_bq_driver*.so" \ + -o -name "*bq_driver*.so" \) \ + -print || true exit 1 fi -echo "============================================================" -echo "Google driver generated" -echo "============================================================" -echo "${GOOGLE_DRIVER}" +echo +echo "Current branch Google Driver:" +echo " ${GOOGLE_DRIVER}" -# ============================================================ -# CERTIFICATE -# ============================================================ +# ------------------------------------------------------------ +# Copy roots.pem for Google Driver +# ------------------------------------------------------------ GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" -if [[ ! -f /opt/odbc-driver/roots.pem ]]; then - echo "ERROR: /opt/odbc-driver/roots.pem was not found." +if [[ -f "/opt/odbc-driver/roots.pem" ]]; then + io::run cp \ + "/opt/odbc-driver/roots.pem" \ + "${GOOGLE_DRIVER_DIR}/roots.pem" +else + echo + echo "ERROR: Required certificate file was not found:" + echo " /opt/odbc-driver/roots.pem" exit 1 fi -io::run cp \ - /opt/odbc-driver/roots.pem \ - "${GOOGLE_DRIVER_DIR}/roots.pem" +# ------------------------------------------------------------ +# ODBC environment +# ------------------------------------------------------------ -# ============================================================ -# ODBC CONFIGURATION -# -# The existing /opt/odbc-driver/odbc.ini must contain the DSNs -# used below. -# -# Google: -# SampleDSNGoogleDriver -# -# Simba: -# SampleDSN -# -# If SampleDSN does not exist, Simba cannot be benchmarked. -# ============================================================ +export ODBCINI="/opt/odbc-driver/odbc.ini" -ODBC_INI="/opt/odbc-driver/odbc.ini" - -if [[ ! -f "${ODBC_INI}" ]]; then +if [[ ! -f "${ODBCINI}" ]]; then + echo echo "ERROR: ODBC configuration was not found:" - echo " ${ODBC_INI}" + echo " ${ODBCINI}" exit 1 fi -export ODBCINI="${ODBC_INI}" - -echo "============================================================" -echo "ODBC configuration" -echo "============================================================" -echo "ODBCINI=${ODBCINI}" - echo -echo "Available DSNs:" -grep -E '^\[[^]]+\]$' "${ODBCINI}" || true - -# ============================================================ -# VERIFY GOOGLE DSN -# ============================================================ +echo "Using ODBCINI=${ODBCINI}" -if ! grep -q '^\[SampleDSNGoogleDriver\]$' "${ODBC_INI}"; then - echo - echo "ERROR: Google DSN [SampleDSNGoogleDriver] was not found in:" - echo " ${ODBC_INI}" - exit 1 -fi +# ------------------------------------------------------------ +# unixODBC / iODBC setup +# ------------------------------------------------------------ -# ============================================================ -# VERIFY SIMBA DSN -# ============================================================ - -if ! grep -q '^\[SampleDSN\]$' "${ODBC_INI}"; then - echo - echo "ERROR: Simba DSN [SampleDSN] was not found in:" - echo " ${ODBC_INI}" - echo - echo "The performance executable can be reused for Simba, but" - echo "the Simba DSN must be available in ODBCINI." - exit 1 +if command -v odbcinst >/dev/null 2>&1; then + export UNIXODBC_INSTALLED=true + echo "unixODBC is installed." +else + export UNIXODBC_INSTALLED=false + export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" + echo "unixODBC is not installed." fi -# ============================================================ -# RESULT FILES -# ============================================================ - -RESULT_DIR="${WORKSPACE_DIR}/benchmark_results" +# ------------------------------------------------------------ +# Unique test table prefix +# ------------------------------------------------------------ -mkdir -p "${RESULT_DIR}" +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=\ +"${TRIGGER_NAME:-benchmark}_${SANITIZED_BRANCH}" -GOOGLE_RESULTS="${RESULT_DIR}/current_bq.txt" -SIMBA_RESULTS="${RESULT_DIR}/current_core.txt" +# ------------------------------------------------------------ +# Helper: verify a DSN exists +# ------------------------------------------------------------ -MAIN_GOOGLE_RESULTS="${RESULT_DIR}/main_bq.txt" +verify_dsn() { + local dsn="$1" -SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" - -rm -f \ - "${GOOGLE_RESULTS}" \ - "${SIMBA_RESULTS}" \ - "${MAIN_GOOGLE_RESULTS}" \ - "${SUMMARY_FILE}" + if ! grep -q "^\[${dsn}\]$" "${ODBCINI}"; then + echo + echo "ERROR: DSN not found in ${ODBCINI}:" + echo " [${dsn}]" + echo + echo "Available DSNs:" + grep '^\[.*\]$' "${ODBCINI}" || true + exit 1 + fi +} -# ============================================================ -# RUN GOOGLE DRIVER -# -# IMPORTANT: -# -# The executable is invoked directly. -# -# This executes the COMPLETE performance_test suite. -# -# There is NO ctest here. +# ------------------------------------------------------------ +# Helper: run complete performance_test suite # -# There is NO individual test filter. +# One invocation of performance_test runs every test registered +# in that executable. # -# With BENCHMARK_ITERATIONS=1: -# -# performance_test -# -# runs exactly once, and every test registered in that executable -# is executed once. -# ============================================================ - -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" - -echo "============================================================" -echo "Running Google Driver performance_test" -echo "============================================================" -echo "DSN : ${ODBC_TESTS_DSN}" -echo "Iterations: ${BENCHMARK_ITERATIONS}" -echo "Executable: ${PERFORMANCE_TEST}" -echo "============================================================" - -GOOGLE_EXIT_CODE=0 - -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - - echo "=== Google Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >>"${GOOGLE_RESULTS}" - - set +e - - "${PERFORMANCE_TEST}" >>"${GOOGLE_RESULTS}" 2>&1 - - RUN_EXIT_CODE=$? +# BENCHMARK_ITERATIONS=1: +# every performance_test test executes once. +# ------------------------------------------------------------ - set -e +run_performance_suite() { + local driver_name="$1" + local dsn="$2" + local output_file="$3" - if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Google Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" - GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} - fi + export ODBC_TESTS_DSN="${dsn}" -done + verify_dsn "${ODBC_TESTS_DSN}" -if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then - echo - echo "ERROR: Google Driver performance_test failed." echo - echo "Last Google benchmark output:" - tail -100 "${GOOGLE_RESULTS}" || true - exit "${GOOGLE_EXIT_CODE}" -fi - -echo "Google Driver performance suite completed successfully." + echo "============================================================" + echo "Running ${driver_name} performance suite" + echo "============================================================" + echo "DSN : ${ODBC_TESTS_DSN}" + echo "Iterations: ${BENCHMARK_ITERATIONS}" + echo "Executable: ${PERFORMANCE_TEST}" + echo "Output : ${output_file}" + echo "============================================================" -# ============================================================ -# RUN SIMBA DRIVER -# -# SAME performance_test executable. -# -# Only the DSN changes. -# -# Therefore the exact same performance test cases are executed -# against Simba. -# ============================================================ + : > "${output_file}" -export ODBCINI="opt/odbc-driver/googlebigqueryodbc/odbc.ini" -export ODBC_TESTS_DSN="SampleDSN" + local test_exit_code=0 + local run_exit=0 -echo "============================================================" -echo "Running Simba Driver performance_test" -echo "============================================================" -echo "DSN : ${ODBC_TESTS_DSN}" -echo "Iterations: ${BENCHMARK_ITERATIONS}" -echo "Executable: ${PERFORMANCE_TEST}" -echo "============================================================" + for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do -SIMBA_EXIT_CODE=0 + echo + echo "${driver_name}: iteration ${i}/${BENCHMARK_ITERATIONS}" -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >> "${output_file}" - echo "=== Simba Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >>"${SIMBA_RESULTS}" + set +e - set +e + "${PERFORMANCE_TEST}" \ + >> "${output_file}" \ + 2>&1 - "${PERFORMANCE_TEST}" >>"${SIMBA_RESULTS}" 2>&1 + run_exit=$? - RUN_EXIT_CODE=$? + set -e - set -e + if [[ ${run_exit} -ne 0 ]]; then + echo + echo "WARNING: ${driver_name} iteration ${i} failed with exit code ${run_exit}" + test_exit_code=${run_exit} + fi + done - if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Simba Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" - SIMBA_EXIT_CODE=${RUN_EXIT_CODE} + if [[ ${test_exit_code} -ne 0 ]]; then + echo + echo "ERROR: ${driver_name} performance_test failed." + echo + echo "Last benchmark output:" + tail -n 50 "${output_file}" || true + return "${test_exit_code}" fi +} -done - -if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then - echo - echo "ERROR: Simba Driver performance_test failed." - echo - echo "Last Simba benchmark output:" - tail -100 "${SIMBA_RESULTS}" || true - exit "${SIMBA_EXIT_CODE}" -fi +# ------------------------------------------------------------ +# 1. Run Simba Driver - Current Branch +# +# Simba is loaded through the existing SampleDSN configuration. +# ------------------------------------------------------------ -echo "Simba Driver performance suite completed successfully." +run_performance_suite \ + "Simba Driver (Current)" \ + "SampleDSN" \ + "${SIMBA_RESULTS}" -# ============================================================ -# DOWNLOAD MAIN GOOGLE DRIVER RESULTS -# ============================================================ +# ------------------------------------------------------------ +# 2. Run Google Driver - Current Branch +# +# IMPORTANT: +# +# SampleDSNGoogleDriver must point to: +# +# ${GOOGLE_DRIVER} +# +# in /opt/odbc-driver/odbc.ini. +# ------------------------------------------------------------ -CURRENT_BRANCH="${BRANCH_NAME:-main}" +run_performance_suite \ + "Google Driver (Current)" \ + "SampleDSNGoogleDriver" \ + "${GOOGLE_RESULTS}" -SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | - sed 's/[^a-zA-Z0-9._-]/_/g')" +# ------------------------------------------------------------ +# 3. Download Google Driver Main Branch results +# +# Main benchmark results are produced by the main branch build +# and used as the baseline here. +# ------------------------------------------------------------ +echo echo "============================================================" -echo "Downloading Google Driver main baseline" +echo "Downloading Google Driver main branch benchmark results" echo "============================================================" set +e @@ -420,63 +384,46 @@ MAIN_DOWNLOAD_EXIT=$? set -e if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then - echo "WARNING: Main Google benchmark result is not available." + echo + echo "WARNING: Main branch benchmark result was not available." rm -f "${MAIN_GOOGLE_RESULTS}" -else - echo "Main Google benchmark downloaded successfully." fi -# ============================================================ -# PARSE AND COMPARE -# -# Comparison: -# -# Simba Driver -# | -# | Google Current vs Simba -# v -# Google Driver Current -# | -# | Google Main vs Google Current -# v -# Google Driver Main -# -# Every test appearing in any of the three result files is -# included in the table. -# ============================================================ +# ------------------------------------------------------------ +# Generate comparison table +# ------------------------------------------------------------ +echo echo "============================================================" -echo "Generating performance comparison" +echo "Generating benchmark comparison" echo "============================================================" +SIMBA_FILE="${SIMBA_RESULTS}" \ +GOOGLE_FILE="${GOOGLE_RESULTS}" \ +MAIN_FILE="${MAIN_GOOGLE_RESULTS}" \ +OUTPUT_FILE="${SUMMARY_FILE}" \ python3 <<'PYTHON' import os import re -SIMBA_FILE = "benchmark_results/current_core.txt" -GOOGLE_FILE = "benchmark_results/current_bq.txt" -MAIN_FILE = "benchmark_results/main_bq.txt" - -OUTPUT_FILE = "benchmark_summary_table.txt" +SIMBA_FILE = os.environ["SIMBA_FILE"] +GOOGLE_FILE = os.environ["GOOGLE_FILE"] +MAIN_FILE = os.environ["MAIN_FILE"] +OUTPUT_FILE = os.environ["OUTPUT_FILE"] def clean_test_name(name): - """ - GTest names can look like: - - TestSuite.TestCase - Instantiation/TestSuite.TestCase - TestSuite.TestCase/parameter - - Keep the TestCase portion so the same test can be compared - between the two drivers. - """ + # GTest name: + # [Instantiation/]TestSuite.TestCase[/Param] + # + # Remove the suite/instantiation prefix so corresponding + # benchmark cases can be compared. if "." in name: name = name.split(".", 1)[1] - # Remove old HTAPI parameter suffixes. + # Preserve compatibility with older output. name = re.sub(r"/(?:With|Without)HTAPI$", "", name) return name @@ -488,9 +435,6 @@ def parse_time_to_ms(value): value = value.strip() - if value == "N/A": - return None - match = re.match(r"^([\d.]+)\s*(\w+)$", value) if not match: @@ -499,32 +443,22 @@ def parse_time_to_ms(value): number = float(match.group(1)) unit = match.group(2).lower() + if unit == "s": + return number * 1000.0 + if unit == "ms": return number - if unit == "s": - return number * 1000 - if unit == "us": - return number / 1000 + return number / 1000.0 if unit == "ns": - return number / 1_000_000 + return number / 1000000.0 return None def parse_gtest_output(filename): - """ - Parse: - - [ OK ] TestSuite.TestCase (123 ms) - - Multiple iterations are supported. - - The median is used if BENCHMARK_ITERATIONS > 1. - """ - samples = {} if not os.path.exists(filename): @@ -535,37 +469,33 @@ def parse_gtest_output(filename): ) with open(filename, "r", errors="replace") as file: - for line in file: - match = pattern.search(line) if not match: continue test_name = clean_test_name(match.group(1)) + duration_ms = parse_time_to_ms(match.group(2)) - duration = parse_time_to_ms(match.group(2)) - - if duration is None: + if duration_ms is None: continue - samples.setdefault(test_name, []).append(duration) + samples.setdefault(test_name, []).append(duration_ms) results = {} for test_name, values in samples.items(): - values.sort() - n = len(values) + count = len(values) - if n % 2 == 1: - median = values[n // 2] + if count % 2 == 1: + median = values[count // 2] else: median = ( - values[n // 2 - 1] + - values[n // 2] + values[count // 2 - 1] + + values[count // 2] ) / 2.0 results[test_name] = median @@ -574,74 +504,61 @@ def parse_gtest_output(filename): def percentage(value, reference): - if value is None or reference is None or reference == 0: return "N/A" - pct = ((value - reference) / reference) * 100 + change = ((value - reference) / reference) * 100.0 - return f"{pct:+.0f}%" + return f"{change:+.0f}%" simba = parse_gtest_output(SIMBA_FILE) - google = parse_gtest_output(GOOGLE_FILE) - main_google = parse_gtest_output(MAIN_FILE) - all_tests = sorted( - set(simba.keys()) | - set(google.keys()) | - set(main_google.keys()) + set(simba) | + set(google) | + set(main_google) ) - rows = [] +for test_name in all_tests: + simba_ms = simba.get(test_name) + google_ms = google.get(test_name) + main_ms = main_google.get(test_name) -for test in all_tests: - - simba_ms = simba.get(test) - - google_ms = google.get(test) - - main_ms = main_google.get(test) + simba_value = ( + f"{simba_ms:.0f}ms" + if simba_ms is not None + else "N/A" + ) - google_vs_simba = percentage( - google_ms, - simba_ms + google_value = ( + f"{google_ms:.0f}ms" + if google_ms is not None + else "N/A" ) - main_vs_google = percentage( - main_ms, - google_ms + main_value = ( + f"{main_ms:.0f}ms" + if main_ms is not None + else "N/A" ) - if simba_ms is None: - simba_value = "N/A" - else: - simba_value = f"{simba_ms:.0f}ms" - - if google_ms is None: - google_value = "N/A" - else: - google_value = ( - f"{google_ms:.0f}ms " - f"({google_vs_simba})" - ) + google_pct = percentage(google_ms, simba_ms) + main_pct = percentage(main_ms, google_ms) - if main_ms is None: - main_value = "N/A" - else: - main_value = ( - f"{main_ms:.0f}ms " - f"({main_vs_google})" - ) + if google_ms is not None: + google_value += f" ({google_pct})" + + if main_ms is not None: + main_value += f" ({main_pct})" rows.append( ( - test, + test_name, simba_value, google_value, main_value, @@ -649,153 +566,116 @@ for test in all_tests: ) -headers = [ +headers = ( "Test Case", - "Simba Driver", + "Simba Driver (Current)", "Google Driver (Current)", "Google Driver (Main)", -] - +) widths = [] for index, header in enumerate(headers): - width = len(header) for row in rows: - width = max( - width, - len(row[index]) - ) + width = max(width, len(row[index])) widths.append(width) table = [] +description = ( + "Only tests executed by the performance_test executable are included. " + "Percentages in Google Driver (Current) are relative to Simba Driver " + "(Current). Percentages in Google Driver (Main) are relative to Google " + "Driver (Current). Negative values are faster and positive values are slower." +) + +table.append(description) +table.append("") table.append( - "| " - + " | ".join( + "| " + + " | ".join( header.ljust(widths[index]) for index, header in enumerate(headers) - ) - + " |" + ) + + " |" ) - table.append( - "| " - + " | ".join( + "|-" + + "-|-".join( "-" * widths[index] for index in range(len(headers)) - ) - + " |" + ) + + "-|" ) - for row in rows: - table.append( - "| " - + " | ".join( + "| " + + " | ".join( row[index].ljust(widths[index]) for index in range(len(headers)) - ) - + " |" + ) + + " |" ) -description = ( - "Percentages in **Google Driver (Current)** are relative to " - "**Simba Driver**. Percentages in **Google Driver (Main)** " - "are relative to **Google Driver (Current)**. " - "Negative values indicate that the Google Driver is faster; " - "positive values indicate that it is slower." -) - - -output = ( - description - + "\n\n" - + "\n".join(table) - + "\n" -) - +output = "\n".join(table) + "\n" with open(OUTPUT_FILE, "w") as file: file.write(output) - print(output) - -print( - f"Compared {len(all_tests)} performance test cases." -) +print(f"Compared {len(all_tests)} performance test cases.") PYTHON -# ============================================================ -# UPLOAD RESULTS -# ============================================================ +# ------------------------------------------------------------ +# Upload results +# ------------------------------------------------------------ RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" +echo echo "============================================================" echo "Uploading benchmark results" echo "============================================================" -echo "Destination:" -echo " ${RESULTS_BUCKET}" -echo "============================================================" - -io::run gcloud storage cp \ - "${GOOGLE_RESULTS}" \ - "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" io::run gcloud storage cp \ "${SIMBA_RESULTS}" \ "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" +io::run gcloud storage cp \ + "${GOOGLE_RESULTS}" \ + "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" + io::run gcloud storage cp \ "${SUMMARY_FILE}" \ "${RESULTS_BUCKET}/benchmark_summary_table.txt" -# ============================================================ -# FINAL SUMMARY -# ============================================================ - echo echo "============================================================" -echo "BENCHMARK COMPLETED SUCCESSFULLY" +echo "Linux benchmark comparison completed successfully" echo "============================================================" - echo -echo "Performance executable:" -echo " ${PERFORMANCE_TEST}" - -echo -echo "Google Driver:" -echo " ${GOOGLE_DRIVER}" - +echo "Simba current:" +echo " ${SIMBA_RESULTS}" echo -echo "Google results:" +echo "Google current:" echo " ${GOOGLE_RESULTS}" - echo -echo "Simba results:" -echo " ${SIMBA_RESULTS}" - +echo "Google main:" +echo " ${MAIN_GOOGLE_RESULTS}" echo echo "Comparison:" echo " ${SUMMARY_FILE}" - echo -echo "GCS:" +echo "Uploaded to:" echo " ${RESULTS_BUCKET}/" - -echo -echo "All tests registered in performance_test were executed." -echo "Benchmark iterations: ${BENCHMARK_ITERATIONS}" - echo "============================================================" +``` From 17205d763487f6db7a54e44a0177cadcdf5ec51c Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 11:09:13 +0000 Subject: [PATCH 12/33] test --- .../builds/integration-bq-driver-asan.sh | 428 +++++++++-- .../integration-production-bq-driver-dm.sh | 681 ++---------------- ci/cloudbuild/builds/lib/benchmark_results.py | 255 +++++++ ci/gha/builds/lib/odbc.ini | 78 -- 4 files changed, 685 insertions(+), 757 deletions(-) create mode 100644 ci/cloudbuild/builds/lib/benchmark_results.py diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 54bf4fd821..fa541b2e34 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -1,18 +1,9 @@ #!/bin/bash # -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 # -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. set -euo pipefail @@ -20,72 +11,393 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/bazel.sh -source module ci/cloudbuild/builds/lib/secrets.sh -source module ci/cloudbuild/builds/lib/unit-tests.sh -source module ci/lib/io.sh +source module ci/cloudbuild/builds/lib/io.sh WORKSPACE_DIR=$(pwd) -# Export as env variable +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +BUILD_DIR="/opt/odbc-performance" + +PERF_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" +DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" + +CURRENT_BRANCH="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH=$( + echo "${CURRENT_BRANCH}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +) + +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +mkdir -p "$RESULTS_DIR" + +echo "============================================================" +echo "ODBC Linux Performance Benchmark" +echo "============================================================" +echo "Current branch : ${CURRENT_BRANCH}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Results dir : ${RESULTS_DIR}" +echo "============================================================" + +# --------------------------------------------------------------------------- +# Vcpkg +# --------------------------------------------------------------------------- + VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION -echo "Using VCPKG_VERSION=$VCPKG_VERSION" - -# Vcpkg install and configure export VCPKG_ROOT=/vcpkg -git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" + +echo "Using VCPKG_VERSION=${VCPKG_VERSION}" + +git clone \ + --branch "$VCPKG_VERSION" \ + https://github.com/microsoft/vcpkg.git \ + "$VCPKG_ROOT" + cd "$VCPKG_ROOT" + git checkout "$VCPKG_VERSION" -# Bootstrap ./bootstrap-vcpkg.sh -disableMetrics cd "$WORKSPACE_DIR" -# This runs all the unit tests -mapfile -t args < <(bazel::common_args) -mapfile -t unit_tests_args < <(unit_tests::bazel_args) -mapfile -t secrets_bazel < <(secrets::bazel_args) +# --------------------------------------------------------------------------- +# Download Google driver artifacts +# --------------------------------------------------------------------------- -io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... +CURRENT_SO="/tmp/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="/tmp/libgoogle_cloud_odbc_bq_driver_main.so" -# Run the integration tests -mapfile -t cmake_args < <(cmake::common_args) +echo "Downloading current Google driver..." -BUILD_DIR="/opt/odbc-driver" -# This is the name of DSN set in odbc.ini -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export LSAN_OPTIONS="use_tls=0:suppressions=/opt/odbc-driver/lsan.supp:print_suppressions=0" +gcloud storage cp \ + "${DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" \ + "$CURRENT_SO" -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +echo "Downloading main Google driver..." -# Check if unixODBC is installed -if command -v odbcinst &>/dev/null; then - # unixODBC is installed, export environment variable - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." -else - # unixODBC is not installed - export UNIXODBC_INSTALLED=false - export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini - export ODBCINI=/opt/odbc-driver/odbc.ini - echo "unixODBC is not installed." +gcloud storage cp \ + "${DRIVER_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" \ + "$MAIN_SO" + +ls -lh \ + "$CURRENT_SO" \ + "$MAIN_SO" + +# --------------------------------------------------------------------------- +# Locate Simba driver +# --------------------------------------------------------------------------- +# +# The Simba integration image installs the Simba driver. +# We intentionally don't use another DSN. +# +# We find the .so installed by the Simba environment and save it as an +# artifact that can be selected at runtime. +# --------------------------------------------------------------------------- + +echo +echo "Searching for Simba ODBC driver..." + +SIMBA_SO="" + +while IFS= read -r candidate; do + if [[ -f "$candidate" ]]; then + case "$candidate" in + *google*cloud*odbc*.so|*Simba*.so|*simba*.so) + SIMBA_SO="$candidate" + break + ;; + esac + fi +done < <( + find \ + /opt \ + /usr \ + /workspace \ + -type f \ + -name "*.so" \ + 2>/dev/null | + sort +) + +if [[ -z "$SIMBA_SO" ]]; then + echo "ERROR: Could not locate Simba .so" + echo + echo "Available ODBC shared libraries:" + find \ + /opt \ + /usr \ + /workspace \ + -type f \ + -name "*.so*" \ + 2>/dev/null | + grep -Ei 'odbc|simba' | + sort || + true + + exit 1 fi -io::run cmake -B "$BUILD_DIR" \ +echo "Simba driver found:" +echo " $SIMBA_SO" + +SIMBA_COPY="/tmp/libgoogle_cloud_odbc_simba_driver.so" + +cp "$SIMBA_SO" "$SIMBA_COPY" + +ls -lh "$SIMBA_COPY" + +# --------------------------------------------------------------------------- +# Build ONLY performance_test +# --------------------------------------------------------------------------- +# +# BUILD_PERFORMANCE_TEST_ONLY does not build the Google driver. +# It builds performance_test + the small testing utility dependencies. +# +# The driver .so is selected at runtime through the DSN. +# --------------------------------------------------------------------------- + +mapfile -t cmake_args < <(cmake::common_args) + +io::run cmake \ + -S "$WORKSPACE_DIR" \ + -B "$BUILD_DIR" \ "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DENABLE_SANITIZER=ON \ - -DODBC_DEMO_TESTING=ON \ - -DODBC_EXAMPLES=ON \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF - -io::run cmake --build cmake-out - -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON + +io::run cmake \ + --build "$BUILD_DIR" \ + --target performance_test \ + --parallel "$(nproc)" + +PERFORMANCE_TEST="$BUILD_DIR/integration_tests/performance_test" + +if [[ ! -x "$PERFORMANCE_TEST" ]]; then + echo "ERROR: performance_test was not created." + + echo "Searching build directory:" + find "$BUILD_DIR" \ + -type f \ + -name "performance_test*" \ + -print + + exit 1 +fi + +echo "performance_test:" +ls -lh "$PERFORMANCE_TEST" + +# --------------------------------------------------------------------------- +# Configure one DSN +# --------------------------------------------------------------------------- +# +# We use ONE DSN. +# +# Before each benchmark run, replace the driver .so referenced by the DSN. +# --------------------------------------------------------------------------- + +DSN_DIR="/tmp/benchmark-odbc" + +mkdir -p "$DSN_DIR" + +ODBC_INI="$DSN_DIR/odbc.ini" + +cat > "$ODBC_INI" < "$output_file" + + local test_exit_code=0 + + for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + | tee -a "$output_file" + + set +e + + "$PERFORMANCE_TEST" \ + >> "$output_file" \ + 2>&1 + + local run_exit_code=$? + + set -e + + if [[ "$run_exit_code" -ne 0 ]]; then + echo \ + "WARNING: ${name} iteration ${i} exited with ${run_exit_code}" + + echo \ + "WARNING: ${name} iteration ${i} exited with ${run_exit_code}" \ + >> "$output_file" + + test_exit_code="$run_exit_code" + fi + done + + if [[ "$test_exit_code" -ne 0 ]]; then + echo "ERROR: ${name} benchmark failed." + return "$test_exit_code" + fi + + echo + echo "${name} benchmark completed." + + return 0 +} + +# --------------------------------------------------------------------------- +# Run Simba +# --------------------------------------------------------------------------- + +run_driver_benchmark \ + "Simba" \ + "$SIMBA_COPY" \ + "$RESULTS_DIR/simba.txt" + +# --------------------------------------------------------------------------- +# Run current Google driver +# --------------------------------------------------------------------------- + +run_driver_benchmark \ + "Google Current" \ + "$CURRENT_SO" \ + "$RESULTS_DIR/current_bq.txt" + +# --------------------------------------------------------------------------- +# Run main Google driver +# --------------------------------------------------------------------------- + +run_driver_benchmark \ + "Google Main" \ + "$MAIN_SO" \ + "$RESULTS_DIR/main_bq.txt" + +# --------------------------------------------------------------------------- +# Generate benchmark summary +# --------------------------------------------------------------------------- + +SUMMARY_FILE="$RESULTS_DIR/benchmark_summary.txt" + +python3 \ + "$WORKSPACE_DIR/ci/cloudbuild/builds/lib/benchmark_results.py" \ + --simba "$RESULTS_DIR/simba.txt" \ + --current "$RESULTS_DIR/current_bq.txt" \ + --main "$RESULTS_DIR/main_bq.txt" \ + --output "$SUMMARY_FILE" + +# --------------------------------------------------------------------------- +# Upload results +# --------------------------------------------------------------------------- + +RESULTS_GCS_PATH="${PERF_BUCKET}/${SANITIZED_BRANCH}" + +echo +echo "Uploading benchmark results..." + +gcloud storage cp \ + "$RESULTS_DIR/simba.txt" \ + "${RESULTS_GCS_PATH}/results/" + +gcloud storage cp \ + "$RESULTS_DIR/current_bq.txt" \ + "${RESULTS_GCS_PATH}/results/" + +gcloud storage cp \ + "$RESULTS_DIR/main_bq.txt" \ + "${RESULTS_GCS_PATH}/results/" + +gcloud storage cp \ + "$SUMMARY_FILE" \ + "${RESULTS_GCS_PATH}/results/" + +echo +echo "============================================================" +echo "Benchmark completed successfully." +echo "============================================================" +echo +echo "Results:" +echo "${RESULTS_GCS_PATH}/results/simba.txt" +echo "${RESULTS_GCS_PATH}/results/current_bq.txt" +echo "${RESULTS_GCS_PATH}/results/main_bq.txt" +echo "${RESULTS_GCS_PATH}/results/benchmark_summary.txt" \ No newline at end of file diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index b23dc6a2f5..5d36db3433 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -1,6 +1,3 @@ -# linux-bq-driver-benchmark.sh - -```bash #!/bin/bash # # Copyright 2025 Google LLC @@ -23,106 +20,58 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/bazel.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) -BUILD_DIR="${WORKSPACE_DIR}/cmake-out" - -# Number of times to run the COMPLETE performance_test suite. -# Default 1 means every performance test runs exactly once. -BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" - -# Branch used for storing current benchmark results. -CURRENT_BRANCH="${BRANCH_NAME:-main}" - -SANITIZED_BRANCH=$(echo "${CURRENT_BRANCH}" | \ - sed 's/[^a-zA-Z0-9._-]/_/g') - -# ------------------------------------------------------------ -# Benchmark result locations -# ------------------------------------------------------------ - -RESULTS_DIR="${WORKSPACE_DIR}/benchmark_results" -mkdir -p "${RESULTS_DIR}" - -SIMBA_RESULTS="${RESULTS_DIR}/performance_benchmark_results_Core.txt" -GOOGLE_RESULTS="${RESULTS_DIR}/performance_benchmark_results_BqDriver.txt" -MAIN_GOOGLE_RESULTS="${RESULTS_DIR}/main_bq.txt" - -SUMMARY_FILE="${RESULTS_DIR}/benchmark_summary_table.txt" - -# ------------------------------------------------------------ -# Header -# ------------------------------------------------------------ - -echo "============================================================" -echo "Linux ODBC Performance Benchmark Comparison" -echo "============================================================" -echo "Current branch : ${CURRENT_BRANCH}" -echo "Benchmark iterations : ${BENCHMARK_ITERATIONS}" -echo "Build directory : ${BUILD_DIR}" -echo -echo "Comparison:" -echo " 1. Simba Driver - Current Branch" -echo " 2. Google Driver - Current Branch" -echo " 3. Google Driver - Main Branch" -echo "============================================================" - -# ------------------------------------------------------------ -# Vcpkg -# ------------------------------------------------------------ +# Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION +echo "Using VCPKG_VERSION=$VCPKG_VERSION" -echo "Using VCPKG_VERSION=${VCPKG_VERSION}" - +# Vcpkg install and configure export VCPKG_ROOT=/vcpkg +git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" +cd "$VCPKG_ROOT" +git checkout "$VCPKG_VERSION" -if [[ ! -d "${VCPKG_ROOT}/.git" ]]; then - git clone \ - --branch "${VCPKG_VERSION}" \ - https://github.com/microsoft/vcpkg.git \ - "${VCPKG_ROOT}" -fi - -cd "${VCPKG_ROOT}" -git checkout "${VCPKG_VERSION}" +# Bootstrap +./bootstrap-vcpkg.sh -disableMetrics -if [[ ! -f "${VCPKG_ROOT}/vcpkg" ]]; then - ./bootstrap-vcpkg.sh -disableMetrics -fi - -cd "${WORKSPACE_DIR}" +cd "$WORKSPACE_DIR" +# This runs all the unit tests +mapfile -t args < <(bazel::common_args) +mapfile -t unit_tests_args < <(unit_tests::bazel_args) +mapfile -t secrets_bazel < <(secrets::bazel_args) -# ------------------------------------------------------------ -# CMake arguments -# ------------------------------------------------------------ +io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... +# Run the integration tests mapfile -t cmake_args < <(cmake::common_args) -# ------------------------------------------------------------ -# Configure the normal Google Driver build -# -# DO NOT use BUILD_PERFORMANCE_TEST_ONLY here. -# -# The benchmark executable must be able to test the current -# branch Google driver, so the normal driver build is required. -# ------------------------------------------------------------ - -echo -echo "============================================================" -echo "Configuring current branch build" -echo "============================================================" +BUILD_DIR="/opt/odbc-driver" +# This is the name of DSN set in odbc.ini +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} -rm -rf "${BUILD_DIR}" +# Check if unixODBC is installed +if command -v odbcinst &>/dev/null; then + # unixODBC is installed, export environment variable + export UNIXODBC_INSTALLED=true + echo "unixODBC is installed." +else + # unixODBC is not installed + export UNIXODBC_INSTALLED=false + export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini + echo "unixODBC is not installed." +fi -io::run cmake \ - -S "${WORKSPACE_DIR}" \ - -B "${BUILD_DIR}" \ +io::run cmake -B "$BUILD_DIR" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_STANDARD=17 \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ @@ -130,552 +79,42 @@ io::run cmake \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF +io::run cmake --build cmake-out -# ------------------------------------------------------------ -# Build ONLY performance_test -# -# This does not run ctest and does not explicitly build every -# project target. CMake builds performance_test and its required -# dependencies. -# ------------------------------------------------------------ - -echo -echo "============================================================" -echo "Building performance_test" -echo "============================================================" - -io::run cmake \ - --build "${BUILD_DIR}" \ - --target performance_test \ - --parallel "$(nproc)" - -# ------------------------------------------------------------ -# Locate performance_test -# ------------------------------------------------------------ - -PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" - -if [[ ! -f "${PERFORMANCE_TEST}" ]]; then - echo - echo "ERROR: performance_test was not found:" - echo " ${PERFORMANCE_TEST}" - echo - echo "Matching files:" - find "${BUILD_DIR}" -type f -name "performance_test*" -print || true - exit 1 -fi - -echo -echo "Performance test executable:" -echo " ${PERFORMANCE_TEST}" - -# ------------------------------------------------------------ -# Verify current branch Google Driver -# ------------------------------------------------------------ +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- -GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" -if [[ ! -f "${GOOGLE_DRIVER}" ]]; then - echo - echo "ERROR: Current branch Google Driver was not generated:" - echo " ${GOOGLE_DRIVER}" - echo - echo "Matching driver files:" - find "${BUILD_DIR}" \ - -type f \ - \( -name "*google_cloud_odbc_bq_driver*.so" \ - -o -name "*bq_driver*.so" \) \ - -print || true +if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" exit 1 fi -echo -echo "Current branch Google Driver:" -echo " ${GOOGLE_DRIVER}" - -# ------------------------------------------------------------ -# Copy roots.pem for Google Driver -# ------------------------------------------------------------ - -GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" - -if [[ -f "/opt/odbc-driver/roots.pem" ]]; then - io::run cp \ - "/opt/odbc-driver/roots.pem" \ - "${GOOGLE_DRIVER_DIR}/roots.pem" -else - echo - echo "ERROR: Required certificate file was not found:" - echo " /opt/odbc-driver/roots.pem" - exit 1 -fi - -# ------------------------------------------------------------ -# ODBC environment -# ------------------------------------------------------------ - -export ODBCINI="/opt/odbc-driver/odbc.ini" - -if [[ ! -f "${ODBCINI}" ]]; then - echo - echo "ERROR: ODBC configuration was not found:" - echo " ${ODBCINI}" - exit 1 -fi - -echo -echo "Using ODBCINI=${ODBCINI}" - -# ------------------------------------------------------------ -# unixODBC / iODBC setup -# ------------------------------------------------------------ - -if command -v odbcinst >/dev/null 2>&1; then - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." -else - export UNIXODBC_INSTALLED=false - export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" - echo "unixODBC is not installed." -fi - -# ------------------------------------------------------------ -# Unique test table prefix -# ------------------------------------------------------------ - -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=\ -"${TRIGGER_NAME:-benchmark}_${SANITIZED_BRANCH}" - -# ------------------------------------------------------------ -# Helper: verify a DSN exists -# ------------------------------------------------------------ - -verify_dsn() { - local dsn="$1" - - if ! grep -q "^\[${dsn}\]$" "${ODBCINI}"; then - echo - echo "ERROR: DSN not found in ${ODBCINI}:" - echo " [${dsn}]" - echo - echo "Available DSNs:" - grep '^\[.*\]$' "${ODBCINI}" || true - exit 1 - fi -} - -# ------------------------------------------------------------ -# Helper: run complete performance_test suite -# -# One invocation of performance_test runs every test registered -# in that executable. -# -# BENCHMARK_ITERATIONS=1: -# every performance_test test executes once. -# ------------------------------------------------------------ - -run_performance_suite() { - local driver_name="$1" - local dsn="$2" - local output_file="$3" - - export ODBC_TESTS_DSN="${dsn}" - - verify_dsn "${ODBC_TESTS_DSN}" - - echo - echo "============================================================" - echo "Running ${driver_name} performance suite" - echo "============================================================" - echo "DSN : ${ODBC_TESTS_DSN}" - echo "Iterations: ${BENCHMARK_ITERATIONS}" - echo "Executable: ${PERFORMANCE_TEST}" - echo "Output : ${output_file}" - echo "============================================================" - - : > "${output_file}" - - local test_exit_code=0 - local run_exit=0 - - for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - - echo - echo "${driver_name}: iteration ${i}/${BENCHMARK_ITERATIONS}" - - echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${output_file}" - - set +e - - "${PERFORMANCE_TEST}" \ - >> "${output_file}" \ - 2>&1 - - run_exit=$? - - set -e - - if [[ ${run_exit} -ne 0 ]]; then - echo - echo "WARNING: ${driver_name} iteration ${i} failed with exit code ${run_exit}" - test_exit_code=${run_exit} - fi - done - - if [[ ${test_exit_code} -ne 0 ]]; then - echo - echo "ERROR: ${driver_name} performance_test failed." - echo - echo "Last benchmark output:" - tail -n 50 "${output_file}" || true - return "${test_exit_code}" - fi -} - -# ------------------------------------------------------------ -# 1. Run Simba Driver - Current Branch -# -# Simba is loaded through the existing SampleDSN configuration. -# ------------------------------------------------------------ - -run_performance_suite \ - "Simba Driver (Current)" \ - "SampleDSN" \ - "${SIMBA_RESULTS}" - -# ------------------------------------------------------------ -# 2. Run Google Driver - Current Branch -# -# IMPORTANT: -# -# SampleDSNGoogleDriver must point to: -# -# ${GOOGLE_DRIVER} -# -# in /opt/odbc-driver/odbc.ini. -# ------------------------------------------------------------ - -run_performance_suite \ - "Google Driver (Current)" \ - "SampleDSNGoogleDriver" \ - "${GOOGLE_RESULTS}" +echo "Google driver found:" +ls -lh "$DRIVER_SO" -# ------------------------------------------------------------ -# 3. Download Google Driver Main Branch results -# -# Main benchmark results are produced by the main branch build -# and used as the baseline here. -# ------------------------------------------------------------ - -echo -echo "============================================================" -echo "Downloading Google Driver main branch benchmark results" -echo "============================================================" - -set +e - -gcloud storage cp \ - "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ - "${MAIN_GOOGLE_RESULTS}" - -MAIN_DOWNLOAD_EXIT=$? - -set -e - -if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then - echo - echo "WARNING: Main branch benchmark result was not available." - rm -f "${MAIN_GOOGLE_RESULTS}" -fi - -# ------------------------------------------------------------ -# Generate comparison table -# ------------------------------------------------------------ - -echo -echo "============================================================" -echo "Generating benchmark comparison" -echo "============================================================" - -SIMBA_FILE="${SIMBA_RESULTS}" \ -GOOGLE_FILE="${GOOGLE_RESULTS}" \ -MAIN_FILE="${MAIN_GOOGLE_RESULTS}" \ -OUTPUT_FILE="${SUMMARY_FILE}" \ -python3 <<'PYTHON' -import os -import re - - -SIMBA_FILE = os.environ["SIMBA_FILE"] -GOOGLE_FILE = os.environ["GOOGLE_FILE"] -MAIN_FILE = os.environ["MAIN_FILE"] -OUTPUT_FILE = os.environ["OUTPUT_FILE"] - - -def clean_test_name(name): - # GTest name: - # [Instantiation/]TestSuite.TestCase[/Param] - # - # Remove the suite/instantiation prefix so corresponding - # benchmark cases can be compared. - - if "." in name: - name = name.split(".", 1)[1] - - # Preserve compatibility with older output. - name = re.sub(r"/(?:With|Without)HTAPI$", "", name) - - return name - - -def parse_time_to_ms(value): - if not value: - return None - - value = value.strip() - - match = re.match(r"^([\d.]+)\s*(\w+)$", value) - - if not match: - return None - - number = float(match.group(1)) - unit = match.group(2).lower() - - if unit == "s": - return number * 1000.0 - - if unit == "ms": - return number - - if unit == "us": - return number / 1000.0 - - if unit == "ns": - return number / 1000000.0 - - return None - - -def parse_gtest_output(filename): - samples = {} - - if not os.path.exists(filename): - return {} - - pattern = re.compile( - r"\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)" - ) - - with open(filename, "r", errors="replace") as file: - for line in file: - match = pattern.search(line) - - if not match: - continue - - test_name = clean_test_name(match.group(1)) - duration_ms = parse_time_to_ms(match.group(2)) - - if duration_ms is None: - continue - - samples.setdefault(test_name, []).append(duration_ms) - - results = {} - - for test_name, values in samples.items(): - values.sort() - - count = len(values) - - if count % 2 == 1: - median = values[count // 2] - else: - median = ( - values[count // 2 - 1] + - values[count // 2] - ) / 2.0 - - results[test_name] = median - - return results - - -def percentage(value, reference): - if value is None or reference is None or reference == 0: - return "N/A" - - change = ((value - reference) / reference) * 100.0 - - return f"{change:+.0f}%" - - -simba = parse_gtest_output(SIMBA_FILE) -google = parse_gtest_output(GOOGLE_FILE) -main_google = parse_gtest_output(MAIN_FILE) - -all_tests = sorted( - set(simba) | - set(google) | - set(main_google) -) - -rows = [] - -for test_name in all_tests: - simba_ms = simba.get(test_name) - google_ms = google.get(test_name) - main_ms = main_google.get(test_name) - - simba_value = ( - f"{simba_ms:.0f}ms" - if simba_ms is not None - else "N/A" - ) - - google_value = ( - f"{google_ms:.0f}ms" - if google_ms is not None - else "N/A" - ) - - main_value = ( - f"{main_ms:.0f}ms" - if main_ms is not None - else "N/A" - ) - - google_pct = percentage(google_ms, simba_ms) - main_pct = percentage(main_ms, google_ms) - - if google_ms is not None: - google_value += f" ({google_pct})" - - if main_ms is not None: - main_value += f" ({main_pct})" - - rows.append( - ( - test_name, - simba_value, - google_value, - main_value, - ) - ) - - -headers = ( - "Test Case", - "Simba Driver (Current)", - "Google Driver (Current)", - "Google Driver (Main)", +# Sanitize branch name for use in GCS path. +SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' ) -widths = [] - -for index, header in enumerate(headers): - width = len(header) +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" - for row in rows: - width = max(width, len(row[index])) +echo "Uploading Google driver artifact..." +echo "Branch: ${SANITIZED_BRANCH}" - widths.append(width) - - -table = [] - -description = ( - "Only tests executed by the performance_test executable are included. " - "Percentages in Google Driver (Current) are relative to Simba Driver " - "(Current). Percentages in Google Driver (Main) are relative to Google " - "Driver (Current). Negative values are faster and positive values are slower." -) - -table.append(description) -table.append("") - -table.append( - "| " + - " | ".join( - header.ljust(widths[index]) - for index, header in enumerate(headers) - ) + - " |" -) - -table.append( - "|-" + - "-|-".join( - "-" * widths[index] - for index in range(len(headers)) - ) + - "-|" -) - -for row in rows: - table.append( - "| " + - " | ".join( - row[index].ljust(widths[index]) - for index in range(len(headers)) - ) + - " |" - ) - - -output = "\n".join(table) + "\n" - -with open(OUTPUT_FILE, "w") as file: - file.write(output) - -print(output) -print(f"Compared {len(all_tests)} performance test cases.") - -PYTHON - -# ------------------------------------------------------------ -# Upload results -# ------------------------------------------------------------ - -RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" - -echo -echo "============================================================" -echo "Uploading benchmark results" -echo "============================================================" - -io::run gcloud storage cp \ - "${SIMBA_RESULTS}" \ - "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" - -io::run gcloud storage cp \ - "${GOOGLE_RESULTS}" \ - "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" +gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -io::run gcloud storage cp \ - "${SUMMARY_FILE}" \ - "${RESULTS_BUCKET}/benchmark_summary_table.txt" +echo "Google driver benchmark artifact uploaded:" +echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -echo -echo "============================================================" -echo "Linux benchmark comparison completed successfully" -echo "============================================================" -echo -echo "Simba current:" -echo " ${SIMBA_RESULTS}" -echo -echo "Google current:" -echo " ${GOOGLE_RESULTS}" -echo -echo "Google main:" -echo " ${MAIN_GOOGLE_RESULTS}" -echo -echo "Comparison:" -echo " ${SUMMARY_FILE}" -echo -echo "Uploaded to:" -echo " ${RESULTS_BUCKET}/" -echo "============================================================" -``` +# Copy the roots.pem file to the .so directory to run test cases. +cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +mapfile -t ctest_args < <(ctest::common_args) +io::run env -C cmake-out ctest "${ctest_args[@]}" diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..ea837a5aca --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 + +import argparse +import math +import re +import statistics +from pathlib import Path + + +TIME_RE = re.compile( + r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)" +) + + +def parse_gtest_output(path): + path = Path(path) + + if not path.exists(): + raise FileNotFoundError(f"Benchmark output not found: {path}") + + samples = {} + + for line in path.read_text(errors="replace").splitlines(): + match = TIME_RE.search(line) + + if not match: + continue + + test_name = match.group(1).strip() + value = float(match.group(2)) + unit = match.group(3) + + if unit == "s": + value *= 1000.0 + elif unit == "us": + value /= 1000.0 + elif unit == "ns": + value /= 1_000_000.0 + + samples.setdefault(test_name, []).append(value) + + return { + test_name: statistics.median(values) + for test_name, values in samples.items() + } + + +def format_ms(value): + if value is None: + return "N/A" + + if value >= 1000: + return f"{value / 1000:.3f}s" + + if value >= 1: + return f"{value:.3f}ms" + + return f"{value * 1000:.3f}us" + + +def percentage_change(new_value, old_value): + if old_value is None or old_value == 0 or new_value is None: + return "N/A" + + pct = ((new_value - old_value) / old_value) * 100 + + if abs(pct) < 0.01: + pct = 0 + + return f"{pct:+.2f}%" + + +def make_table(title, rows): + print() + print(title) + print("=" * len(title)) + + headers = [ + "Test Case", + "Simba", + "Google Current", + "Current vs Simba", + "Google Main", + "Main vs Current", + ] + + widths = [ + max( + len(headers[0]), + *(len(row[0]) for row in rows), + ), + max( + len(headers[1]), + *(len(row[1]) for row in rows), + ), + max( + len(headers[2]), + *(len(row[2]) for row in rows), + ), + max( + len(headers[3]), + *(len(row[3]) for row in rows), + ), + max( + len(headers[4]), + *(len(row[4]) for row in rows), + ), + max( + len(headers[5]), + *(len(row[5]) for row in rows), + ), + ] + + def format_row(values): + return " | ".join( + value.ljust(width) + for value, width in zip(values, widths) + ) + + print(format_row(headers)) + print("-+-".join("-" * width for width in widths)) + + for row in rows: + print(format_row(row)) + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--simba", + required=True, + help="Simba benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + args = parser.parse_args() + + simba = parse_gtest_output(args.simba) + current = parse_gtest_output(args.current) + main_branch = parse_gtest_output(args.main) + + test_names = sorted( + set(simba) + | set(current) + | set(main_branch) + ) + + rows = [] + + for test_name in test_names: + simba_ms = simba.get(test_name) + current_ms = current.get(test_name) + main_ms = main_branch.get(test_name) + + rows.append( + ( + test_name, + format_ms(simba_ms), + format_ms(current_ms), + percentage_change(current_ms, simba_ms), + format_ms(main_ms), + percentage_change(main_ms, current_ms), + ) + ) + + output_lines = [] + + headers = [ + "Test Case", + "Simba", + "Google Current", + "Current vs Simba", + "Google Main", + "Main vs Current", + ] + + widths = [ + max(len(headers[0]), *(len(row[0]) for row in rows)), + max(len(headers[1]), *(len(row[1]) for row in rows)), + max(len(headers[2]), *(len(row[2]) for row in rows)), + max(len(headers[3]), *(len(row[3]) for row in rows)), + max(len(headers[4]), *(len(row[4]) for row in rows)), + max(len(headers[5]), *(len(row[5]) for row in rows)), + ] + + def format_row(values): + return " | ".join( + value.ljust(width) + for value, width in zip(values, widths) + ) + + output_lines.append( + "ODBC Performance Benchmark Summary" + ) + output_lines.append( + "==================================" + ) + output_lines.append("") + output_lines.append(format_row(headers)) + output_lines.append( + "-+-".join("-" * width for width in widths) + ) + + for row in rows: + output_lines.append(format_row(row)) + + summary = "\n".join(output_lines) + + Path(args.output).write_text(summary + "\n") + + print() + print(summary) + + if not simba: + print( + "WARNING: No Simba benchmark results were found." + ) + + if not current: + print( + "ERROR: No current Google benchmark results were found." + ) + return 1 + + if not main_branch: + print( + "ERROR: No main Google benchmark results were found." + ) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/ci/gha/builds/lib/odbc.ini b/ci/gha/builds/lib/odbc.ini index 5092a47b40..879d7ced8b 100644 --- a/ci/gha/builds/lib/odbc.ini +++ b/ci/gha/builds/lib/odbc.ini @@ -8,7 +8,6 @@ TraceFile=/tmp/odbc.log [ODBC Data Sources] SampleDSNGoogleDriver=ODBC Driver for BigQuery ODBCTransactionsTestsDSN=ODBC Driver for BigQuery -SampleDSN = ODBC Driver for BigQuery [SampleDSNGoogleDriver] @@ -157,80 +156,3 @@ LargeResultsTempTableExpirationTime=3600000 # given. Result pages have at most as many rows as are specified by the RowsFetchedPerBlock # connection property. #HTAPI_MinActivationRatio=3 - -[SampleDSN] - -PropertyWithoutValue - -Value with equals=I=am=a=value - -# Description: DSN Description. -# This key is not necessary and is only to give a description of the data source. - Description = ODBC Driver for BigQuery - -# Driver: The location where the ODBC driver is installed to. - Driver = /opt/odbc-driver/googlebigqueryodbc/lib/libgooglebigqueryodbc_sb64.so - -# These values can be set here, or on the connection string. -# Catalog: The catalog to connect to. This is a required setting. -Catalog=bigquery-devtools-drivers - -# SQLDialect: The SQL Dialect to use. There are two SQL dialects: -# 0 = BigQuery Legacy SQL -# 1 = BigQuery Standard SQL (SQL 11) -SQLDialect=1 - -# OAuth Mechanism: The OAuth mechanism to use. There are two choices: -# 0 = Service Authentication -# 1 = User Authentication -# -# This is a required setting. -OAuthMechanism=0 - -# RefreshToken: The Refresh Token used. This can be generated from the Windows connection dialog. -# It can also be generated by executing the following steps: -# 1. Get an Authentication by logging into Google from the following URL: -# https://accounts.google.com/o/oauth2/auth?scope=https://www.googleapis.com/auth/bigquery&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=977385342095.apps.googleusercontent.com&hl=en&from_login=1&as=76356ac9e8ce640b&pli=1&authuser=0 -# 2. Run the get_refresh_token.sh shell script and pass in the Authentication Token received in step 1. -# 3. Copy the Refresh Token (the text on the right-side of the colon, without the trailing or leading spaces) from the output of the script. -# This is a required setting. -# RefreshToken= - -# Email: For Service Authentication, this is a required setting. It is your GENERATED service account email (not a typical Gmail account). -# It is unique and associated with at least one public/private key pair. -Email=bq-devtools-drivers-test@bigquery-devtools-drivers.iam.gserviceaccount.com - -# KeyFile Path: For Service Authentication, this is a required setting. This is the path to the stored keyfile (.p12). -KeyFilePath=/opt/odbc-driver/connection/key.json - -# Used to specify the full path of the PEM formatted file containing trusted SSL CA certificates. -# If an empty string is passed in for the configuration, the driver expects the trusted SSL CA -# certificates can be found in the file named cacerts.pem located in the same directory as the -# driver's shared library. -#TrustedCerts= - -# AllowLargeResults: When set to 1, the driver allows for result sets in responses to be larger than 128 MB. -AllowLargeResults=0 - -# LargeResultsDataSetId: DatasetId to store temporary tables created. This is a required setting if AllowLargeResults is set to 1. -LargeResultsDataSetId=_bqodbc_temp_tables - -# LargeResultsTempTableExpirationTime: Time in milliseconds before the temporary tables created expire. This is a required setting if AllowLargeResults is set to 1. -LargeResultsTempTableExpirationTime=3600000 - -# EnableHTAPI: This set to 1 by default. -#EnableHTAPI=1 - -# HTAPI_MinResultsSize: An integer representing the minimum required number of rows in the results of -# a query to enable the High-Throughput API. This means that when the results of a query are -# gathered, they must have at least this many rows or the High-Throughput API will not be used to -# retrieve them. -#HTAPI_MinResultsSize=1000 - -# HTAPI_MinActivationRatio: An integer representing the minimum required ratio of rows to block size -# to enable the High-Throughput API. This means that when the results from a query are gathered, -# if there are not at least N times as many rows in the results as there are in the first page of -# results, then the High-Throughput API will not be used to retrieve them, where N is the ratio -# given. Result pages have at most as many rows as are specified by the RowsFetchedPerBlock -# connection property. -#HTAPI_MinActivationRatio=3 From dcbdb1bdf3450a962d4c72828395813b828c6b8c Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 11:22:28 +0000 Subject: [PATCH 13/33] test --- .../integration_tests/odbc_driver_tests/connection_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc index 7d640c85b0..698485f7b1 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc @@ -988,7 +988,7 @@ TEST(ConnectionTest, SuccessForExternalAuthWithBYOIDProperties) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { +/*TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { auto conn = std::make_shared(); std::string conn_str = kDefaultConnectionString + @@ -1012,7 +1012,7 @@ TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { EXPECT_STREQ(user_email, kImpersonatedAccountEmail.c_str()); EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); -} +}*/ TEST(ConnectionTest, VerifyServiceAccountImpersonationEmailInvalidFails) { auto conn = std::make_shared(); From f36e300e96709af65f2d69eb882dbc32e6035684 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 11:52:43 +0000 Subject: [PATCH 14/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index fa541b2e34..52773e936a 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -11,7 +11,8 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/io.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) From 86a2c055288390f4142b8aa829472e194af5316b Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 13:26:35 +0000 Subject: [PATCH 15/33] test --- .../builds/integration-bq-driver-asan.sh | 509 +++++++++--------- .../integration-bq-driver-asan-ci.yaml | 2 +- 2 files changed, 248 insertions(+), 263 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 52773e936a..e4a0c73ecc 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -8,10 +8,6 @@ set -euo pipefail source "$(dirname "$0")/../../lib/init.sh" -source module ci/install-dependencies.sh - -source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/secrets.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) @@ -20,385 +16,374 @@ WORKSPACE_DIR=$(pwd) # Configuration # --------------------------------------------------------------------------- -BUILD_DIR="/opt/odbc-performance" - -PERF_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" -DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" - BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" -CURRENT_BRANCH="${BRANCH_NAME:-main}" +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" -SANITIZED_BRANCH=$( - echo "${CURRENT_BRANCH}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' -) +# performance_test is built by the current source tree. +BUILD_DIR="/opt/odbc-driver" -RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" +PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" -mkdir -p "$RESULTS_DIR" +# Depending on the CMake layout, use the binary generated in cmake-out +# if the above path does not exist. +if [[ ! -x "$PERFORMANCE_TEST" ]]; then + PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/integration_tests/performance_test" +fi + +if [[ ! -x "$PERFORMANCE_TEST" ]]; then + PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/integration_tests/performance_test" +fi echo "============================================================" -echo "ODBC Linux Performance Benchmark" -echo "============================================================" -echo "Current branch : ${CURRENT_BRANCH}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Results dir : ${RESULTS_DIR}" +echo "ODBC Performance Benchmark" echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo # --------------------------------------------------------------------------- -# Vcpkg +# Sanitize branch name # --------------------------------------------------------------------------- -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) -export VCPKG_VERSION -export VCPKG_ROOT=/vcpkg - -echo "Using VCPKG_VERSION=${VCPKG_VERSION}" - -git clone \ - --branch "$VCPKG_VERSION" \ - https://github.com/microsoft/vcpkg.git \ - "$VCPKG_ROOT" - -cd "$VCPKG_ROOT" - -git checkout "$VCPKG_VERSION" - -./bootstrap-vcpkg.sh -disableMetrics +SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +) -cd "$WORKSPACE_DIR" +echo "Sanitized branch: ${SANITIZED_BRANCH}" # --------------------------------------------------------------------------- -# Download Google driver artifacts +# Temporary benchmark directory # --------------------------------------------------------------------------- -CURRENT_SO="/tmp/libgoogle_cloud_odbc_bq_driver_current.so" -MAIN_SO="/tmp/libgoogle_cloud_odbc_bq_driver_main.so" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" -echo "Downloading current Google driver..." +rm -rf "$RESULTS_DIR" +mkdir -p "$RESULTS_DIR" -gcloud storage cp \ - "${DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" \ - "$CURRENT_SO" +CURRENT_RESULT="${RESULTS_DIR}/current.txt" +MAIN_RESULT="${RESULTS_DIR}/main.txt" +SIMBA_RESULT="${RESULTS_DIR}/simba.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary.txt" -echo "Downloading main Google driver..." +# --------------------------------------------------------------------------- +# Google driver .so locations +# --------------------------------------------------------------------------- -gcloud storage cp \ - "${DRIVER_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" \ - "$MAIN_SO" +CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" -ls -lh \ - "$CURRENT_SO" \ - "$MAIN_SO" +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" # --------------------------------------------------------------------------- -# Locate Simba driver -# --------------------------------------------------------------------------- +# Simba configuration # -# The Simba integration image installs the Simba driver. -# We intentionally don't use another DSN. +# The Simba dependency setup already installs: # -# We find the .so installed by the Simba environment and save it as an -# artifact that can be selected at runtime. +# /opt/odbc-driver/googlebigqueryodbc/odbc.ini +# +# and sets: +# +# ODBCINI=/opt/odbc-driver/googlebigqueryodbc/odbc.ini +# +# We intentionally do not create another DSN. # --------------------------------------------------------------------------- -echo -echo "Searching for Simba ODBC driver..." - -SIMBA_SO="" - -while IFS= read -r candidate; do - if [[ -f "$candidate" ]]; then - case "$candidate" in - *google*cloud*odbc*.so|*Simba*.so|*simba*.so) - SIMBA_SO="$candidate" - break - ;; - esac - fi -done < <( - find \ - /opt \ - /usr \ - /workspace \ - -type f \ - -name "*.so" \ - 2>/dev/null | - sort -) - -if [[ -z "$SIMBA_SO" ]]; then - echo "ERROR: Could not locate Simba .so" - echo - echo "Available ODBC shared libraries:" - find \ - /opt \ - /usr \ - /workspace \ - -type f \ - -name "*.so*" \ - 2>/dev/null | - grep -Ei 'odbc|simba' | - sort || - true +SIMBA_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" +if [[ ! -f "$SIMBA_ODBCINI" ]]; then + echo "ERROR: Simba odbc.ini was not found:" + echo " ${SIMBA_ODBCINI}" exit 1 fi -echo "Simba driver found:" -echo " $SIMBA_SO" - -SIMBA_COPY="/tmp/libgoogle_cloud_odbc_simba_driver.so" - -cp "$SIMBA_SO" "$SIMBA_COPY" - -ls -lh "$SIMBA_COPY" - # --------------------------------------------------------------------------- -# Build ONLY performance_test +# Download current Google driver # --------------------------------------------------------------------------- -# -# BUILD_PERFORMANCE_TEST_ONLY does not build the Google driver. -# It builds performance_test + the small testing utility dependencies. -# -# The driver .so is selected at runtime through the DSN. -# --------------------------------------------------------------------------- - -mapfile -t cmake_args < <(cmake::common_args) - -io::run cmake \ - -S "$WORKSPACE_DIR" \ - -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_CXX_STANDARD=17 \ - -DBUILD_PERFORMANCE_TEST_ONLY=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON -io::run cmake \ - --build "$BUILD_DIR" \ - --target performance_test \ - --parallel "$(nproc)" - -PERFORMANCE_TEST="$BUILD_DIR/integration_tests/performance_test" +echo +echo "============================================================" +echo "Downloading Google driver for current branch" +echo "============================================================" -if [[ ! -x "$PERFORMANCE_TEST" ]]; then - echo "ERROR: performance_test was not created." +echo "GCS:" +echo " ${CURRENT_SO_GCS}" - echo "Searching build directory:" - find "$BUILD_DIR" \ - -type f \ - -name "performance_test*" \ - -print +gcloud storage cp \ + "$CURRENT_SO_GCS" \ + "$CURRENT_SO" +if [[ ! -f "$CURRENT_SO" ]]; then + echo "ERROR: Current Google driver was not downloaded." exit 1 fi -echo "performance_test:" -ls -lh "$PERFORMANCE_TEST" +ls -lh "$CURRENT_SO" # --------------------------------------------------------------------------- -# Configure one DSN +# Download Google driver from main # --------------------------------------------------------------------------- -# -# We use ONE DSN. -# -# Before each benchmark run, replace the driver .so referenced by the DSN. -# --------------------------------------------------------------------------- - -DSN_DIR="/tmp/benchmark-odbc" - -mkdir -p "$DSN_DIR" -ODBC_INI="$DSN_DIR/odbc.ini" - -cat > "$ODBC_INI" </dev/null || true -if [[ -f "/opt/odbc-driver/connection/key.json" ]]; then - export GOOGLE_APPLICATION_CREDENTIALS="/opt/odbc-driver/connection/key.json" - export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY="/opt/odbc-driver/connection/key.json" -else - echo "ERROR: Service account key was not found:" - echo " /opt/odbc-driver/connection/key.json" exit 1 fi -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" +echo "performance_test:" +echo " ${PERFORMANCE_TEST}" # --------------------------------------------------------------------------- -# Run benchmark helper +# Helper: run benchmark # --------------------------------------------------------------------------- -run_driver_benchmark() { +run_benchmark() { local name="$1" - local source_so="$2" - local output_file="$3" + local output_file="$2" + local driver_so="$3" + local dsn="$4" echo echo "============================================================" - echo "Benchmark: ${name}" - echo "Driver: ${source_so}" - echo "Iterations: ${BENCHMARK_ITERATIONS}" - echo "Output: ${output_file}" + echo "Running benchmark: ${name}" echo "============================================================" - - if [[ ! -f "$source_so" ]]; then - echo "ERROR: Driver .so does not exist:" - echo "$source_so" - return 1 - fi - - cp \ - "$source_so" \ - "${DRIVER_RUNTIME_DIR}/libgoogle_cloud_odbc_bq_driver.so" - - echo "Runtime driver:" - ls -lh \ - "${DRIVER_RUNTIME_DIR}/libgoogle_cloud_odbc_bq_driver.so" + echo "Driver:" + echo " ${driver_so}" + echo "DSN:" + echo " ${dsn}" + echo "Iterations:" + echo " ${BENCHMARK_ITERATIONS}" : > "$output_file" local test_exit_code=0 - for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo + echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - | tee -a "$output_file" + >> "$output_file" set +e - "$PERFORMANCE_TEST" \ - >> "$output_file" \ - 2>&1 + ODBCINI="$dsn" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "$PERFORMANCE_TEST" \ + >> "$output_file" 2>&1 - local run_exit_code=$? + run_exit=$? set -e - if [[ "$run_exit_code" -ne 0 ]]; then - echo \ - "WARNING: ${name} iteration ${i} exited with ${run_exit_code}" - - echo \ - "WARNING: ${name} iteration ${i} exited with ${run_exit_code}" \ - >> "$output_file" - - test_exit_code="$run_exit_code" + if [[ "$run_exit" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + test_exit_code="$run_exit" fi done + echo + echo "Raw result:" + echo " ${output_file}" + if [[ "$test_exit_code" -ne 0 ]]; then echo "ERROR: ${name} benchmark failed." return "$test_exit_code" fi - echo - echo "${name} benchmark completed." - return 0 } # --------------------------------------------------------------------------- -# Run Simba +# 1. Google Current +# +# Reuse the existing Google DSN. +# We don't create a new DSN. +# +# The only thing changed is the driver .so referenced by the DSN. # --------------------------------------------------------------------------- -run_driver_benchmark \ - "Simba" \ - "$SIMBA_COPY" \ - "$RESULTS_DIR/simba.txt" +echo +echo "============================================================" +echo "Preparing Google Current benchmark" +echo "============================================================" + +# Backup the existing odbc.ini because it contains the current Google DSN. +GOOGLE_ODBCINI="${RESULTS_DIR}/google_odbc.ini" + +if [[ -n "${ODBCINI:-}" && -f "${ODBCINI}" ]]; then + cp "${ODBCINI}" "$GOOGLE_ODBCINI" +else + GOOGLE_ODBCINI="${WORKSPACE_DIR}/odbc.ini" + + if [[ ! -f "$GOOGLE_ODBCINI" ]]; then + echo "ERROR: Google driver odbc.ini was not found." + echo "ODBCINI=${ODBCINI:-}" + exit 1 + fi +fi + +echo "Google ODBC configuration:" +echo " ${GOOGLE_ODBCINI}" # --------------------------------------------------------------------------- -# Run current Google driver +# IMPORTANT: +# Replace only the Driver= line in the existing Google DSN. +# +# We are NOT creating another DSN. # --------------------------------------------------------------------------- -run_driver_benchmark \ +GOOGLE_CURRENT_ODBCINI="${RESULTS_DIR}/google_current_odbc.ini" + +sed \ + "s|^Driver=.*|Driver=${CURRENT_SO}|" \ + "$GOOGLE_ODBCINI" \ + > "$GOOGLE_CURRENT_ODBCINI" + +export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" + +run_benchmark \ "Google Current" \ + "$CURRENT_RESULT" \ "$CURRENT_SO" \ - "$RESULTS_DIR/current_bq.txt" + "$GOOGLE_CURRENT_ODBCINI" # --------------------------------------------------------------------------- -# Run main Google driver +# 2. Google Main # --------------------------------------------------------------------------- -run_driver_benchmark \ +echo +echo "============================================================" +echo "Preparing Google Main benchmark" +echo "============================================================" + +GOOGLE_MAIN_ODBCINI="${RESULTS_DIR}/google_main_odbc.ini" + +sed \ + "s|^Driver=.*|Driver=${MAIN_SO}|" \ + "$GOOGLE_ODBCINI" \ + > "$GOOGLE_MAIN_ODBCINI" + +run_benchmark \ "Google Main" \ + "$MAIN_RESULT" \ "$MAIN_SO" \ - "$RESULTS_DIR/main_bq.txt" + "$GOOGLE_MAIN_ODBCINI" # --------------------------------------------------------------------------- -# Generate benchmark summary +# 3. Simba +# +# Simba's existing dependency setup already provides: +# +# /opt/odbc-driver/googlebigqueryodbc/odbc.ini +# +# Do not create another DSN. # --------------------------------------------------------------------------- -SUMMARY_FILE="$RESULTS_DIR/benchmark_summary.txt" +echo +echo "============================================================" +echo "Preparing Simba benchmark" +echo "============================================================" -python3 \ - "$WORKSPACE_DIR/ci/cloudbuild/builds/lib/benchmark_results.py" \ - --simba "$RESULTS_DIR/simba.txt" \ - --current "$RESULTS_DIR/current_bq.txt" \ - --main "$RESULTS_DIR/main_bq.txt" \ - --output "$SUMMARY_FILE" +export ODBC_TESTS_DSN="${SIMBA_ODBC_TESTS_DSN:-SampleDSN}" + +run_benchmark \ + "Simba" \ + "$SIMBA_RESULT" \ + "/opt/odbc-driver/googlebigqueryodbc" \ + "$SIMBA_ODBCINI" # --------------------------------------------------------------------------- -# Upload results +# Parse results using EXISTING benchmark_results.py # --------------------------------------------------------------------------- -RESULTS_GCS_PATH="${PERF_BUCKET}/${SANITIZED_BRANCH}" - echo -echo "Uploading benchmark results..." +echo "============================================================" +echo "Generating benchmark comparison" +echo "============================================================" -gcloud storage cp \ - "$RESULTS_DIR/simba.txt" \ - "${RESULTS_GCS_PATH}/results/" +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/scripts/benchmark_results.py" -gcloud storage cp \ - "$RESULTS_DIR/current_bq.txt" \ - "${RESULTS_GCS_PATH}/results/" +if [[ ! -f "$PARSER" ]]; then + echo "ERROR: Existing benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi -gcloud storage cp \ - "$RESULTS_DIR/main_bq.txt" \ - "${RESULTS_GCS_PATH}/results/" +python3 "$PARSER" \ + --simba "$SIMBA_RESULT" \ + --current "$CURRENT_RESULT" \ + --main "$MAIN_RESULT" \ + --output "$SUMMARY_RESULT" -gcloud storage cp \ - "$SUMMARY_FILE" \ - "${RESULTS_GCS_PATH}/results/" +echo +echo "============================================================" +echo "Benchmark Summary" +echo "============================================================" + +cat "$SUMMARY_RESULT" + +# --------------------------------------------------------------------------- +# Upload raw results + summary +# --------------------------------------------------------------------------- + +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/benchmarks" echo echo "============================================================" -echo "Benchmark completed successfully." +echo "Uploading benchmark results" echo "============================================================" + +gcloud storage cp \ + "$CURRENT_RESULT" \ + "$MAIN_RESULT" \ + "$SIMBA_RESULT" \ + "$SUMMARY_RESULT" \ + "${RESULTS_BUCKET}/" + +echo +echo "Benchmark results uploaded to:" +echo " ${RESULTS_BUCKET}/" + echo -echo "Results:" -echo "${RESULTS_GCS_PATH}/results/simba.txt" -echo "${RESULTS_GCS_PATH}/results/current_bq.txt" -echo "${RESULTS_GCS_PATH}/results/main_bq.txt" -echo "${RESULTS_GCS_PATH}/results/benchmark_summary.txt" \ No newline at end of file +echo "============================================================" +echo "Benchmark completed successfully" +echo "============================================================" \ No newline at end of file diff --git a/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml b/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml index 9679ab1118..09c0051043 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml @@ -21,7 +21,7 @@ github: name: integration-bq-driver-asan-ci substitutions: _BUILD_NAME: integration-bq-driver-asan - _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER,DRIVER_MANAGER_SETUP' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS From 572a37c41be250dfd277d3ca22b08ff2dc254775 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 13:39:07 +0000 Subject: [PATCH 16/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index e4a0c73ecc..838aa14dcd 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -5,9 +5,14 @@ # Licensed under the Apache License, Version 2.0 # + set -euo pipefail source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/secrets.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) From 9adcb7fdc93a26abb8485d637c252ade43ad1579 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 14:10:29 +0000 Subject: [PATCH 17/33] test --- .../integration-bq-driver-asan-ci.yaml | 2 +- .../driver-manager-setup-google-driver.sh | 164 ++++++++++++++---- 2 files changed, 129 insertions(+), 37 deletions(-) diff --git a/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml b/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml index 09c0051043..9679ab1118 100644 --- a/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml +++ b/ci/cloudbuild/triggers/integration-bq-driver-asan-ci.yaml @@ -21,7 +21,7 @@ github: name: integration-bq-driver-asan-ci substitutions: _BUILD_NAME: integration-bq-driver-asan - _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER,DRIVER_MANAGER_SETUP' + _DEPENDENCIES: 'BAZEL,iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS diff --git a/ci/dependencies/driver-manager-setup-google-driver.sh b/ci/dependencies/driver-manager-setup-google-driver.sh index 17ca70f06c..fae1fe33c4 100644 --- a/ci/dependencies/driver-manager-setup-google-driver.sh +++ b/ci/dependencies/driver-manager-setup-google-driver.sh @@ -16,43 +16,135 @@ set -euo pipefail +# ============================================================================ +# Google BigQuery ODBC Driver +# ============================================================================ + # Make our include guard clean against set -o nounset. -test -n "${CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__:-}" || declare -i CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__=0 -if ((CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__++ != 0)); then - return 0 -fi # include guard -CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" -export CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR - -export GCS_BUCKET=bq-dev-tools-testing-drivers - -# Check gcloud is installed. -echo "Verifying google cloud SDK is installed using GCS Bucket: "${GCS_BUCKET} -if [ "$(gsutil ls gs://${GCS_BUCKET}/odbc | grep -c odbc-driver.zip)" -eq 0 ]; then - echo 'ODBC driver not found for download: exiting...' - exit 1 -fi +test -n "${CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__:-}" || \ + declare -i CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__=0 + +if ((CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__++ == 0)); then + + CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" + export CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR + + export GCS_BUCKET=bq-dev-tools-testing-drivers + + # Check Google driver is available. + echo "Verifying Google BigQuery ODBC driver using GCS Bucket: ${GCS_BUCKET}" + + if [ "$(gsutil ls "gs://${GCS_BUCKET}/odbc" | grep -c 'odbc-driver.zip')" -eq 0 ]; then + echo 'Google BigQuery ODBC driver not found for download: exiting...' + exit 1 + fi + + # Configure connection credentials. + echo 'Configuring Google driver connection credentials...' + + mkdir -p /opt/odbc-driver/connection + cd /opt/odbc-driver + + gcloud secrets versions access latest \ + --secret=service-account-auth-keys \ + --out-file="/opt/odbc-driver/connection/key.json" + + echo 'Verifying Google driver connection keys file size...' + + if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then + echo 'Invalid connection keys: exiting...' + exit 1 + fi + + # Configure Google driver environment variables. + echo 'Configuring environment variables for Google BigQuery ODBC driver...' + + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/usr/local/lib/" + export ODBCSYSINI=/opt/odbc-driver + export ODBCINI=/opt/odbc-driver/odbc.ini + export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/opt/odbc-driver/connection/key.json + export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc.ini + export GOOGLEBIGQUERYODBCINI_UTF16=/opt/odbc-driver/googlebigqueryodbc_utf16.ini + export GOOGLEBIGQUERYODBCINI_UTF8=/opt/odbc-driver/googlebigqueryodbc_utf8.ini + + cd "$CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" + + echo '**** Google BigQuery ODBC Driver setup END ****' -# Configure connection credentials for the driver. -echo 'Configuring Connection Credentials...' -mkdir -p /opt/odbc-driver/connection -cd /opt/odbc-driver -gcloud secrets versions access latest --secret=service-account-auth-keys --out-file="/opt/odbc-driver/connection/key.json" -echo 'Verifying Connection Keys File Size...' -if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then - echo 'Invalid connection keys: exiting...' - exit 1 fi -# Configure environment variables -echo 'Configuring Environment Variables For ODBC Driver...' -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib/ -export ODBCSYSINI=/opt/odbc-driver -export ODBCINI=/opt/odbc-driver/odbc.ini -export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/opt/odbc-driver/connection/key.json -export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc.ini -export GOOGLEBIGQUERYODBCINI_UTF16=/opt/odbc-driver/googlebigqueryodbc_utf16.ini -export GOOGLEBIGQUERYODBCINI_UTF8=/opt/odbc-driver/googlebigqueryodbc_utf8.ini -cd "$CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" - -echo '**** ODBC Driver installation END****' + +# ============================================================================ +# Simba ODBC Driver +# ============================================================================ + +# Make our include guard clean against set -o nounset. +test -n "${CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__:-}" || \ + declare -i CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__=0 + +if ((CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__++ == 0)); then + + CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" + export CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR + + export GCS_BUCKET=bq-dev-tools-testing-drivers + export DRIVER_VERSION=3.3.1.3003 + + # Check Simba driver is available. + echo "Verifying Simba ODBC driver using GCS Bucket: ${GCS_BUCKET}" + + if [ "$(gsutil ls "gs://${GCS_BUCKET}/odbc" | grep -c "odbc-driver.${DRIVER_VERSION}.zip")" -eq 0 ]; then + echo 'Simba ODBC driver not found for download: exiting...' + exit 1 + fi + + # Configure connection credentials. + echo 'Configuring Simba connection credentials...' + + mkdir -p /opt/odbc-driver/connection + cd /opt/odbc-driver + + gcloud secrets versions access latest \ + --secret=service-account-auth-keys \ + --out-file="/opt/odbc-driver/connection/key.json" + + echo 'Verifying Simba connection keys file size...' + + if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then + echo 'Invalid connection keys: exiting...' + exit 1 + fi + + # Install Simba ODBC driver. + echo 'Installing Simba ODBC driver...' + + gsutil -m cp \ + "gs://${GCS_BUCKET}/odbc/odbc-driver.${DRIVER_VERSION}.zip" \ + . + + unzip -qq "odbc-driver.${DRIVER_VERSION}.zip" + + echo 'Verifying Simba driver install directory...' + + if [ "$( + shopt -s nullglob + set -- /opt/odbc-driver/*googlebigqueryodbc* + echo $# + )" -eq 0 ]; then + echo 'Simba ODBC driver not installed: exiting...' + exit 1 + fi + + # Configure Simba environment variables. + echo 'Configuring environment variables for Simba ODBC driver...' + + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/usr/local/lib/" + export ODBCINI=/opt/odbc-driver/googlebigqueryodbc/odbc.ini + export ODBCINSTINI=/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini + export SIMBAGOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc/lib/simba.googlebigqueryodbc.ini + + cd "$CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" + + echo '**** Simba ODBC Driver setup END ****' + +fi \ No newline at end of file From ed16ab3b76974cdc6c550d06ef20ee1b5afcfe2a Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 14:35:52 +0000 Subject: [PATCH 18/33] test --- .../builds/integration-bq-driver-asan.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 838aa14dcd..2101a3e512 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -28,6 +28,29 @@ PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" # performance_test is built by the current source tree. BUILD_DIR="/opt/odbc-driver" +# --------------------------------------------------------------------------- +# Build performance_test +# --------------------------------------------------------------------------- + +echo +echo "============================================================" +echo "Building performance_test" +echo "============================================================" + +mapfile -t cmake_args < <(cmake::common_args) + +io::run cmake \ + -S "$WORKSPACE_DIR" \ + -B "$BUILD_DIR" \ + "${cmake_args[@]}" \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON + +io::run cmake \ + --build "$BUILD_DIR" \ + --target performance_test \ + --parallel "$(nproc)" + PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" # Depending on the CMake layout, use the binary generated in cmake-out From 2acf8a6e07b6e90eaed3c81c980007e4f04ab9dc Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 14:48:18 +0000 Subject: [PATCH 19/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 2101a3e512..597bfb5c49 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -26,7 +26,7 @@ BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" # performance_test is built by the current source tree. -BUILD_DIR="/opt/odbc-driver" +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" # --------------------------------------------------------------------------- # Build performance_test @@ -47,17 +47,11 @@ io::run cmake \ -DBUILD_PERFORMANCE_TEST_ONLY=ON io::run cmake \ - --build "$BUILD_DIR" \ + --build "$WORKSPACE_DIR/cmake-out" \ --target performance_test \ --parallel "$(nproc)" -PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" - -# Depending on the CMake layout, use the binary generated in cmake-out -# if the above path does not exist. -if [[ ! -x "$PERFORMANCE_TEST" ]]; then - PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/integration_tests/performance_test" -fi +PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/integration_tests/performance_test" if [[ ! -x "$PERFORMANCE_TEST" ]]; then PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/integration_tests/performance_test" From 484732964ee438349422fe3a4073b5338fc4a0bd Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 15:27:35 +0000 Subject: [PATCH 20/33] test --- .../builds/integration-bq-driver-asan.sh | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 597bfb5c49..7f3c0ea4fd 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -269,19 +269,12 @@ echo "Preparing Google Current benchmark" echo "============================================================" # Backup the existing odbc.ini because it contains the current Google DSN. -GOOGLE_ODBCINI="${RESULTS_DIR}/google_odbc.ini" +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" -if [[ -n "${ODBCINI:-}" && -f "${ODBCINI}" ]]; then - cp "${ODBCINI}" "$GOOGLE_ODBCINI" -else - GOOGLE_ODBCINI="${WORKSPACE_DIR}/odbc.ini" - if [[ ! -f "$GOOGLE_ODBCINI" ]]; then - echo "ERROR: Google driver odbc.ini was not found." - echo "ODBCINI=${ODBCINI:-}" - exit 1 - fi -fi +gcloud storage cp \ + "$CURRENT_SO_GCS" \ + "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" echo "Google ODBC configuration:" echo " ${GOOGLE_ODBCINI}" @@ -293,20 +286,14 @@ echo " ${GOOGLE_ODBCINI}" # We are NOT creating another DSN. # --------------------------------------------------------------------------- -GOOGLE_CURRENT_ODBCINI="${RESULTS_DIR}/google_current_odbc.ini" - -sed \ - "s|^Driver=.*|Driver=${CURRENT_SO}|" \ - "$GOOGLE_ODBCINI" \ - > "$GOOGLE_CURRENT_ODBCINI" export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" run_benchmark \ "Google Current" \ "$CURRENT_RESULT" \ - "$CURRENT_SO" \ - "$GOOGLE_CURRENT_ODBCINI" + "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" \ + "$GOOGLE_ODBCINI" # --------------------------------------------------------------------------- # 2. Google Main @@ -317,18 +304,18 @@ echo "============================================================" echo "Preparing Google Main benchmark" echo "============================================================" -GOOGLE_MAIN_ODBCINI="${RESULTS_DIR}/google_main_odbc.ini" +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" + -sed \ - "s|^Driver=.*|Driver=${MAIN_SO}|" \ - "$GOOGLE_ODBCINI" \ - > "$GOOGLE_MAIN_ODBCINI" +gcloud storage cp \ + "$MAIN_SO_GCS" \ + "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" run_benchmark \ "Google Main" \ "$MAIN_RESULT" \ - "$MAIN_SO" \ - "$GOOGLE_MAIN_ODBCINI" + "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" \ + "$GOOGLE_ODBCINI" # --------------------------------------------------------------------------- # 3. Simba @@ -362,7 +349,7 @@ echo "============================================================" echo "Generating benchmark comparison" echo "============================================================" -PARSER="${WORKSPACE_DIR}/ci/cloudbuild/scripts/benchmark_results.py" +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" if [[ ! -f "$PARSER" ]]; then echo "ERROR: Existing benchmark_results.py was not found:" From 4aa076de9cdf840f1c6d57258fd40284e83895e1 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 16:09:08 +0000 Subject: [PATCH 21/33] tset --- .../builds/integration-bq-driver-asan.sh | 313 ++++++++---------- 1 file changed, 137 insertions(+), 176 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 7f3c0ea4fd..1a2da945d3 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -5,7 +5,6 @@ # Licensed under the Apache License, Version 2.0 # - set -euo pipefail source "$(dirname "$0")/../../lib/init.sh" @@ -15,7 +14,7 @@ source module ci/cloudbuild/builds/lib/cmake.sh source module ci/cloudbuild/builds/lib/secrets.sh source module ci/lib/io.sh -WORKSPACE_DIR=$(pwd) +WORKSPACE_DIR="$(pwd)" # --------------------------------------------------------------------------- # Configuration @@ -25,75 +24,45 @@ BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" -# performance_test is built by the current source tree. BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +mkdir -p "${RESULTS_DIR}" # --------------------------------------------------------------------------- -# Build performance_test +# Branch # --------------------------------------------------------------------------- -echo -echo "============================================================" -echo "Building performance_test" -echo "============================================================" - -mapfile -t cmake_args < <(cmake::common_args) - -io::run cmake \ - -S "$WORKSPACE_DIR" \ - -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_CXX_STANDARD=17 \ - -DBUILD_PERFORMANCE_TEST_ONLY=ON +BRANCH_NAME="${BRANCH_NAME:-main}" -io::run cmake \ - --build "$WORKSPACE_DIR/cmake-out" \ - --target performance_test \ - --parallel "$(nproc)" - -PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/integration_tests/performance_test" - -if [[ ! -x "$PERFORMANCE_TEST" ]]; then - PERFORMANCE_TEST="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/integration_tests/performance_test" -fi +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +)" echo "============================================================" -echo "ODBC Performance Benchmark" +echo "Linux ODBC Performance Benchmark" echo "============================================================" -echo "Branch : ${BRANCH_NAME}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Workspace : ${WORKSPACE_DIR}" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" echo # --------------------------------------------------------------------------- -# Sanitize branch name -# --------------------------------------------------------------------------- - -SANITIZED_BRANCH=$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' -) - -echo "Sanitized branch: ${SANITIZED_BRANCH}" - -# --------------------------------------------------------------------------- -# Temporary benchmark directory +# Result files # --------------------------------------------------------------------------- -RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" - -rm -rf "$RESULTS_DIR" -mkdir -p "$RESULTS_DIR" - CURRENT_RESULT="${RESULTS_DIR}/current.txt" MAIN_RESULT="${RESULTS_DIR}/main.txt" SIMBA_RESULT="${RESULTS_DIR}/simba.txt" SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary.txt" # --------------------------------------------------------------------------- -# Google driver .so locations +# Driver locations # --------------------------------------------------------------------------- +DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" @@ -101,100 +70,109 @@ CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_b MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" # --------------------------------------------------------------------------- -# Simba configuration -# -# The Simba dependency setup already installs: -# -# /opt/odbc-driver/googlebigqueryodbc/odbc.ini -# -# and sets: -# -# ODBCINI=/opt/odbc-driver/googlebigqueryodbc/odbc.ini -# -# We intentionally do not create another DSN. +# ODBC configuration # --------------------------------------------------------------------------- +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" SIMBA_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -if [[ ! -f "$SIMBA_ODBCINI" ]]; then - echo "ERROR: Simba odbc.ini was not found:" - echo " ${SIMBA_ODBCINI}" - exit 1 -fi +export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" # --------------------------------------------------------------------------- -# Download current Google driver +# Build performance_test # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Downloading Google driver for current branch" +echo "Building performance_test" echo "============================================================" -echo "GCS:" -echo " ${CURRENT_SO_GCS}" +mapfile -t cmake_args < <(cmake::common_args) -gcloud storage cp \ - "$CURRENT_SO_GCS" \ - "$CURRENT_SO" +io::run cmake \ + -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + "${cmake_args[@]}" \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON + +io::run cmake \ + --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)" + +PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + +if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" +fi + +if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true -if [[ ! -f "$CURRENT_SO" ]]; then - echo "ERROR: Current Google driver was not downloaded." exit 1 fi -ls -lh "$CURRENT_SO" +echo "performance_test:" +echo " ${PERFORMANCE_TEST}" # --------------------------------------------------------------------------- -# Download Google driver from main +# Validate ODBC configuration +# --------------------------------------------------------------------------- + +if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 +fi + +if [[ ! -f "${SIMBA_ODBCINI}" ]]; then + echo "ERROR: Simba ODBC configuration not found:" + echo " ${SIMBA_ODBCINI}" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Download Google Current driver # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Downloading Google driver from main" +echo "Downloading Google Current driver" echo "============================================================" - echo "GCS:" -echo " ${MAIN_SO_GCS}" +echo " ${CURRENT_SO_GCS}" gcloud storage cp \ - "$MAIN_SO_GCS" \ - "$MAIN_SO" - -if [[ ! -f "$MAIN_SO" ]]; then - echo "ERROR: Main Google driver was not downloaded." - exit 1 -fi + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" -ls -lh "$MAIN_SO" +ls -lh "${CURRENT_SO}" # --------------------------------------------------------------------------- -# Locate performance_test +# Download Google Main driver # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Locating performance_test" +echo "Downloading Google Main driver" echo "============================================================" +echo "GCS:" +echo " ${MAIN_SO_GCS}" -if [[ ! -x "$PERFORMANCE_TEST" ]]; then - echo "ERROR: performance_test executable was not found." - - echo - echo "Searching cmake-out:" - find "${WORKSPACE_DIR}/cmake-out" \ - -type f \ - -name "performance_test" \ - -print 2>/dev/null || true - - exit 1 -fi +gcloud storage cp \ + "${MAIN_SO_GCS}" \ + "${MAIN_SO}" -echo "performance_test:" -echo " ${PERFORMANCE_TEST}" +ls -lh "${MAIN_SO}" # --------------------------------------------------------------------------- -# Helper: run benchmark +# Run benchmark # --------------------------------------------------------------------------- run_benchmark() { @@ -214,7 +192,7 @@ run_benchmark() { echo "Iterations:" echo " ${BENCHMARK_ITERATIONS}" - : > "$output_file" + : > "${output_file}" local test_exit_code=0 @@ -223,22 +201,22 @@ run_benchmark() { echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "$output_file" + >> "${output_file}" set +e - ODBCINI="$dsn" \ + ODBCINI="${dsn}" \ ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ - "$PERFORMANCE_TEST" \ - >> "$output_file" 2>&1 + "${PERFORMANCE_TEST}" \ + >> "${output_file}" 2>&1 run_exit=$? set -e - if [[ "$run_exit" -ne 0 ]]; then + if [[ "${run_exit}" -ne 0 ]]; then echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" - test_exit_code="$run_exit" + test_exit_code="${run_exit}" fi done @@ -246,102 +224,85 @@ run_benchmark() { echo "Raw result:" echo " ${output_file}" - if [[ "$test_exit_code" -ne 0 ]]; then - echo "ERROR: ${name} benchmark failed." - return "$test_exit_code" + if [[ "${test_exit_code}" -ne 0 ]]; then + echo + echo "============================================================" + echo "Benchmark failure output: ${name}" + echo "============================================================" + + cat "${output_file}" + + echo + echo "============================================================" + + return "${test_exit_code}" fi return 0 } # --------------------------------------------------------------------------- -# 1. Google Current -# -# Reuse the existing Google DSN. -# We don't create a new DSN. +# Google Current # -# The only thing changed is the driver .so referenced by the DSN. +# The existing Google DSN points to DRIVER_PATH. +# Replace the driver binary before running the benchmark. # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Preparing Google Current benchmark" +echo "Preparing Google Current" echo "============================================================" -# Backup the existing odbc.ini because it contains the current Google DSN. -GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" - - -gcloud storage cp \ - "$CURRENT_SO_GCS" \ - "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -echo "Google ODBC configuration:" -echo " ${GOOGLE_ODBCINI}" - -# --------------------------------------------------------------------------- -# IMPORTANT: -# Replace only the Driver= line in the existing Google DSN. -# -# We are NOT creating another DSN. -# --------------------------------------------------------------------------- +cp "${CURRENT_SO}" "${DRIVER_PATH}" +ls -lh "${DRIVER_PATH}" export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" run_benchmark \ "Google Current" \ - "$CURRENT_RESULT" \ - "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" \ - "$GOOGLE_ODBCINI" + "${CURRENT_RESULT}" \ + "${DRIVER_PATH}" \ + "${GOOGLE_ODBCINI}" # --------------------------------------------------------------------------- -# 2. Google Main +# Google Main # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Preparing Google Main benchmark" +echo "Preparing Google Main" echo "============================================================" -GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" - +cp "${MAIN_SO}" "${DRIVER_PATH}" -gcloud storage cp \ - "$MAIN_SO_GCS" \ - "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +ls -lh "${DRIVER_PATH}" run_benchmark \ "Google Main" \ - "$MAIN_RESULT" \ - "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" \ - "$GOOGLE_ODBCINI" + "${MAIN_RESULT}" \ + "${DRIVER_PATH}" \ + "${GOOGLE_ODBCINI}" # --------------------------------------------------------------------------- -# 3. Simba -# -# Simba's existing dependency setup already provides: -# -# /opt/odbc-driver/googlebigqueryodbc/odbc.ini -# -# Do not create another DSN. +# Simba # --------------------------------------------------------------------------- echo echo "============================================================" -echo "Preparing Simba benchmark" +echo "Preparing Simba" echo "============================================================" export ODBC_TESTS_DSN="${SIMBA_ODBC_TESTS_DSN:-SampleDSN}" run_benchmark \ "Simba" \ - "$SIMBA_RESULT" \ + "${SIMBA_RESULT}" \ "/opt/odbc-driver/googlebigqueryodbc" \ - "$SIMBA_ODBCINI" + "${SIMBA_ODBCINI}" # --------------------------------------------------------------------------- -# Parse results using EXISTING benchmark_results.py +# Generate comparison # --------------------------------------------------------------------------- echo @@ -351,27 +312,27 @@ echo "============================================================" PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" -if [[ ! -f "$PARSER" ]]; then - echo "ERROR: Existing benchmark_results.py was not found:" - echo " ${PARSER}" +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" exit 1 fi -python3 "$PARSER" \ - --simba "$SIMBA_RESULT" \ - --current "$CURRENT_RESULT" \ - --main "$MAIN_RESULT" \ - --output "$SUMMARY_RESULT" +python3 "${PARSER}" \ + --simba "${SIMBA_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --output "${SUMMARY_RESULT}" echo echo "============================================================" echo "Benchmark Summary" echo "============================================================" -cat "$SUMMARY_RESULT" +cat "${SUMMARY_RESULT}" # --------------------------------------------------------------------------- -# Upload raw results + summary +# Upload results # --------------------------------------------------------------------------- RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/benchmarks" @@ -382,14 +343,14 @@ echo "Uploading benchmark results" echo "============================================================" gcloud storage cp \ - "$CURRENT_RESULT" \ - "$MAIN_RESULT" \ - "$SIMBA_RESULT" \ - "$SUMMARY_RESULT" \ + "${CURRENT_RESULT}" \ + "${MAIN_RESULT}" \ + "${SIMBA_RESULT}" \ + "${SUMMARY_RESULT}" \ "${RESULTS_BUCKET}/" echo -echo "Benchmark results uploaded to:" +echo "Results uploaded to:" echo " ${RESULTS_BUCKET}/" echo From b3ef17b454d170c9e50e41312a8698b8503781ae Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 16:20:58 +0000 Subject: [PATCH 22/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 1a2da945d3..b6e6bad550 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -258,6 +258,7 @@ cp "${CURRENT_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" +export ODBCINI="${GOOGLE_ODBCINI}" run_benchmark \ "Google Current" \ @@ -274,6 +275,7 @@ echo "============================================================" echo "Preparing Google Main" echo "============================================================" +export ODBCINI="${GOOGLE_ODBCINI}" cp "${MAIN_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" @@ -294,6 +296,7 @@ echo "Preparing Simba" echo "============================================================" export ODBC_TESTS_DSN="${SIMBA_ODBC_TESTS_DSN:-SampleDSN}" +export ODBCINI="${SIMBA_ODBCINI}" run_benchmark \ "Simba" \ From 0ae3a60385d3531e360a9d5ee2d1e6198259db5a Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 16:31:51 +0000 Subject: [PATCH 23/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index b6e6bad550..615c883c6f 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -247,7 +247,7 @@ run_benchmark() { # The existing Google DSN points to DRIVER_PATH. # Replace the driver binary before running the benchmark. # --------------------------------------------------------------------------- - +cp /opt/odbc-driver/roots.pem "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/roots.pem" echo echo "============================================================" echo "Preparing Google Current" From 7af6b86217fccf1d5ced1ab335c759a222d89870 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 18:07:29 +0000 Subject: [PATCH 24/33] test --- .../examples/catalog_performance_example.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index 3e442a03a1..8fc1ff0b89 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -390,11 +390,11 @@ INSTANTIATE_TEST_SUITE_P( std::make_tuple("new_timestamp_table", "SELECT * FROM " "`bigquery-devtools-drivers.kirltest.new_timestamp_" - "table` LIMIT 1000000"), + "table` LIMIT 10"), std::make_tuple("all_bq_types_2", "SELECT * FROM " "`bigquery-devtools-drivers.INTEGRATION_TEST_FORMAT." - "all_bq_types_2` LIMIT 1000000"), + "all_bq_types_2` LIMIT 10"), std::make_tuple( "nyc311_service_requests", "SELECT nyc311.unique_key AS V1, nyc311.descriptor AS V2, " @@ -412,7 +412,7 @@ INSTANTIATE_TEST_SUITE_P( "CAST(nyc311.closed_date AS STRING) AS V22 FROM " "`bigquery-public-data.new_york_311.311_service_requests` AS " "nyc311 " - "LIMIT 1000000;") + "LIMIT 10;") // TODO: Re-enable this benchmark once HTAPI Arrow supports all data // types. Currently SQLExecDirect fails with: // "[Google][ODBC BigQuery Driver] Internal Error: Unsupported arrow From 08742f208a78df34cddf3c675db7898d713888cb Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 18:23:08 +0000 Subject: [PATCH 25/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 615c883c6f..5d11b15591 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -259,6 +259,7 @@ ls -lh "${DRIVER_PATH}" export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" export ODBCINI="${GOOGLE_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" run_benchmark \ "Google Current" \ @@ -276,6 +277,7 @@ echo "Preparing Google Main" echo "============================================================" export ODBCINI="${GOOGLE_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" cp "${MAIN_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" @@ -297,6 +299,7 @@ echo "============================================================" export ODBC_TESTS_DSN="${SIMBA_ODBC_TESTS_DSN:-SampleDSN}" export ODBCINI="${SIMBA_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" run_benchmark \ "Simba" \ From 44bf33d9886e78ab7fb2122350b966e160525bf4 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 18 Aug 2026 19:48:39 +0000 Subject: [PATCH 26/33] test --- .../builds/integration-bq-driver-asan.sh | 60 +++++-------------- ci/cloudbuild/builds/lib/benchmark_results.py | 26 ++++---- 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 5d11b15591..e992ccdcf2 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -54,7 +54,7 @@ echo CURRENT_RESULT="${RESULTS_DIR}/current.txt" MAIN_RESULT="${RESULTS_DIR}/main.txt" -SIMBA_RESULT="${RESULTS_DIR}/simba.txt" +EXISTING_RESULT="${RESULTS_DIR}/existing.txt" SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary.txt" # --------------------------------------------------------------------------- @@ -74,7 +74,7 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_d # --------------------------------------------------------------------------- GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" -SIMBA_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" @@ -131,9 +131,9 @@ if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then exit 1 fi -if [[ ! -f "${SIMBA_ODBCINI}" ]]; then - echo "ERROR: Simba ODBC configuration not found:" - echo " ${SIMBA_ODBCINI}" +if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" exit 1 fi @@ -142,11 +142,7 @@ fi # --------------------------------------------------------------------------- echo -echo "============================================================" echo "Downloading Google Current driver" -echo "============================================================" -echo "GCS:" -echo " ${CURRENT_SO_GCS}" gcloud storage cp \ "${CURRENT_SO_GCS}" \ @@ -159,11 +155,7 @@ ls -lh "${CURRENT_SO}" # --------------------------------------------------------------------------- echo -echo "============================================================" echo "Downloading Google Main driver" -echo "============================================================" -echo "GCS:" -echo " ${MAIN_SO_GCS}" gcloud storage cp \ "${MAIN_SO_GCS}" \ @@ -182,23 +174,13 @@ run_benchmark() { local dsn="$4" echo - echo "============================================================" echo "Running benchmark: ${name}" - echo "============================================================" - echo "Driver:" - echo " ${driver_so}" - echo "DSN:" - echo " ${dsn}" - echo "Iterations:" - echo " ${BENCHMARK_ITERATIONS}" : > "${output_file}" local test_exit_code=0 for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do - echo - echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ >> "${output_file}" @@ -249,9 +231,7 @@ run_benchmark() { # --------------------------------------------------------------------------- cp /opt/odbc-driver/roots.pem "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/roots.pem" echo -echo "============================================================" echo "Preparing Google Current" -echo "============================================================" cp "${CURRENT_SO}" "${DRIVER_PATH}" @@ -272,9 +252,7 @@ run_benchmark \ # --------------------------------------------------------------------------- echo -echo "============================================================" echo "Preparing Google Main" -echo "============================================================" export ODBCINI="${GOOGLE_ODBCINI}" export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" @@ -289,23 +267,21 @@ run_benchmark \ "${GOOGLE_ODBCINI}" # --------------------------------------------------------------------------- -# Simba +# Existing # --------------------------------------------------------------------------- echo -echo "============================================================" -echo "Preparing Simba" -echo "============================================================" +echo "Preparing Existing Driver" -export ODBC_TESTS_DSN="${SIMBA_ODBC_TESTS_DSN:-SampleDSN}" -export ODBCINI="${SIMBA_ODBCINI}" +export ODBC_TESTS_DSN="${EXISTING_ODBC_TESTS_DSN:-SampleDSN}" +export ODBCINI="${EXISTING_ODBCINI}" export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" run_benchmark \ - "Simba" \ - "${SIMBA_RESULT}" \ + "Existing" \ + "${EXISTING_RESULT}" \ "/opt/odbc-driver/googlebigqueryodbc" \ - "${SIMBA_ODBCINI}" + "${EXISTING_ODBCINI}" # --------------------------------------------------------------------------- # Generate comparison @@ -325,17 +301,11 @@ if [[ ! -f "${PARSER}" ]]; then fi python3 "${PARSER}" \ - --simba "${SIMBA_RESULT}" \ + --existing "${EXISTING_RESULT}" \ --current "${CURRENT_RESULT}" \ --main "${MAIN_RESULT}" \ --output "${SUMMARY_RESULT}" -echo -echo "============================================================" -echo "Benchmark Summary" -echo "============================================================" - -cat "${SUMMARY_RESULT}" # --------------------------------------------------------------------------- # Upload results @@ -344,14 +314,12 @@ cat "${SUMMARY_RESULT}" RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/benchmarks" echo -echo "============================================================" echo "Uploading benchmark results" -echo "============================================================" gcloud storage cp \ "${CURRENT_RESULT}" \ "${MAIN_RESULT}" \ - "${SIMBA_RESULT}" \ + "${EXISTING_RESULT}" \ "${SUMMARY_RESULT}" \ "${RESULTS_BUCKET}/" diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index ea837a5aca..ea52384444 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -77,9 +77,9 @@ def make_table(title, rows): headers = [ "Test Case", - "Simba", + "Existing", "Google Current", - "Current vs Simba", + "Current vs Existing", "Google Main", "Main vs Current", ] @@ -130,9 +130,9 @@ def main(): ) parser.add_argument( - "--simba", + "--existing", required=True, - help="Simba benchmark output", + help="Existing benchmark output", ) parser.add_argument( @@ -155,12 +155,12 @@ def main(): args = parser.parse_args() - simba = parse_gtest_output(args.simba) + existing = parse_gtest_output(args.existing) current = parse_gtest_output(args.current) main_branch = parse_gtest_output(args.main) test_names = sorted( - set(simba) + set(existing) | set(current) | set(main_branch) ) @@ -168,16 +168,16 @@ def main(): rows = [] for test_name in test_names: - simba_ms = simba.get(test_name) + existing_ms = existing.get(test_name) current_ms = current.get(test_name) main_ms = main_branch.get(test_name) rows.append( ( test_name, - format_ms(simba_ms), + format_ms(existing_ms), format_ms(current_ms), - percentage_change(current_ms, simba_ms), + percentage_change(current_ms, existing_ms), format_ms(main_ms), percentage_change(main_ms, current_ms), ) @@ -187,9 +187,9 @@ def main(): headers = [ "Test Case", - "Simba", + "Existing", "Google Current", - "Current vs Simba", + "Current vs Existing", "Google Main", "Main vs Current", ] @@ -231,9 +231,9 @@ def format_row(values): print() print(summary) - if not simba: + if not existing: print( - "WARNING: No Simba benchmark results were found." + "WARNING: No Existing benchmark results were found." ) if not current: From 256090bcddeaf3dcccb669b354eb02f10bcfe785 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 19 Aug 2026 06:55:02 +0000 Subject: [PATCH 27/33] test --- .../builds/integration-bq-driver-asan.sh | 38 +- ci/cloudbuild/builds/lib/benchmark_results.py | 349 ++++++++++-------- 2 files changed, 219 insertions(+), 168 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index e992ccdcf2..4ffea1c083 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -67,7 +67,7 @@ CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" # --------------------------------------------------------------------------- # ODBC configuration @@ -143,25 +143,29 @@ fi echo echo "Downloading Google Current driver" +echo "------------------------------------------------------------" gcloud storage cp \ "${CURRENT_SO_GCS}" \ "${CURRENT_SO}" -ls -lh "${CURRENT_SO}" - # --------------------------------------------------------------------------- # Download Google Main driver # --------------------------------------------------------------------------- echo -echo "Downloading Google Main driver" +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" -gcloud storage cp \ - "${MAIN_SO_GCS}" \ - "${MAIN_SO}" -ls -lh "${MAIN_SO}" +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi # --------------------------------------------------------------------------- # Run benchmark @@ -175,6 +179,7 @@ run_benchmark() { echo echo "Running benchmark: ${name}" + echo "------------------------------------------------------------" : > "${output_file}" @@ -232,6 +237,7 @@ run_benchmark() { cp /opt/odbc-driver/roots.pem "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/roots.pem" echo echo "Preparing Google Current" +echo "------------------------------------------------------------" cp "${CURRENT_SO}" "${DRIVER_PATH}" @@ -253,18 +259,24 @@ run_benchmark \ echo echo "Preparing Google Main" +echo "------------------------------------------------------------" -export ODBCINI="${GOOGLE_ODBCINI}" + + if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + + export ODBCINI="${GOOGLE_ODBCINI}" export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" cp "${MAIN_SO}" "${DRIVER_PATH}" -ls -lh "${DRIVER_PATH}" - -run_benchmark \ + run_benchmark \ "Google Main" \ "${MAIN_RESULT}" \ "${DRIVER_PATH}" \ "${GOOGLE_ODBCINI}" +else + echo "Google Main benchmark skipped: main driver artifact unavailable." + : > "$MAIN_RESULT" +fi # --------------------------------------------------------------------------- # Existing @@ -272,6 +284,7 @@ run_benchmark \ echo echo "Preparing Existing Driver" +echo "------------------------------------------------------------" export ODBC_TESTS_DSN="${EXISTING_ODBC_TESTS_DSN:-SampleDSN}" export ODBCINI="${EXISTING_ODBCINI}" @@ -315,6 +328,7 @@ RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/benchmarks" echo echo "Uploading benchmark results" +echo "------------------------------------------------------------" gcloud storage cp \ "${CURRENT_RESULT}" \ diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index ea52384444..0cf6b56736 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -1,22 +1,63 @@ #!/usr/bin/env python3 import argparse -import math import re import statistics from pathlib import Path TIME_RE = re.compile( - r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)" + r"\[\s*OK\s*\]\s+(\S+)\s+\(([\d.]+)\s*(ns|us|ms|s)\)" ) +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + # GTest names can be: + # Instantiation/TestSuite.TestCase/Param + # Remove everything before the first '.' + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy suffixes so old/new benchmark names compare correctly. + name = re.sub(r"/(?:With|Without)HTAPI$", "", name) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a numeric GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + if unit == "us": + return value / 1000.0 + if unit == "ns": + return value / 1_000_000.0 + return value + + +def format_ms(value): + """Format milliseconds similarly to the existing benchmark table.""" + if value is None: + return "N/A" + + # Keep the table output in milliseconds. + return f"{value:.0f}ms" + + def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + Each test may appear multiple times because the benchmark is run for + multiple iterations. Return the median execution time in milliseconds. + """ path = Path(path) if not path.exists(): - raise FileNotFoundError(f"Benchmark output not found: {path}") + raise FileNotFoundError( + f"Benchmark output not found: {path}" + ) samples = {} @@ -26,102 +67,55 @@ def parse_gtest_output(path): if not match: continue - test_name = match.group(1).strip() + test_name = clean_test_name(match.group(1)) value = float(match.group(2)) - unit = match.group(3) - - if unit == "s": - value *= 1000.0 - elif unit == "us": - value /= 1000.0 - elif unit == "ns": - value /= 1_000_000.0 - - samples.setdefault(test_name, []).append(value) + unit = match.group(3).lower() - return { - test_name: statistics.median(values) - for test_name, values in samples.items() - } + value_ms = parse_time_to_ms(value, unit) + samples.setdefault(test_name, []).append(value_ms) -def format_ms(value): - if value is None: - return "N/A" - - if value >= 1000: - return f"{value / 1000:.3f}s" + results = {} - if value >= 1: - return f"{value:.3f}ms" + for test_name, values in samples.items(): + median = statistics.median(values) + results[test_name] = median - return f"{value * 1000:.3f}us" - - -def percentage_change(new_value, old_value): - if old_value is None or old_value == 0 or new_value is None: - return "N/A" + if len(values) > 1: + print( + f"{test_name}: " + f"median={median:.0f}ms of {len(values)} runs " + f"(min={min(values):.0f}ms " + f"max={max(values):.0f}ms)" + ) - pct = ((new_value - old_value) / old_value) * 100 + return results - if abs(pct) < 0.01: - pct = 0 - return f"{pct:+.2f}%" +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + Negative = faster/improvement + Positive = slower/degradation + """ + if ( + value_ms is None + or reference_ms is None + or reference_ms == 0 + ): + return " (N/A)" -def make_table(title, rows): - print() - print(title) - print("=" * len(title)) - - headers = [ - "Test Case", - "Existing", - "Google Current", - "Current vs Existing", - "Google Main", - "Main vs Current", - ] - - widths = [ - max( - len(headers[0]), - *(len(row[0]) for row in rows), - ), - max( - len(headers[1]), - *(len(row[1]) for row in rows), - ), - max( - len(headers[2]), - *(len(row[2]) for row in rows), - ), - max( - len(headers[3]), - *(len(row[3]) for row in rows), - ), - max( - len(headers[4]), - *(len(row[4]) for row in rows), - ), - max( - len(headers[5]), - *(len(row[5]) for row in rows), - ), - ] - - def format_row(values): - return " | ".join( - value.ljust(width) - for value, width in zip(values, widths) - ) + pct = round( + ((value_ms - reference_ms) / reference_ms) * 100 + ) - print(format_row(headers)) - print("-+-".join("-" * width for width in widths)) + if pct > 0: + return f" (+{pct}%)" + if pct < 0: + return f" ({pct}%)" - for row in rows: - print(format_row(row)) + return " (0%)" def main(): @@ -132,7 +126,7 @@ def main(): parser.add_argument( "--existing", required=True, - help="Existing benchmark output", + help="Existing driver benchmark output", ) parser.add_argument( @@ -153,100 +147,143 @@ def main(): help="Summary output file", ) + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + args = parser.parse_args() - existing = parse_gtest_output(args.existing) - current = parse_gtest_output(args.current) - main_branch = parse_gtest_output(args.main) + existing_data = parse_gtest_output(args.existing) + current_data = parse_gtest_output(args.current) + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found." + ) - test_names = sorted( - set(existing) - | set(current) - | set(main_branch) + all_tests = ( + set(existing_data) + | set(current_data) + | set(main_data) ) + sorted_tests = sorted(all_tests) + rows = [] - for test_name in test_names: - existing_ms = existing.get(test_name) - current_ms = current.get(test_name) - main_ms = main_branch.get(test_name) + for test_name in sorted_tests: + existing_ms = existing_data.get(test_name) + current_ms = current_data.get(test_name) + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + current_raw = format_ms(current_ms) + main_raw = format_ms(main_ms) + + current_pct = "" + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + main_pct = "" + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + main_value = f"{main_raw}{main_pct}" rows.append( ( test_name, - format_ms(existing_ms), - format_ms(current_ms), - percentage_change(current_ms, existing_ms), - format_ms(main_ms), - percentage_change(main_ms, current_ms), + existing_raw, + current_value, + main_value, ) ) - output_lines = [] - - headers = [ - "Test Case", - "Existing", - "Google Current", - "Current vs Existing", - "Google Main", - "Main vs Current", - ] - - widths = [ - max(len(headers[0]), *(len(row[0]) for row in rows)), - max(len(headers[1]), *(len(row[1]) for row in rows)), - max(len(headers[2]), *(len(row[2]) for row in rows)), - max(len(headers[3]), *(len(row[3]) for row in rows)), - max(len(headers[4]), *(len(row[4]) for row in rows)), - max(len(headers[5]), *(len(row[5]) for row in rows)), - ] - - def format_row(values): - return " | ".join( - value.ljust(width) - for value, width in zip(values, widths) - ) + # Match the GitHub Actions benchmark table format. + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" - output_lines.append( - "ODBC Performance Benchmark Summary" + w1 = ( + max([len(h1)] + [len(row[0]) for row in rows]) + if rows + else len(h1) ) - output_lines.append( - "==================================" + w2 = ( + max([len(h2)] + [len(row[1]) for row in rows]) + if rows + else len(h2) ) - output_lines.append("") - output_lines.append(format_row(headers)) - output_lines.append( - "-+-".join("-" * width for width in widths) + w3 = ( + max([len(h3)] + [len(row[2]) for row in rows]) + if rows + else len(h3) + ) + w4 = ( + max([len(h4)] + [len(row[3]) for row in rows]) + if rows + else len(h4) ) - for row in rows: - output_lines.append(format_row(row)) - - summary = "\n".join(output_lines) + table = ( + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*\n\n" + ) - Path(args.output).write_text(summary + "\n") + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) - print() - print(summary) + table += ( + "|-" + ("-" * w1) + + "-|-" + ("-" * w2) + + "-|-" + ("-" * w3) + + "-|-" + ("-" * w4) + + "-|\n" + ) - if not existing: - print( - "WARNING: No Existing benchmark results were found." + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" ) - if not current: - print( - "ERROR: No current Google benchmark results were found." - ) - return 1 + output_path = Path(args.output) + output_path.write_text(table) - if not main_branch: - print( - "ERROR: No main Google benchmark results were found." - ) - return 1 + print() + print(table) return 0 From a86e22c9fe5e678ec4e11c7b37e8a194e347d601 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 19 Aug 2026 08:20:21 +0000 Subject: [PATCH 28/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 10 +++++----- .../builds/integration-production-bq-driver-dm.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 4ffea1c083..38663fa4a4 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -22,7 +22,7 @@ WORKSPACE_DIR="$(pwd)" BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" -PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" BUILD_DIR="${WORKSPACE_DIR}/cmake-out" RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" @@ -55,7 +55,7 @@ echo CURRENT_RESULT="${RESULTS_DIR}/current.txt" MAIN_RESULT="${RESULTS_DIR}/main.txt" EXISTING_RESULT="${RESULTS_DIR}/existing.txt" -SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" # --------------------------------------------------------------------------- # Driver locations @@ -66,8 +66,8 @@ DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_b CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" -CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" # --------------------------------------------------------------------------- # ODBC configuration @@ -324,7 +324,7 @@ python3 "${PARSER}" \ # Upload results # --------------------------------------------------------------------------- -RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/benchmarks" +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" echo echo "Uploading benchmark results" diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 5d36db3433..9e2593c420 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -102,17 +102,17 @@ SANITIZED_BRANCH=$( sed -E 's/[^a-zA-Z0-9._-]/_/g' ) -PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf-drivers" +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" echo "Uploading Google driver artifact..." echo "Branch: ${SANITIZED_BRANCH}" gcloud storage cp \ "$DRIVER_SO" \ - "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" echo "Google driver benchmark artifact uploaded:" -echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" +echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" From 10cbece99e22ee79714e9666e7a3b0ba3950811a Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 19 Aug 2026 09:34:59 +0000 Subject: [PATCH 29/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 38663fa4a4..5745b65d7d 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -27,6 +27,7 @@ PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" BUILD_DIR="${WORKSPACE_DIR}/cmake-out" RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" +rm -rf "${RESULTS_DIR}" mkdir -p "${RESULTS_DIR}" # --------------------------------------------------------------------------- From 54a2f99225f92971b1120e35b471dc6d452c0c61 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 19 Aug 2026 10:05:17 +0000 Subject: [PATCH 30/33] test --- .../builds/integration-bq-driver-asan.sh | 7 +- .../builds/linux-bq-driver-benchmark.sh | 800 ------------------ .../benchmark-bq-driver-unixODBC-ci.yaml | 35 - 3 files changed, 2 insertions(+), 840 deletions(-) delete mode 100644 ci/cloudbuild/builds/linux-bq-driver-benchmark.sh delete mode 100644 ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 5745b65d7d..c41d0f9219 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -244,7 +244,7 @@ cp "${CURRENT_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" -export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" export ODBCINI="${GOOGLE_ODBCINI}" export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" @@ -264,9 +264,6 @@ echo "------------------------------------------------------------" if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then - - export ODBCINI="${GOOGLE_ODBCINI}" -export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" cp "${MAIN_SO}" "${DRIVER_PATH}" run_benchmark \ @@ -287,7 +284,7 @@ echo echo "Preparing Existing Driver" echo "------------------------------------------------------------" -export ODBC_TESTS_DSN="${EXISTING_ODBC_TESTS_DSN:-SampleDSN}" +export ODBC_TESTS_DSN="SampleDSN" export ODBCINI="${EXISTING_ODBCINI}" export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh deleted file mode 100644 index abd984df0c..0000000000 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ /dev/null @@ -1,800 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -source "$(dirname "$0")/../../lib/init.sh" -source module ci/install-dependencies.sh - -source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/lib/io.sh - -WORKSPACE_DIR="$(pwd)" - -# ============================================================ -# VCPKG -# ============================================================ - -VCPKG_VERSION="$(cat /tmp/vcpkg-version.txt)" -export VCPKG_VERSION - -echo "============================================================" -echo "VCPKG" -echo "============================================================" -echo "Using VCPKG_VERSION=${VCPKG_VERSION}" - -export VCPKG_ROOT=/vcpkg - -if [[ ! -d "${VCPKG_ROOT}/.git" ]]; then - git clone \ - --branch "${VCPKG_VERSION}" \ - https://github.com/microsoft/vcpkg.git \ - "${VCPKG_ROOT}" -fi - -cd "${VCPKG_ROOT}" - -git checkout "${VCPKG_VERSION}" - -./bootstrap-vcpkg.sh -disableMetrics - -cd "${WORKSPACE_DIR}" - -# ============================================================ -# COMMON CMAKE ARGUMENTS -# ============================================================ - -mapfile -t cmake_args < <(cmake::common_args) - -# ============================================================ -# BENCHMARK CONFIGURATION -# ============================================================ - -BUILD_DIR="${WORKSPACE_DIR}/cmake-out" - -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export ODBC_TRANSACTIONS_TESTS_DSN="ODBCTransactionsTestsDSN" - -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" - -BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-1}" - -echo "============================================================" -echo "Benchmark configuration" -echo "============================================================" -echo "Workspace : ${WORKSPACE_DIR}" -echo "Build directory : ${BUILD_DIR}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Google DSN : ${ODBC_TESTS_DSN}" -echo "============================================================" - -# ============================================================ -# UNIXODBC -# ============================================================ - -if command -v odbcinst &>/dev/null; then - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." -else - export UNIXODBC_INSTALLED=false - export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini - echo "unixODBC is not installed." - echo "Using ODBCINSTINI=${ODBCINSTINI}" -fi - -# ============================================================ -# CONFIGURE CMAKE -# -# IMPORTANT: -# This is a separate benchmark build. -# -# BUILD_PERFORMANCE_TEST_ONLY=ON means we are configuring the -# performance-test build rather than configuring/running the -# complete integration-test suite. -# ============================================================ - -echo "============================================================" -echo "Configuring performance_test" -echo "============================================================" - -io::run cmake -S "${WORKSPACE_DIR}" -B "${BUILD_DIR}" \ - "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=ON \ - -DODBC_EXAMPLES=ON \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF - -# ============================================================ -# BUILD ONLY performance_test -# ============================================================ - -echo "============================================================" -echo "Building ONLY performance_test" -echo "============================================================" - -io::run cmake --build "${BUILD_DIR}" \ - --target google_cloud_odbc_bq_driver \ - --config Release \ - --parallel "$(nproc)" - -io::run cmake --build "${BUILD_DIR}" \ - --target performance_test \ - --config Release \ - --parallel "$(nproc)" - -# ============================================================ -# LOCATE PERFORMANCE TEST -# ============================================================ - -PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" - -if [[ ! -f "${PERFORMANCE_TEST}" ]]; then - echo "ERROR: performance_test was not found:" - echo " ${PERFORMANCE_TEST}" - echo - echo "Searching build directory:" - find "${BUILD_DIR}" -type f -name "performance_test*" -print - exit 1 -fi - -echo "============================================================" -echo "performance_test found" -echo "============================================================" -echo "${PERFORMANCE_TEST}" - -# ============================================================ -# GOOGLE DRIVER -# -# The Google driver is generated by the performance_test build. -# We do NOT download an existing Google .so. -# ============================================================ - -GOOGLE_DRIVER="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -if [[ ! -f "${GOOGLE_DRIVER}" ]]; then - echo "ERROR: Google driver was not generated by the performance build:" - echo " ${GOOGLE_DRIVER}" - echo - echo "Generated shared libraries:" - find "${BUILD_DIR}" -type f -name "*.so" -print - exit 1 -fi - -echo "============================================================" -echo "Google driver generated" -echo "============================================================" -echo "${GOOGLE_DRIVER}" - -# ============================================================ -# CERTIFICATE -# ============================================================ - -GOOGLE_DRIVER_DIR="${BUILD_DIR}/google/cloud/odbc" - -if [[ ! -f /opt/odbc-driver/roots.pem ]]; then - echo "ERROR: /opt/odbc-driver/roots.pem was not found." - exit 1 -fi - -io::run cp \ - /opt/odbc-driver/roots.pem \ - "${GOOGLE_DRIVER_DIR}/roots.pem" - -# ============================================================ -# ODBC CONFIGURATION -# -# The existing /opt/odbc-driver/odbc.ini must contain the DSNs -# used below. -# -# Google: -# SampleDSNGoogleDriver -# -# Simba: -# SampleDSN -# -# If SampleDSN does not exist, Simba cannot be benchmarked. -# ============================================================ - -ODBC_INI="/opt/odbc-driver/odbc.ini" - -if [[ ! -f "${ODBC_INI}" ]]; then - echo "ERROR: ODBC configuration was not found:" - echo " ${ODBC_INI}" - exit 1 -fi - -export ODBCINI="${ODBC_INI}" - -echo "============================================================" -echo "ODBC configuration" -echo "============================================================" -echo "ODBCINI=${ODBCINI}" - -echo -echo "Available DSNs:" -grep -E '^\[[^]]+\]$' "${ODBCINI}" || true - -# ============================================================ -# VERIFY GOOGLE DSN -# ============================================================ - -if ! grep -q '^\[SampleDSNGoogleDriver\]$' "${ODBC_INI}"; then - echo - echo "ERROR: Google DSN [SampleDSNGoogleDriver] was not found in:" - echo " ${ODBC_INI}" - exit 1 -fi - -# ============================================================ -# VERIFY SIMBA DSN -# ============================================================ - -if ! grep -q '^\[SampleDSN\]$' "${ODBC_INI}"; then - echo - echo "ERROR: Simba DSN [SampleDSN] was not found in:" - echo " ${ODBC_INI}" - echo - echo "The performance executable can be reused for Simba, but" - echo "the Simba DSN must be available in ODBCINI." - exit 1 -fi - -# ============================================================ -# RESULT FILES -# ============================================================ - -RESULT_DIR="${WORKSPACE_DIR}/benchmark_results" - -mkdir -p "${RESULT_DIR}" - -GOOGLE_RESULTS="${RESULT_DIR}/current_bq.txt" -SIMBA_RESULTS="${RESULT_DIR}/current_core.txt" - -MAIN_GOOGLE_RESULTS="${RESULT_DIR}/main_bq.txt" - -SUMMARY_FILE="${WORKSPACE_DIR}/benchmark_summary_table.txt" - -rm -f \ - "${GOOGLE_RESULTS}" \ - "${SIMBA_RESULTS}" \ - "${MAIN_GOOGLE_RESULTS}" \ - "${SUMMARY_FILE}" - -# ============================================================ -# RUN GOOGLE DRIVER -# -# IMPORTANT: -# -# The executable is invoked directly. -# -# This executes the COMPLETE performance_test suite. -# -# There is NO ctest here. -# -# There is NO individual test filter. -# -# With BENCHMARK_ITERATIONS=1: -# -# performance_test -# -# runs exactly once, and every test registered in that executable -# is executed once. -# ============================================================ - -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" - -echo "============================================================" -echo "Running Google Driver performance_test" -echo "============================================================" -echo "DSN : ${ODBC_TESTS_DSN}" -echo "Iterations: ${BENCHMARK_ITERATIONS}" -echo "Executable: ${PERFORMANCE_TEST}" -echo "============================================================" - -GOOGLE_EXIT_CODE=0 - -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - - echo "=== Google Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >>"${GOOGLE_RESULTS}" - - set +e - - "${PERFORMANCE_TEST}" >>"${GOOGLE_RESULTS}" 2>&1 - - RUN_EXIT_CODE=$? - - set -e - - if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Google Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" - GOOGLE_EXIT_CODE=${RUN_EXIT_CODE} - fi - -done - -if [[ ${GOOGLE_EXIT_CODE} -ne 0 ]]; then - echo - echo "ERROR: Google Driver performance_test failed." - echo - echo "Last Google benchmark output:" - tail -100 "${GOOGLE_RESULTS}" || true - exit "${GOOGLE_EXIT_CODE}" -fi - -echo "Google Driver performance suite completed successfully." - -# ============================================================ -# RUN SIMBA DRIVER -# -# SAME performance_test executable. -# -# Only the DSN changes. -# -# Therefore the exact same performance test cases are executed -# against Simba. -# ============================================================ - -export ODBC_TESTS_DSN="SampleDSN" - -echo "============================================================" -echo "Running Simba Driver performance_test" -echo "============================================================" -echo "DSN : ${ODBC_TESTS_DSN}" -echo "Iterations: ${BENCHMARK_ITERATIONS}" -echo "Executable: ${PERFORMANCE_TEST}" -echo "============================================================" - -SIMBA_EXIT_CODE=0 - -for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do - - echo "=== Simba Driver iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >>"${SIMBA_RESULTS}" - - set +e - - "${PERFORMANCE_TEST}" >>"${SIMBA_RESULTS}" 2>&1 - - RUN_EXIT_CODE=$? - - set -e - - if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - echo "WARNING: Simba Driver iteration ${i} failed with exit code ${RUN_EXIT_CODE}" - SIMBA_EXIT_CODE=${RUN_EXIT_CODE} - fi - -done - -if [[ ${SIMBA_EXIT_CODE} -ne 0 ]]; then - echo - echo "ERROR: Simba Driver performance_test failed." - echo - echo "Last Simba benchmark output:" - tail -100 "${SIMBA_RESULTS}" || true - exit "${SIMBA_EXIT_CODE}" -fi - -echo "Simba Driver performance suite completed successfully." - -# ============================================================ -# DOWNLOAD MAIN GOOGLE DRIVER RESULTS -# ============================================================ - -CURRENT_BRANCH="${BRANCH_NAME:-main}" - -SANITIZED_BRANCH="$(echo "${CURRENT_BRANCH}" | - sed 's/[^a-zA-Z0-9._-]/_/g')" - -echo "============================================================" -echo "Downloading Google Driver main baseline" -echo "============================================================" - -set +e - -gcloud storage cp \ - "gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt" \ - "${MAIN_GOOGLE_RESULTS}" - -MAIN_DOWNLOAD_EXIT=$? - -set -e - -if [[ ${MAIN_DOWNLOAD_EXIT} -ne 0 ]]; then - echo "WARNING: Main Google benchmark result is not available." - rm -f "${MAIN_GOOGLE_RESULTS}" -else - echo "Main Google benchmark downloaded successfully." -fi - -# ============================================================ -# PARSE AND COMPARE -# -# Comparison: -# -# Simba Driver -# | -# | Google Current vs Simba -# v -# Google Driver Current -# | -# | Google Main vs Google Current -# v -# Google Driver Main -# -# Every test appearing in any of the three result files is -# included in the table. -# ============================================================ - -echo "============================================================" -echo "Generating performance comparison" -echo "============================================================" - -python3 <<'PYTHON' -import os -import re - - -SIMBA_FILE = "benchmark_results/current_core.txt" -GOOGLE_FILE = "benchmark_results/current_bq.txt" -MAIN_FILE = "benchmark_results/main_bq.txt" - -OUTPUT_FILE = "benchmark_summary_table.txt" - - -def clean_test_name(name): - """ - GTest names can look like: - - TestSuite.TestCase - Instantiation/TestSuite.TestCase - TestSuite.TestCase/parameter - - Keep the TestCase portion so the same test can be compared - between the two drivers. - """ - - if "." in name: - name = name.split(".", 1)[1] - - # Remove old HTAPI parameter suffixes. - name = re.sub(r"/(?:With|Without)HTAPI$", "", name) - - return name - - -def parse_time_to_ms(value): - if not value: - return None - - value = value.strip() - - if value == "N/A": - return None - - match = re.match(r"^([\d.]+)\s*(\w+)$", value) - - if not match: - return None - - number = float(match.group(1)) - unit = match.group(2).lower() - - if unit == "ms": - return number - - if unit == "s": - return number * 1000 - - if unit == "us": - return number / 1000 - - if unit == "ns": - return number / 1_000_000 - - return None - - -def parse_gtest_output(filename): - """ - Parse: - - [ OK ] TestSuite.TestCase (123 ms) - - Multiple iterations are supported. - - The median is used if BENCHMARK_ITERATIONS > 1. - """ - - samples = {} - - if not os.path.exists(filename): - return {} - - pattern = re.compile( - r"\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)" - ) - - with open(filename, "r", errors="replace") as file: - - for line in file: - - match = pattern.search(line) - - if not match: - continue - - test_name = clean_test_name(match.group(1)) - - duration = parse_time_to_ms(match.group(2)) - - if duration is None: - continue - - samples.setdefault(test_name, []).append(duration) - - results = {} - - for test_name, values in samples.items(): - - values.sort() - - n = len(values) - - if n % 2 == 1: - median = values[n // 2] - else: - median = ( - values[n // 2 - 1] + - values[n // 2] - ) / 2.0 - - results[test_name] = median - - return results - - -def percentage(value, reference): - - if value is None or reference is None or reference == 0: - return "N/A" - - pct = ((value - reference) / reference) * 100 - - return f"{pct:+.0f}%" - - -simba = parse_gtest_output(SIMBA_FILE) - -google = parse_gtest_output(GOOGLE_FILE) - -main_google = parse_gtest_output(MAIN_FILE) - - -all_tests = sorted( - set(simba.keys()) | - set(google.keys()) | - set(main_google.keys()) -) - - -rows = [] - - -for test in all_tests: - - simba_ms = simba.get(test) - - google_ms = google.get(test) - - main_ms = main_google.get(test) - - google_vs_simba = percentage( - google_ms, - simba_ms - ) - - main_vs_google = percentage( - main_ms, - google_ms - ) - - if simba_ms is None: - simba_value = "N/A" - else: - simba_value = f"{simba_ms:.0f}ms" - - if google_ms is None: - google_value = "N/A" - else: - google_value = ( - f"{google_ms:.0f}ms " - f"({google_vs_simba})" - ) - - if main_ms is None: - main_value = "N/A" - else: - main_value = ( - f"{main_ms:.0f}ms " - f"({main_vs_google})" - ) - - rows.append( - ( - test, - simba_value, - google_value, - main_value, - ) - ) - - -headers = [ - "Test Case", - "Simba Driver", - "Google Driver (Current)", - "Google Driver (Main)", -] - - -widths = [] - -for index, header in enumerate(headers): - - width = len(header) - - for row in rows: - width = max( - width, - len(row[index]) - ) - - widths.append(width) - - -table = [] - - -table.append( - "| " - + " | ".join( - header.ljust(widths[index]) - for index, header in enumerate(headers) - ) - + " |" -) - - -table.append( - "| " - + " | ".join( - "-" * widths[index] - for index in range(len(headers)) - ) - + " |" -) - - -for row in rows: - - table.append( - "| " - + " | ".join( - row[index].ljust(widths[index]) - for index in range(len(headers)) - ) - + " |" - ) - - -description = ( - "Percentages in **Google Driver (Current)** are relative to " - "**Simba Driver**. Percentages in **Google Driver (Main)** " - "are relative to **Google Driver (Current)**. " - "Negative values indicate that the Google Driver is faster; " - "positive values indicate that it is slower." -) - - -output = ( - description - + "\n\n" - + "\n".join(table) - + "\n" -) - - -with open(OUTPUT_FILE, "w") as file: - file.write(output) - - -print(output) - -print( - f"Compared {len(all_tests)} performance test cases." -) - -PYTHON - -# ============================================================ -# UPLOAD RESULTS -# ============================================================ - -RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf/${SANITIZED_BRANCH}/results" - -echo "============================================================" -echo "Uploading benchmark results" -echo "============================================================" -echo "Destination:" -echo " ${RESULTS_BUCKET}" -echo "============================================================" - -io::run gcloud storage cp \ - "${GOOGLE_RESULTS}" \ - "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" - -io::run gcloud storage cp \ - "${SIMBA_RESULTS}" \ - "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" - -io::run gcloud storage cp \ - "${SUMMARY_FILE}" \ - "${RESULTS_BUCKET}/benchmark_summary_table.txt" - -# ============================================================ -# FINAL SUMMARY -# ============================================================ - -echo -echo "============================================================" -echo "BENCHMARK COMPLETED SUCCESSFULLY" -echo "============================================================" - -echo -echo "Performance executable:" -echo " ${PERFORMANCE_TEST}" - -echo -echo "Google Driver:" -echo " ${GOOGLE_DRIVER}" - -echo -echo "Google results:" -echo " ${GOOGLE_RESULTS}" - -echo -echo "Simba results:" -echo " ${SIMBA_RESULTS}" - -echo -echo "Comparison:" -echo " ${SUMMARY_FILE}" - -echo -echo "GCS:" -echo " ${RESULTS_BUCKET}/" - -echo -echo "All tests registered in performance_test were executed." -echo "Benchmark iterations: ${BENCHMARK_ITERATIONS}" - -echo "============================================================" diff --git a/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml b/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml deleted file mode 100644 index 582906248d..0000000000 --- a/ci/cloudbuild/triggers/benchmark-bq-driver-unixODBC-ci.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -filename: ci/cloudbuild/cloudbuild.yaml - -github: - name: cpp-bigquery-odbc - owner: googleapis - push: - branch: ^main$ - -name: benchmark-bq-driver-unixODBC-ci - -substitutions: - _BUILD_NAME: linux-bq-driver-benchmark - _DEPENDENCIES: 'BAZEL,unixODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' - _DISTRO: ubuntu-20.04-release - _TRIGGER_TYPE: ci - -includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS - -tags: -- ci -- benchmark From f4ba3b58dfdae00c130eb633f833ad584ee3cef8 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Thu, 20 Aug 2026 08:32:34 +0000 Subject: [PATCH 31/33] test --- .../builds/integration-bq-driver-asan.sh | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index c41d0f9219..73d57e2409 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -74,6 +74,85 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" # ODBC configuration # --------------------------------------------------------------------------- +# Export as env variable +VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +export VCPKG_VERSION +echo "Using VCPKG_VERSION=$VCPKG_VERSION" + +# Vcpkg install and configure +export VCPKG_ROOT=/vcpkg +git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" +cd "$VCPKG_ROOT" +git checkout "$VCPKG_VERSION" + +# Bootstrap +./bootstrap-vcpkg.sh -disableMetrics + +cd "$WORKSPACE_DIR" +mapfile -t cmake_args < <(cmake::common_args) + +# This is the name of DSN set in odbc.ini +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} + +# Check if unixODBC is installed +if command -v odbcinst &>/dev/null; then + # unixODBC is installed, export environment variable + export UNIXODBC_INSTALLED=true + echo "unixODBC is installed." +else + # unixODBC is not installed + export UNIXODBC_INSTALLED=false + export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini + echo "unixODBC is not installed." +fi + +io::run cmake -B "$BUILD_DIR" \ + "${cmake_args[@]}" \ + -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ + -DCMAKE_CXX_STANDARD=17 \ + -DODBC_INTEGRATION_TESTING=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON \ + -DODBC_DEMO_TESTING=ON \ + -DODBC_EXAMPLES=ON \ + -DODBC_UNIT_TESTING=OFF \ + -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF +io::run cmake --build cmake-out + + +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 +fi + +echo "Google driver found:" +ls -lh "$DRIVER_SO" + +# Sanitize branch name for use in GCS path. +SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +) + +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +echo "Uploading Google driver artifact..." +echo "Branch: ${SANITIZED_BRANCH}" + +gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + +echo "Google driver benchmark artifact uploaded:" +echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" @@ -88,7 +167,7 @@ echo "============================================================" echo "Building performance_test" echo "============================================================" -mapfile -t cmake_args < <(cmake::common_args) + io::run cmake \ -S "${WORKSPACE_DIR}" \ From 4998a4bf9961278d954553bfd768a4613b299f88 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Thu, 20 Aug 2026 09:17:49 +0000 Subject: [PATCH 32/33] test --- ci/cloudbuild/builds/integration-bq-driver-asan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 73d57e2409..7908cd0fa6 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -370,7 +370,7 @@ export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" run_benchmark \ "Existing" \ "${EXISTING_RESULT}" \ - "/opt/odbc-driver/googlebigqueryodbc" \ + "/opt/odbc-driver/googlebigqueryodbc/lib/libgooglebigqueryodbc_sb64.so" \ "${EXISTING_ODBCINI}" # --------------------------------------------------------------------------- From 431bcba17ddfaff050a8234c08e65a5af814f334 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Thu, 20 Aug 2026 11:37:38 +0000 Subject: [PATCH 33/33] test --- .../builds/integration-bq-driver-asan.sh | 117 ++++-------------- 1 file changed, 22 insertions(+), 95 deletions(-) diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 7908cd0fa6..a696e7533c 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -2,8 +2,17 @@ # # Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail @@ -74,85 +83,6 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" # ODBC configuration # --------------------------------------------------------------------------- -# Export as env variable -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) -export VCPKG_VERSION -echo "Using VCPKG_VERSION=$VCPKG_VERSION" - -# Vcpkg install and configure -export VCPKG_ROOT=/vcpkg -git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" -cd "$VCPKG_ROOT" -git checkout "$VCPKG_VERSION" - -# Bootstrap -./bootstrap-vcpkg.sh -disableMetrics - -cd "$WORKSPACE_DIR" -mapfile -t cmake_args < <(cmake::common_args) - -# This is the name of DSN set in odbc.ini -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} - -# Check if unixODBC is installed -if command -v odbcinst &>/dev/null; then - # unixODBC is installed, export environment variable - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." -else - # unixODBC is not installed - export UNIXODBC_INSTALLED=false - export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini - echo "unixODBC is not installed." -fi - -io::run cmake -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=ON \ - -DODBC_EXAMPLES=ON \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -io::run cmake --build cmake-out - - -# --------------------------------------------------------------------------- -# Publish Google driver .so for performance benchmarks -# --------------------------------------------------------------------------- - -DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -if [[ ! -f "$DRIVER_SO" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" - echo " $DRIVER_SO" - exit 1 -fi - -echo "Google driver found:" -ls -lh "$DRIVER_SO" - -# Sanitize branch name for use in GCS path. -SANITIZED_BRANCH=$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' -) - -PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" - -echo "Uploading Google driver artifact..." -echo "Branch: ${SANITIZED_BRANCH}" - -gcloud storage cp \ - "$DRIVER_SO" \ - "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" - -echo "Google driver benchmark artifact uploaded:" -echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" - GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" @@ -167,7 +97,7 @@ echo "============================================================" echo "Building performance_test" echo "============================================================" - +mapfile -t cmake_args < <(cmake::common_args) io::run cmake \ -S "${WORKSPACE_DIR}" \ @@ -237,7 +167,6 @@ echo echo "Downloading Google driver from main" echo "------------------------------------------------------------" - if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then echo "Main Google driver downloaded successfully." HAS_MAIN_DRIVER=true @@ -261,21 +190,21 @@ run_benchmark() { echo "Running benchmark: ${name}" echo "------------------------------------------------------------" - : > "${output_file}" + : >"${output_file}" local test_exit_code=0 for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ - >> "${output_file}" + >>"${output_file}" set +e ODBCINI="${dsn}" \ ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ "${PERFORMANCE_TEST}" \ - >> "${output_file}" 2>&1 + >>"${output_file}" 2>&1 run_exit=$? @@ -341,18 +270,17 @@ echo echo "Preparing Google Main" echo "------------------------------------------------------------" - - if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then -cp "${MAIN_SO}" "${DRIVER_PATH}" +if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + cp "${MAIN_SO}" "${DRIVER_PATH}" run_benchmark \ - "Google Main" \ - "${MAIN_RESULT}" \ - "${DRIVER_PATH}" \ - "${GOOGLE_ODBCINI}" + "Google Main" \ + "${MAIN_RESULT}" \ + "${DRIVER_PATH}" \ + "${GOOGLE_ODBCINI}" else echo "Google Main benchmark skipped: main driver artifact unavailable." - : > "$MAIN_RESULT" + : >"$MAIN_RESULT" fi # --------------------------------------------------------------------------- @@ -370,7 +298,7 @@ export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" run_benchmark \ "Existing" \ "${EXISTING_RESULT}" \ - "/opt/odbc-driver/googlebigqueryodbc/lib/libgooglebigqueryodbc_sb64.so" \ + "/opt/odbc-driver/googlebigqueryodbc" \ "${EXISTING_ODBCINI}" # --------------------------------------------------------------------------- @@ -396,7 +324,6 @@ python3 "${PARSER}" \ --main "${MAIN_RESULT}" \ --output "${SUMMARY_RESULT}" - # --------------------------------------------------------------------------- # Upload results # ---------------------------------------------------------------------------