diff --git a/ci/cloudbuild/builds/integration-bq-driver-asan.sh b/ci/cloudbuild/builds/integration-bq-driver-asan.sh index 54bf4fd821..a696e7533c 100755 --- a/ci/cloudbuild/builds/integration-bq-driver-asan.sh +++ b/ci/cloudbuild/builds/integration-bq-driver-asan.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,72 +20,332 @@ source "$(dirname "$0")/../../lib/init.sh" source module ci/install-dependencies.sh source module ci/cloudbuild/builds/lib/cmake.sh -source module ci/cloudbuild/builds/lib/bazel.sh source module ci/cloudbuild/builds/lib/secrets.sh -source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh -WORKSPACE_DIR=$(pwd) +WORKSPACE_DIR="$(pwd)" -# Export as env variable -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) -export VCPKG_VERSION -echo "Using VCPKG_VERSION=$VCPKG_VERSION" +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- -# Vcpkg install and configure -export VCPKG_ROOT=/vcpkg -git clone --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git "$VCPKG_ROOT" -cd "$VCPKG_ROOT" -git checkout "$VCPKG_VERSION" +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" -# Bootstrap -./bootstrap-vcpkg.sh -disableMetrics +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" -cd "$WORKSPACE_DIR" +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" -# This runs all the unit tests -mapfile -t args < <(bazel::common_args) -mapfile -t unit_tests_args < <(unit_tests::bazel_args) -mapfile -t secrets_bazel < <(secrets::bazel_args) +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" -io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... +# --------------------------------------------------------------------------- +# Branch +# --------------------------------------------------------------------------- + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +)" + +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo + +# --------------------------------------------------------------------------- +# Result files +# --------------------------------------------------------------------------- + +CURRENT_RESULT="${RESULTS_DIR}/current.txt" +MAIN_RESULT="${RESULTS_DIR}/main.txt" +EXISTING_RESULT="${RESULTS_DIR}/existing.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" + +# --------------------------------------------------------------------------- +# Driver locations +# --------------------------------------------------------------------------- + +DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" + +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" + +# --------------------------------------------------------------------------- +# ODBC configuration +# --------------------------------------------------------------------------- + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +export ODBC_TESTS_DSN="${ODBC_TESTS_DSN:-SampleDSNGoogleDriver}" + +# --------------------------------------------------------------------------- +# Build performance_test +# --------------------------------------------------------------------------- + +echo +echo "============================================================" +echo "Building performance_test" +echo "============================================================" -# Run the integration tests mapfile -t cmake_args < <(cmake::common_args) -BUILD_DIR="/opt/odbc-driver" -# This is the name of DSN set in odbc.ini +io::run cmake \ + -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + "${cmake_args[@]}" \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON + +io::run cmake \ + --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)" + +PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + +if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" +fi + +if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true + + exit 1 +fi + +echo "performance_test:" +echo " ${PERFORMANCE_TEST}" + +# --------------------------------------------------------------------------- +# Validate ODBC configuration +# --------------------------------------------------------------------------- + +if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 +fi + +if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Download Google Current driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google Current driver" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + +# --------------------------------------------------------------------------- +# Download Google Main driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" + +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi + +# --------------------------------------------------------------------------- +# Run benchmark +# --------------------------------------------------------------------------- + +run_benchmark() { + local name="$1" + local output_file="$2" + local driver_so="$3" + local dsn="$4" + + echo + echo "Running benchmark: ${name}" + echo "------------------------------------------------------------" + + : >"${output_file}" + + local test_exit_code=0 + + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >>"${output_file}" + + set +e + + ODBCINI="${dsn}" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "${PERFORMANCE_TEST}" \ + >>"${output_file}" 2>&1 + + run_exit=$? + + set -e + + if [[ "${run_exit}" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + test_exit_code="${run_exit}" + fi + done + + echo + echo "Raw result:" + echo " ${output_file}" + + if [[ "${test_exit_code}" -ne 0 ]]; then + echo + echo "============================================================" + echo "Benchmark failure output: ${name}" + echo "============================================================" + + cat "${output_file}" + + echo + echo "============================================================" + + return "${test_exit_code}" + fi + + return 0 +} + +# --------------------------------------------------------------------------- +# Google Current +# +# The existing Google DSN points to DRIVER_PATH. +# Replace the driver binary before running the benchmark. +# --------------------------------------------------------------------------- +cp /opt/odbc-driver/roots.pem "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/roots.pem" +echo +echo "Preparing Google Current" +echo "------------------------------------------------------------" + +cp "${CURRENT_SO}" "${DRIVER_PATH}" + +ls -lh "${DRIVER_PATH}" + export ODBC_TESTS_DSN="SampleDSNGoogleDriver" -export LSAN_OPTIONS="use_tls=0:suppressions=/opt/odbc-driver/lsan.supp:print_suppressions=0" +export ODBCINI="${GOOGLE_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +run_benchmark \ + "Google Current" \ + "${CURRENT_RESULT}" \ + "${DRIVER_PATH}" \ + "${GOOGLE_ODBCINI}" -# Check if unixODBC is installed -if command -v odbcinst &>/dev/null; then - # unixODBC is installed, export environment variable - export UNIXODBC_INSTALLED=true - echo "unixODBC is installed." +# --------------------------------------------------------------------------- +# Google Main +# --------------------------------------------------------------------------- + +echo +echo "Preparing Google Main" +echo "------------------------------------------------------------" + +if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + cp "${MAIN_SO}" "${DRIVER_PATH}" + + run_benchmark \ + "Google Main" \ + "${MAIN_RESULT}" \ + "${DRIVER_PATH}" \ + "${GOOGLE_ODBCINI}" else - # unixODBC is not installed - export UNIXODBC_INSTALLED=false - export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini - export ODBCINI=/opt/odbc-driver/odbc.ini - echo "unixODBC is not installed." + echo "Google Main benchmark skipped: main driver artifact unavailable." + : >"$MAIN_RESULT" fi -io::run cmake -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DENABLE_SANITIZER=ON \ - -DODBC_DEMO_TESTING=ON \ - -DODBC_EXAMPLES=ON \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF - -io::run cmake --build cmake-out - -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" +# --------------------------------------------------------------------------- +# Existing +# --------------------------------------------------------------------------- + +echo +echo "Preparing Existing Driver" +echo "------------------------------------------------------------" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBCINI="${EXISTING_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + +run_benchmark \ + "Existing" \ + "${EXISTING_RESULT}" \ + "/opt/odbc-driver/googlebigqueryodbc" \ + "${EXISTING_ODBCINI}" + +# --------------------------------------------------------------------------- +# Generate comparison +# --------------------------------------------------------------------------- + +echo +echo "============================================================" +echo "Generating benchmark comparison" +echo "============================================================" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --output "${SUMMARY_RESULT}" + +# --------------------------------------------------------------------------- +# Upload results +# --------------------------------------------------------------------------- + +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" + +echo +echo "Uploading benchmark results" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_RESULT}" \ + "${MAIN_RESULT}" \ + "${EXISTING_RESULT}" \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/" + +echo +echo "============================================================" +echo "Benchmark completed successfully" +echo "============================================================" \ No newline at end of file diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..9e2593c420 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -81,6 +81,39 @@ io::run cmake -B "$BUILD_DIR" \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 +fi + +echo "Google driver found:" +ls -lh "$DRIVER_SO" + +# Sanitize branch name for use in GCS path. +SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +) + +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +echo "Uploading Google driver artifact..." +echo "Branch: ${SANITIZED_BRANCH}" + +gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + +echo "Google driver benchmark artifact uploaded:" +echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..0cf6b56736 --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 + +import argparse +import re +import statistics +from pathlib import Path + + +TIME_RE = re.compile( + r"\[\s*OK\s*\]\s+(\S+)\s+\(([\d.]+)\s*(ns|us|ms|s)\)" +) + + +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + # GTest names can be: + # Instantiation/TestSuite.TestCase/Param + # Remove everything before the first '.' + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy suffixes so old/new benchmark names compare correctly. + name = re.sub(r"/(?:With|Without)HTAPI$", "", name) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a numeric GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + if unit == "us": + return value / 1000.0 + if unit == "ns": + return value / 1_000_000.0 + return value + + +def format_ms(value): + """Format milliseconds similarly to the existing benchmark table.""" + if value is None: + return "N/A" + + # Keep the table output in milliseconds. + return f"{value:.0f}ms" + + +def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + Each test may appear multiple times because the benchmark is run for + multiple iterations. Return the median execution time in milliseconds. + """ + path = Path(path) + + if not path.exists(): + raise FileNotFoundError( + f"Benchmark output not found: {path}" + ) + + samples = {} + + for line in path.read_text(errors="replace").splitlines(): + match = TIME_RE.search(line) + + if not match: + continue + + test_name = clean_test_name(match.group(1)) + value = float(match.group(2)) + unit = match.group(3).lower() + + value_ms = parse_time_to_ms(value, unit) + + samples.setdefault(test_name, []).append(value_ms) + + results = {} + + for test_name, values in samples.items(): + median = statistics.median(values) + results[test_name] = median + + if len(values) > 1: + print( + f"{test_name}: " + f"median={median:.0f}ms of {len(values)} runs " + f"(min={min(values):.0f}ms " + f"max={max(values):.0f}ms)" + ) + + return results + + +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + + Negative = faster/improvement + Positive = slower/degradation + """ + if ( + value_ms is None + or reference_ms is None + or reference_ms == 0 + ): + return " (N/A)" + + pct = round( + ((value_ms - reference_ms) / reference_ms) * 100 + ) + + if pct > 0: + return f" (+{pct}%)" + if pct < 0: + return f" ({pct}%)" + + return " (0%)" + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--existing", + required=True, + help="Existing driver benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + + args = parser.parse_args() + + existing_data = parse_gtest_output(args.existing) + current_data = parse_gtest_output(args.current) + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found." + ) + + all_tests = ( + set(existing_data) + | set(current_data) + | set(main_data) + ) + + sorted_tests = sorted(all_tests) + + rows = [] + + for test_name in sorted_tests: + existing_ms = existing_data.get(test_name) + current_ms = current_data.get(test_name) + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + current_raw = format_ms(current_ms) + main_raw = format_ms(main_ms) + + current_pct = "" + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + main_pct = "" + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + main_value = f"{main_raw}{main_pct}" + + rows.append( + ( + test_name, + existing_raw, + current_value, + main_value, + ) + ) + + # Match the GitHub Actions benchmark table format. + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" + + w1 = ( + max([len(h1)] + [len(row[0]) for row in rows]) + if rows + else len(h1) + ) + w2 = ( + max([len(h2)] + [len(row[1]) for row in rows]) + if rows + else len(h2) + ) + w3 = ( + max([len(h3)] + [len(row[2]) for row in rows]) + if rows + else len(h3) + ) + w4 = ( + max([len(h4)] + [len(row[3]) for row in rows]) + if rows + else len(h4) + ) + + table = ( + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*\n\n" + ) + + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) + + table += ( + "|-" + ("-" * w1) + + "-|-" + ("-" * w2) + + "-|-" + ("-" * w3) + + "-|-" + ("-" * w4) + + "-|\n" + ) + + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" + ) + + output_path = Path(args.output) + output_path.write_text(table) + + print() + print(table) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/ci/dependencies/driver-manager-setup-google-driver.sh b/ci/dependencies/driver-manager-setup-google-driver.sh index 17ca70f06c..fae1fe33c4 100644 --- a/ci/dependencies/driver-manager-setup-google-driver.sh +++ b/ci/dependencies/driver-manager-setup-google-driver.sh @@ -16,43 +16,135 @@ set -euo pipefail +# ============================================================================ +# Google BigQuery ODBC Driver +# ============================================================================ + # Make our include guard clean against set -o nounset. -test -n "${CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__:-}" || declare -i CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__=0 -if ((CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__++ != 0)); then - return 0 -fi # include guard -CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" -export CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR - -export GCS_BUCKET=bq-dev-tools-testing-drivers - -# Check gcloud is installed. -echo "Verifying google cloud SDK is installed using GCS Bucket: "${GCS_BUCKET} -if [ "$(gsutil ls gs://${GCS_BUCKET}/odbc | grep -c odbc-driver.zip)" -eq 0 ]; then - echo 'ODBC driver not found for download: exiting...' - exit 1 -fi +test -n "${CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__:-}" || \ + declare -i CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__=0 + +if ((CI_DEPENDENCIES_GOOGLE_DRIVER_MANAGER_SETUP_SH__++ == 0)); then + + CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" + export CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR + + export GCS_BUCKET=bq-dev-tools-testing-drivers + + # Check Google driver is available. + echo "Verifying Google BigQuery ODBC driver using GCS Bucket: ${GCS_BUCKET}" + + if [ "$(gsutil ls "gs://${GCS_BUCKET}/odbc" | grep -c 'odbc-driver.zip')" -eq 0 ]; then + echo 'Google BigQuery ODBC driver not found for download: exiting...' + exit 1 + fi + + # Configure connection credentials. + echo 'Configuring Google driver connection credentials...' + + mkdir -p /opt/odbc-driver/connection + cd /opt/odbc-driver + + gcloud secrets versions access latest \ + --secret=service-account-auth-keys \ + --out-file="/opt/odbc-driver/connection/key.json" + + echo 'Verifying Google driver connection keys file size...' + + if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then + echo 'Invalid connection keys: exiting...' + exit 1 + fi + + # Configure Google driver environment variables. + echo 'Configuring environment variables for Google BigQuery ODBC driver...' + + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/usr/local/lib/" + export ODBCSYSINI=/opt/odbc-driver + export ODBCINI=/opt/odbc-driver/odbc.ini + export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/opt/odbc-driver/connection/key.json + export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc.ini + export GOOGLEBIGQUERYODBCINI_UTF16=/opt/odbc-driver/googlebigqueryodbc_utf16.ini + export GOOGLEBIGQUERYODBCINI_UTF8=/opt/odbc-driver/googlebigqueryodbc_utf8.ini + + cd "$CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" + + echo '**** Google BigQuery ODBC Driver setup END ****' -# Configure connection credentials for the driver. -echo 'Configuring Connection Credentials...' -mkdir -p /opt/odbc-driver/connection -cd /opt/odbc-driver -gcloud secrets versions access latest --secret=service-account-auth-keys --out-file="/opt/odbc-driver/connection/key.json" -echo 'Verifying Connection Keys File Size...' -if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then - echo 'Invalid connection keys: exiting...' - exit 1 fi -# Configure environment variables -echo 'Configuring Environment Variables For ODBC Driver...' -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib/ -export ODBCSYSINI=/opt/odbc-driver -export ODBCINI=/opt/odbc-driver/odbc.ini -export CPP_BIGQUERY_ODBC_TEST_SERVICE_ACCOUNT_AUTH_KEY=/opt/odbc-driver/connection/key.json -export GOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc.ini -export GOOGLEBIGQUERYODBCINI_UTF16=/opt/odbc-driver/googlebigqueryodbc_utf16.ini -export GOOGLEBIGQUERYODBCINI_UTF8=/opt/odbc-driver/googlebigqueryodbc_utf8.ini -cd "$CPP_GOOGLE_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" - -echo '**** ODBC Driver installation END****' + +# ============================================================================ +# Simba ODBC Driver +# ============================================================================ + +# Make our include guard clean against set -o nounset. +test -n "${CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__:-}" || \ + declare -i CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__=0 + +if ((CI_DEPENDENCIES_DRIVER_MANAGER_SETUP_SH__++ == 0)); then + + CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR="$(pwd)" + export CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR + + export GCS_BUCKET=bq-dev-tools-testing-drivers + export DRIVER_VERSION=3.3.1.3003 + + # Check Simba driver is available. + echo "Verifying Simba ODBC driver using GCS Bucket: ${GCS_BUCKET}" + + if [ "$(gsutil ls "gs://${GCS_BUCKET}/odbc" | grep -c "odbc-driver.${DRIVER_VERSION}.zip")" -eq 0 ]; then + echo 'Simba ODBC driver not found for download: exiting...' + exit 1 + fi + + # Configure connection credentials. + echo 'Configuring Simba connection credentials...' + + mkdir -p /opt/odbc-driver/connection + cd /opt/odbc-driver + + gcloud secrets versions access latest \ + --secret=service-account-auth-keys \ + --out-file="/opt/odbc-driver/connection/key.json" + + echo 'Verifying Simba connection keys file size...' + + if [ "$(stat -c%s /opt/odbc-driver/connection/key.json)" -lt 100 ]; then + echo 'Invalid connection keys: exiting...' + exit 1 + fi + + # Install Simba ODBC driver. + echo 'Installing Simba ODBC driver...' + + gsutil -m cp \ + "gs://${GCS_BUCKET}/odbc/odbc-driver.${DRIVER_VERSION}.zip" \ + . + + unzip -qq "odbc-driver.${DRIVER_VERSION}.zip" + + echo 'Verifying Simba driver install directory...' + + if [ "$( + shopt -s nullglob + set -- /opt/odbc-driver/*googlebigqueryodbc* + echo $# + )" -eq 0 ]; then + echo 'Simba ODBC driver not installed: exiting...' + exit 1 + fi + + # Configure Simba environment variables. + echo 'Configuring environment variables for Simba ODBC driver...' + + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/usr/local/lib/" + export ODBCINI=/opt/odbc-driver/googlebigqueryodbc/odbc.ini + export ODBCINSTINI=/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini + export SIMBAGOOGLEBIGQUERYODBCINI=/opt/odbc-driver/googlebigqueryodbc/lib/simba.googlebigqueryodbc.ini + + cd "$CPP_BIGQUERY_ODBC_DRIVER_MANAGER_SETUP_CURR_DIR" + + echo '**** Simba ODBC Driver setup END ****' + +fi \ No newline at end of file diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc index 7d640c85b0..698485f7b1 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/connection_test.cc @@ -988,7 +988,7 @@ TEST(ConnectionTest, SuccessForExternalAuthWithBYOIDProperties) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { +/*TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { auto conn = std::make_shared(); std::string conn_str = kDefaultConnectionString + @@ -1012,7 +1012,7 @@ TEST(ConnectionTest, VerifyServiceAccountImpersonationEmail) { EXPECT_STREQ(user_email, kImpersonatedAccountEmail.c_str()); EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); -} +}*/ TEST(ConnectionTest, VerifyServiceAccountImpersonationEmailInvalidFails) { auto conn = std::make_shared(); diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index 3e442a03a1..8fc1ff0b89 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -390,11 +390,11 @@ INSTANTIATE_TEST_SUITE_P( std::make_tuple("new_timestamp_table", "SELECT * FROM " "`bigquery-devtools-drivers.kirltest.new_timestamp_" - "table` LIMIT 1000000"), + "table` LIMIT 10"), std::make_tuple("all_bq_types_2", "SELECT * FROM " "`bigquery-devtools-drivers.INTEGRATION_TEST_FORMAT." - "all_bq_types_2` LIMIT 1000000"), + "all_bq_types_2` LIMIT 10"), std::make_tuple( "nyc311_service_requests", "SELECT nyc311.unique_key AS V1, nyc311.descriptor AS V2, " @@ -412,7 +412,7 @@ INSTANTIATE_TEST_SUITE_P( "CAST(nyc311.closed_date AS STRING) AS V22 FROM " "`bigquery-public-data.new_york_311.311_service_requests` AS " "nyc311 " - "LIMIT 1000000;") + "LIMIT 10;") // TODO: Re-enable this benchmark once HTAPI Arrow supports all data // types. Currently SQLExecDirect fails with: // "[Google][ODBC BigQuery Driver] Internal Error: Unsupported arrow