From 9789de68946e856dd7ac662ef0986e082fca510d Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 12:22:14 +0000 Subject: [PATCH 01/33] impl(bq_driver): Adding performance pipeline for linux --- ci/cloudbuild/builds/lib/benchmark_results.py | 417 ++++++++++++++++++ .../builds/linux-bq-driver-benchmark.sh | 403 +++++++++++++++++ .../linux-bq-driver-benchmark-ci.yaml | 29 ++ 3 files changed, 849 insertions(+) create mode 100644 ci/cloudbuild/builds/lib/benchmark_results.py create mode 100755 ci/cloudbuild/builds/linux-bq-driver-benchmark.sh create mode 100644 ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..a26391a8f7 --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,417 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import re +import statistics +from pathlib import Path + +TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") + +FAILED_RE = re.compile( + r"\[\s*FAILED\s*\]\s+(.+?)(?:\s+\([\d.]+\s*(?:ns|us|ms|s)\))?\s*$" +) + + +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + name = name.strip().rstrip(",") + + # Ignore GTest summary lines such as: + # + # 19 tests, listed below: + # + if re.match(r"^\d+\s+tests?,\s+listed below:$", name): + return None + + # Remove GTest parameter description. + # + # Example: + # + # Benchmark/all_bq_types_2, where GetParam() = + # ("all_bq_types_2", "SELECT * FROM ...") + # + # becomes: + # + # Benchmark/all_bq_types_2 + # + name = re.sub( + r",\s*where\s+GetParam\(\)\s*=.*$", + "", + name, + ) + + # Remove everything before the first '.'. + # + # Example: + # + # Instantiation/TestSuite.TestCase/0 + # + # becomes: + # + # TestCase/0 + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy HTAPI suffixes. + name = re.sub( + r"/(?:With|Without)HTAPI$", + "", + name, + ) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + + if unit == "us": + return value / 1000.0 + + if unit == "ns": + return value / 1_000_000.0 + + return value + + +def format_ms(value): + """Format milliseconds for the benchmark table.""" + if value is None: + return "N/A" + + return f"{value:.0f}ms" + + +def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + A benchmark is run multiple times. + + Rules: + 1. Test passes in every iteration: + -> use median execution time. + + 2. Test fails in ANY iteration: + -> return None, displayed as N/A. + + 3. Test is completely missing: + -> return None, displayed as N/A. + """ + path = Path(path) + + if not path.exists(): + raise FileNotFoundError(f"Benchmark output not found: {path}") + + samples = {} + failed_tests = set() + all_tests = set() + + for line in path.read_text(errors="replace").splitlines(): + + # --------------------------------------------------------------- + # Successful test + # --------------------------------------------------------------- + + time_match = TIME_RE.search(line) + + if time_match: + test_name = clean_test_name(time_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + value = float(time_match.group(2)) + unit = time_match.group(3).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + test_name, + [], + ).append(value_ms) + + all_tests.add(test_name) + + continue + + # --------------------------------------------------------------- + # Failed test + # --------------------------------------------------------------- + + failed_match = FAILED_RE.search(line) + + if failed_match: + test_name = clean_test_name(failed_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + failed_tests.add(test_name) + all_tests.add(test_name) + + # ------------------------------------------------------------------- + # Build final result. + # ------------------------------------------------------------------- + + results = {} + + for test_name in all_tests: + + # If a test failed even once, report N/A. + if test_name in failed_tests: + results[test_name] = None + continue + + values = samples.get( + test_name, + [], + ) + + if not values: + results[test_name] = None + continue + + # Same behavior as the GitHub Actions implementation: + # median of all successful iterations. + results[test_name] = statistics.median(values) + + return results + + +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + + Negative = faster/improvement. + Positive = slower/degradation. + """ + if value_ms is None or reference_ms is None or reference_ms == 0: + return " (N/A)" + + pct = round(((value_ms - reference_ms) / reference_ms) * 100) + + if pct > 0: + return f" (+{pct}%)" + + if pct < 0: + return f" ({pct}%)" + + return " (0%)" + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--existing", + required=True, + help="Existing driver benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + + args = parser.parse_args() + + # ------------------------------------------------------------------- + # Parse benchmark outputs. + # ------------------------------------------------------------------- + + existing_data = parse_gtest_output(args.existing) + + current_data = parse_gtest_output(args.current) + + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found. " + "Existing Driver will be shown as N/A." + ) + + # ------------------------------------------------------------------- + # Union of all test names. + # ------------------------------------------------------------------- + + all_tests = ( + set(existing_data.keys()) | set(current_data.keys()) | set(main_data.keys()) + ) + + sorted_tests = sorted(all_tests) + + rows = [] + + for test_name in sorted_tests: + + existing_ms = existing_data.get(test_name) + + current_ms = current_data.get(test_name) + + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + + current_raw = format_ms(current_ms) + + main_raw = format_ms(main_ms) + + # --------------------------------------------------------------- + # Current vs Existing + # --------------------------------------------------------------- + + current_pct = "" + + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + # --------------------------------------------------------------- + # Main vs Current + # --------------------------------------------------------------- + + main_pct = "" + + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + + main_value = f"{main_raw}{main_pct}" + + rows.append( + ( + test_name, + existing_raw, + current_value, + main_value, + ) + ) + + # ------------------------------------------------------------------- + # Generate Markdown table. + # + # This intentionally follows the existing GHA table format. + # ------------------------------------------------------------------- + + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" + + w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) + + w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) + + w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) + + w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) + + table = ( + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*" + "\n\n" + ) + + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) + + table += ( + "|-" + + ("-" * w1) + + "-|-" + + ("-" * w2) + + "-|-" + + ("-" * w3) + + "-|-" + + ("-" * w4) + + "-|\n" + ) + + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" + ) + + # ------------------------------------------------------------------- + # Write output. + # ------------------------------------------------------------------- + + output_path = Path(args.output) + + output_path.write_text(table) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh new file mode 100755 index 0000000000..e92483b63f --- /dev/null +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -0,0 +1,403 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +# ============================================================================ +# Configuration +# ============================================================================ + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" + +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${BUILD_DIR}" +rm -rf "${RESULTS_DIR}" + +mkdir -p "${RESULTS_DIR}" + +# ============================================================================ +# Branch +# ============================================================================ + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +)" + +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo + +# ============================================================================ +# Result files +# ============================================================================ + +CURRENT_RESULT="${RESULTS_DIR}/current.txt" +MAIN_RESULT="${RESULTS_DIR}/main.txt" +EXISTING_RESULT="${RESULTS_DIR}/existing.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" + +# ============================================================================ +# Driver locations +# ============================================================================ + +DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" + +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" + +# ============================================================================ +# ODBC configuration +# ============================================================================ + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +# Export VCPKG version. +VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +export VCPKG_VERSION + +echo "Using VCPKG_VERSION=$VCPKG_VERSION" + +# ============================================================================ +# Vcpkg install and configure +# ============================================================================ + +export VCPKG_ROOT=/vcpkg + +git clone --branch "$VCPKG_VERSION" \ + https://github.com/microsoft/vcpkg.git \ + "$VCPKG_ROOT" + +cd "$VCPKG_ROOT" +git checkout "$VCPKG_VERSION" + +# Bootstrap. +./bootstrap-vcpkg.sh -disableMetrics + +cd "$WORKSPACE_DIR" + +# This is the name of DSN set in odbc.ini. +mapfile -t cmake_args < <(cmake::common_args) + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini +export ODBCINI="${GOOGLE_ODBCINI}" + +io::run cmake -B "$BUILD_DIR" \ + "${cmake_args[@]}" \ + -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ + -DCMAKE_CXX_STANDARD=17 \ + -DODBC_INTEGRATION_TESTING=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON \ + -DODBC_DEMO_TESTING=OFF \ + -DODBC_EXAMPLES=OFF \ + -DODBC_UNIT_TESTING=OFF \ + -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF + +io::run cmake --build "${BUILD_DIR}" + +# ============================================================================ +# Publish Google driver .so for performance benchmarks +# ============================================================================ + +DRIVER_SO="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "${DRIVER_SO}" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " ${DRIVER_SO}" + exit 1 +fi + +echo "Google driver found:" +ls -lh "${DRIVER_SO}" + +HAS_MAIN_DRIVER=false + +if [[ "${BRANCH_NAME}" == "main" ]]; then + echo "Main branch: uploading Google driver artifact..." + + gcloud storage cp \ + "${DRIVER_SO}" \ + "${MAIN_SO_GCS}" + + echo "Uploaded:" + echo " ${MAIN_SO_GCS}" +else + echo "Non-main branch: downloading Google Main driver..." + + if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true + else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + fi +fi + +# ============================================================================ +# Validate ODBC configuration +# ============================================================================ + +if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 +fi + +if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 +fi + +# ============================================================================ +# Run benchmark +# ============================================================================ + +run_benchmark() { + local name="$1" + local output_file="$2" + local dsn="$3" + local bq_tests_flag="$4" + + echo + echo "Reconfiguring performance_test for ${name}" + echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" + echo "------------------------------------------------------------" + + io::run cmake -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" + + io::run cmake --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)" + + PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + fi + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true + + return 1 + fi + + echo "performance_test:" + echo " ${PERFORMANCE_TEST}" + + : >"${output_file}" + + local failed_iterations=0 + + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo + echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" + + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >>"${output_file}" + + echo "=== ODBC tests DSN is====${ODBC_TESTS_DSN}===" + echo "=== ODBCINI is====${dsn}===" + + set +e + + ODBCINI="${dsn}" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "${PERFORMANCE_TEST}" \ + >>"${output_file}" 2>&1 + + run_exit=$? + + set -e + + if [[ "${run_exit}" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + echo "WARNING: Continuing with remaining iterations." + + echo "============================================================" + echo "Detailed failure for ${name}, iteration ${i}:" + echo "============================================================" + + grep -E -B 20 -A 20 \ + '\[ *FAILED *\]|Failure|FAILED|ERROR|Error|error|SQLSTATE|Diagnostic|diagnostic|SQL_ERROR|SQLConnect|SQLDriverConnect|Exception|exception' \ + "${output_file}" || true + + echo "============================================================" + + failed_iterations=$((failed_iterations + 1)) + fi + done + + echo + echo "${name} benchmark completed." + echo "Failed iterations: ${failed_iterations}/${BENCHMARK_ITERATIONS}" + echo "Raw result: ${output_file}" + + # Benchmark failures must not fail Cloud Build. + return 0 +} + +# ============================================================================ +# Google Current +# +# The existing Google DSN points to DRIVER_PATH. +# Replace the driver binary before running the benchmark. +# ============================================================================ + +cp \ + /opt/odbc-driver/roots.pem \ + "${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/roots.pem" + +echo +echo "Preparing Google Current" +echo "------------------------------------------------------------" + +echo "Google Current driver:" +ls -lh "${DRIVER_SO}" + +run_benchmark \ + "Google Current" \ + "${CURRENT_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" + +# ============================================================================ +# Google Main +# ============================================================================ + +echo +echo "Preparing Google Main" +echo "------------------------------------------------------------" + +if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + cp "${MAIN_SO}" "${DRIVER_PATH}" + + ls -lh "${DRIVER_PATH}" + + run_benchmark \ + "Google Main" \ + "${MAIN_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" +else + echo "Google Main benchmark skipped: main driver artifact unavailable." + : >"$MAIN_RESULT" +fi + +# ============================================================================ +# Existing +# ============================================================================ + +echo +echo "Preparing Existing Driver" +echo "------------------------------------------------------------" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBCINI="${EXISTING_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + +run_benchmark \ + "Existing" \ + "${EXISTING_RESULT}" \ + "${EXISTING_ODBCINI}" \ + "OFF" + +# ============================================================================ +# Generate comparison +# ============================================================================ + +echo +echo "============================================================" +echo "Generating benchmark comparison" +echo "============================================================" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --output "${SUMMARY_RESULT}" + +# ============================================================================ +# Upload results +# ============================================================================ + +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" + +echo +echo "Uploading benchmark results" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_RESULT}" \ + "${MAIN_RESULT}" \ + "${EXISTING_RESULT}" \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/" + +echo +echo "============================================================" +echo "Benchmark completed successfully" +echo "============================================================" diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..089990fb8a --- /dev/null +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + branch: ^main$ +name: linux-bq-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-bq-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci From 6c9b60d1b869fc3b658877f61192a17882a6b02f Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 13:56:38 +0000 Subject: [PATCH 02/33] test --- .../integration-production-bq-driver-dm.sh | 37 +++++- .../builds/linux-bq-driver-benchmark.sh | 107 +++++------------- 2 files changed, 67 insertions(+), 77 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..96d1412c20 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -75,11 +75,46 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_CXX_STANDARD=17 \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=ON \ + -DODBC_DEMO_TESTING=OFF \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then + DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + + if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 + fi + + echo "Google driver found:" + ls -lh "$DRIVER_SO" + + # Sanitize branch name for use in GCS path. + SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' + ) + + PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + + echo "Uploading Google driver artifact..." + echo "Branch: ${SANITIZED_BRANCH}" + + gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + + echo "Google driver benchmark artifact uploaded:" + echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +fi # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index e92483b63f..076eae454e 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -88,28 +88,6 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -# Export VCPKG version. -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) -export VCPKG_VERSION - -echo "Using VCPKG_VERSION=$VCPKG_VERSION" - -# ============================================================================ -# Vcpkg install and configure -# ============================================================================ - -export VCPKG_ROOT=/vcpkg - -git clone --branch "$VCPKG_VERSION" \ - https://github.com/microsoft/vcpkg.git \ - "$VCPKG_ROOT" - -cd "$VCPKG_ROOT" -git checkout "$VCPKG_VERSION" - -# Bootstrap. -./bootstrap-vcpkg.sh -disableMetrics - cd "$WORKSPACE_DIR" # This is the name of DSN set in odbc.ini. @@ -124,57 +102,6 @@ export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_ export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini export ODBCINI="${GOOGLE_ODBCINI}" -io::run cmake -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=OFF \ - -DODBC_EXAMPLES=OFF \ - -DODBC_UNIT_TESTING=OFF \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF - -io::run cmake --build "${BUILD_DIR}" - -# ============================================================================ -# Publish Google driver .so for performance benchmarks -# ============================================================================ - -DRIVER_SO="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -if [[ ! -f "${DRIVER_SO}" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" - echo " ${DRIVER_SO}" - exit 1 -fi - -echo "Google driver found:" -ls -lh "${DRIVER_SO}" - -HAS_MAIN_DRIVER=false - -if [[ "${BRANCH_NAME}" == "main" ]]; then - echo "Main branch: uploading Google driver artifact..." - - gcloud storage cp \ - "${DRIVER_SO}" \ - "${MAIN_SO_GCS}" - - echo "Uploaded:" - echo " ${MAIN_SO_GCS}" -else - echo "Non-main branch: downloading Google Main driver..." - - if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then - echo "Main Google driver downloaded successfully." - HAS_MAIN_DRIVER=true - else - echo "WARNING: Main Google driver was not found." - echo "WARNING: Google Main benchmark will be skipped." - fi -fi - # ============================================================================ # Validate ODBC configuration # ============================================================================ @@ -191,6 +118,35 @@ if [[ ! -f "${EXISTING_ODBCINI}" ]]; then exit 1 fi +# --------------------------------------------------------------------------- +# Download Google Current driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google Current driver" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + +# --------------------------------------------------------------------------- +# Download Google Main driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" + +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi + # ============================================================================ # Run benchmark # ============================================================================ @@ -303,8 +259,8 @@ echo echo "Preparing Google Current" echo "------------------------------------------------------------" -echo "Google Current driver:" -ls -lh "${DRIVER_SO}" +cp "${CURRENT_SO}" "${DRIVER_PATH}" +ls -lh "${DRIVER_PATH}" run_benchmark \ "Google Current" \ @@ -322,7 +278,6 @@ echo "------------------------------------------------------------" if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then cp "${MAIN_SO}" "${DRIVER_PATH}" - ls -lh "${DRIVER_PATH}" run_benchmark \ From d6e21b48a2804caa7fc834a799df1a4a384c383d Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 14:05:06 +0000 Subject: [PATCH 03/33] test --- .../examples/catalog_performance_example.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index 3e442a03a1..8fc1ff0b89 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -390,11 +390,11 @@ INSTANTIATE_TEST_SUITE_P( std::make_tuple("new_timestamp_table", "SELECT * FROM " "`bigquery-devtools-drivers.kirltest.new_timestamp_" - "table` LIMIT 1000000"), + "table` LIMIT 10"), std::make_tuple("all_bq_types_2", "SELECT * FROM " "`bigquery-devtools-drivers.INTEGRATION_TEST_FORMAT." - "all_bq_types_2` LIMIT 1000000"), + "all_bq_types_2` LIMIT 10"), std::make_tuple( "nyc311_service_requests", "SELECT nyc311.unique_key AS V1, nyc311.descriptor AS V2, " @@ -412,7 +412,7 @@ INSTANTIATE_TEST_SUITE_P( "CAST(nyc311.closed_date AS STRING) AS V22 FROM " "`bigquery-public-data.new_york_311.311_service_requests` AS " "nyc311 " - "LIMIT 1000000;") + "LIMIT 10;") // TODO: Re-enable this benchmark once HTAPI Arrow supports all data // types. Currently SQLExecDirect fails with: // "[Google][ODBC BigQuery Driver] Internal Error: Unsupported arrow From 86c12cd4dfe96975f85315bc4920f3514c9b722e Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 14:36:49 +0000 Subject: [PATCH 04/33] test --- ci/cloudbuild/builds/bq-driver-release.sh | 24 +++++++++++++++++++ .../integration-production-bq-driver-dm.sh | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index a18385d934..bdf0d915d9 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -60,6 +60,12 @@ fi # Run the integration tests mapfile -t cmake_args < <(cmake::common_args) +echo "============================================================" +echo "CMake common args" +echo "============================================================" +printf ' %s\n' "${cmake_args[@]}" +echo "============================================================" + BUILD_DIR="/opt/odbc-driver" # This is the name of DSN set in odbc.ini export ODBC_TESTS_DSN="SampleDSNGoogleDriver" @@ -80,6 +86,24 @@ io::run cmake -B "$BUILD_DIR" \ -DPROJECT_VERSION="${VERSION}" io::run cmake --build cmake-out + +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +echo "===== DRIVER =====" +ls -lh "${DRIVER_SO}" + +echo "===== LDD =====" +ldd "${DRIVER_SO}" | grep -iE 'arrow|protobuf|absl|grpc' || true + +echo "===== RPATH/RUNPATH =====" +readelf -d "${DRIVER_SO}" | grep -E 'RPATH|RUNPATH' || true + +echo "===== NEEDED =====" +readelf -d "${DRIVER_SO}" | grep NEEDED || true + +echo "===== ARROW UNDEFINED SYMBOL =====" +nm -D "${DRIVER_SO}" 2>/dev/null | + grep '_ZSteqIN5arrow5ArrayEEbRKSt10shared_ptrIT_EDn' || true # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 96d1412c20..22c259cdf8 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -52,6 +52,12 @@ io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" -- # Run the integration tests mapfile -t cmake_args < <(cmake::common_args) +echo "============================================================" +echo "CMake common args" +echo "============================================================" +printf ' %s\n' "${cmake_args[@]}" +echo "============================================================" + BUILD_DIR="/opt/odbc-driver" # This is the name of DSN set in odbc.ini export ODBC_TESTS_DSN="SampleDSNGoogleDriver" @@ -81,6 +87,24 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out + +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +echo "===== DRIVER =====" +ls -lh "${DRIVER_SO}" + +echo "===== LDD =====" +ldd "${DRIVER_SO}" | grep -iE 'arrow|protobuf|absl|grpc' || true + +echo "===== RPATH/RUNPATH =====" +readelf -d "${DRIVER_SO}" | grep -E 'RPATH|RUNPATH' || true + +echo "===== NEEDED =====" +readelf -d "${DRIVER_SO}" | grep NEEDED || true + +echo "===== ARROW UNDEFINED SYMBOL =====" +nm -D "${DRIVER_SO}" 2>/dev/null | + grep '_ZSteqIN5arrow5ArrayEEbRKSt10shared_ptrIT_EDn' || true # --------------------------------------------------------------------------- # Publish Google driver .so for performance benchmarks # --------------------------------------------------------------------------- From d4bd525fd24a75241293ca5d39882770ebda7a62 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 15:09:28 +0000 Subject: [PATCH 05/33] test --- .../dockerfiles/ubuntu-22.04-install.Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index d541edeeb7..ff33a74df9 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -32,6 +32,8 @@ RUN apt-get update && \ git \ gcc \ g++ \ + gcc-11 \ + g++-11 \ # Required by Ubsan in Ubuntu 22.04 libunwind-12-dev \ libc++-12-dev \ @@ -69,12 +71,12 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 -# Set clang as default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 +# Set GCC 11 as the default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -ENV CC=clang -ENV CXX=clang++ +ENV CC=gcc +ENV CXX=g++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From 05a9995948e029ef38c8ef1d7048b6e687076061 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 15:37:02 +0000 Subject: [PATCH 06/33] test --- .../ubuntu-22.04-install.Dockerfile | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index ff33a74df9..2072450646 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -78,6 +78,10 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ ENV CC=gcc ENV CXX=g++ +# Keep clang/clang++ available for clang-tidy and other tooling. +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 + # Install modern CMake locally RUN mkdir -p /opt/cmake && \ curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1-linux-x86_64.tar.gz \ @@ -96,8 +100,36 @@ RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded -# files and any temporary artifacts after a successful build to keep the -# image smaller (and with fewer layers) +# files and any temporary artifacts after a successful build to keep +# the image smaller (and with fewer layers). + +WORKDIR /var/tmp/build/abseil-cpp +RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_BUILD_TESTING=OFF \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -S . -B cmake-out -GNinja && \ + cmake --build cmake-out --target install && \ + ldconfig && \ + cd /var/tmp && rm -fr build + +WORKDIR /var/tmp/build/googletest +RUN curl -fsSL https://github.com/google/googletest/archive/v1.15.2.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE="Release" \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -S . -B cmake-out -GNinja && \ + cmake --build cmake-out --target install && \ + ldconfig && \ + cd /var/tmp && rm -fr build + # Install ctcache to speed up our clang-tidy build WORKDIR /var/tmp/build RUN curl -fsSL https://github.com/matus-chochlik/ctcache/archive/0ad2e227e8a981a9c1a6060ee6c8ec144bb976c6.tar.gz | \ From 94ba4f027529bdfd374f0307f4fddd0a8ed9c7f8 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 16:29:20 +0000 Subject: [PATCH 07/33] test --- .../dockerfiles/ubuntu-22.04-install.Dockerfile | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 2072450646..398beff18c 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -32,8 +32,6 @@ RUN apt-get update && \ git \ gcc \ g++ \ - gcc-11 \ - g++-11 \ # Required by Ubsan in Ubuntu 22.04 libunwind-12-dev \ libc++-12-dev \ @@ -71,17 +69,13 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 -# Set GCC 11 as the default compiler -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 - -ENV CC=gcc -ENV CXX=g++ - -# Keep clang/clang++ available for clang-tidy and other tooling. +# Set clang as default RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 +ENV CC=clang +ENV CXX=clang++ + # Install modern CMake locally RUN mkdir -p /opt/cmake && \ curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1-linux-x86_64.tar.gz \ From fe77184e5c28406e568449dc694a6fd2e4e04348 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 17:11:01 +0000 Subject: [PATCH 08/33] test --- ci/cloudbuild/builds/bq-driver-release.sh | 24 ---- .../integration-production-bq-driver-dm.sh | 61 +--------- .../builds/linux-bq-driver-benchmark.sh | 110 +++++++++++++----- .../ubuntu-22.04-install.Dockerfile | 32 +---- .../linux-bq-driver-benchmark-ci.yaml | 2 +- 5 files changed, 82 insertions(+), 147 deletions(-) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index bdf0d915d9..a18385d934 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -60,12 +60,6 @@ fi # Run the integration tests mapfile -t cmake_args < <(cmake::common_args) -echo "============================================================" -echo "CMake common args" -echo "============================================================" -printf ' %s\n' "${cmake_args[@]}" -echo "============================================================" - BUILD_DIR="/opt/odbc-driver" # This is the name of DSN set in odbc.ini export ODBC_TESTS_DSN="SampleDSNGoogleDriver" @@ -86,24 +80,6 @@ io::run cmake -B "$BUILD_DIR" \ -DPROJECT_VERSION="${VERSION}" io::run cmake --build cmake-out - -DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -echo "===== DRIVER =====" -ls -lh "${DRIVER_SO}" - -echo "===== LDD =====" -ldd "${DRIVER_SO}" | grep -iE 'arrow|protobuf|absl|grpc' || true - -echo "===== RPATH/RUNPATH =====" -readelf -d "${DRIVER_SO}" | grep -E 'RPATH|RUNPATH' || true - -echo "===== NEEDED =====" -readelf -d "${DRIVER_SO}" | grep NEEDED || true - -echo "===== ARROW UNDEFINED SYMBOL =====" -nm -D "${DRIVER_SO}" 2>/dev/null | - grep '_ZSteqIN5arrow5ArrayEEbRKSt10shared_ptrIT_EDn' || true # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 22c259cdf8..2524de72a4 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -52,12 +52,6 @@ io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" -- # Run the integration tests mapfile -t cmake_args < <(cmake::common_args) -echo "============================================================" -echo "CMake common args" -echo "============================================================" -printf ' %s\n' "${cmake_args[@]}" -echo "============================================================" - BUILD_DIR="/opt/odbc-driver" # This is the name of DSN set in odbc.ini export ODBC_TESTS_DSN="SampleDSNGoogleDriver" @@ -81,65 +75,12 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_CXX_STANDARD=17 \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=OFF \ + -DODBC_DEMO_TESTING=ON \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=Release \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out -DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -echo "===== DRIVER =====" -ls -lh "${DRIVER_SO}" - -echo "===== LDD =====" -ldd "${DRIVER_SO}" | grep -iE 'arrow|protobuf|absl|grpc' || true - -echo "===== RPATH/RUNPATH =====" -readelf -d "${DRIVER_SO}" | grep -E 'RPATH|RUNPATH' || true - -echo "===== NEEDED =====" -readelf -d "${DRIVER_SO}" | grep NEEDED || true - -echo "===== ARROW UNDEFINED SYMBOL =====" -nm -D "${DRIVER_SO}" 2>/dev/null | - grep '_ZSteqIN5arrow5ArrayEEbRKSt10shared_ptrIT_EDn' || true -# --------------------------------------------------------------------------- -# Publish Google driver .so for performance benchmarks -# --------------------------------------------------------------------------- - -if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then - DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - - if [[ ! -f "$DRIVER_SO" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" - echo " $DRIVER_SO" - exit 1 - fi - - echo "Google driver found:" - ls -lh "$DRIVER_SO" - - # Sanitize branch name for use in GCS path. - SANITIZED_BRANCH=$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' - ) - - PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" - - echo "Uploading Google driver artifact..." - echo "Branch: ${SANITIZED_BRANCH}" - - gcloud storage cp \ - "$DRIVER_SO" \ - "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" - - echo "Google driver benchmark artifact uploaded:" - echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" -fi - # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 076eae454e..6b8430a306 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -88,6 +88,28 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" +# Export VCPKG version. +VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) +export VCPKG_VERSION + +echo "Using VCPKG_VERSION=$VCPKG_VERSION" + +# ============================================================================ +# Vcpkg install and configure +# ============================================================================ + +export VCPKG_ROOT=/vcpkg + +git clone --branch "$VCPKG_VERSION" \ + https://github.com/microsoft/vcpkg.git \ + "$VCPKG_ROOT" + +cd "$VCPKG_ROOT" +git checkout "$VCPKG_VERSION" + +# Bootstrap. +./bootstrap-vcpkg.sh -disableMetrics + cd "$WORKSPACE_DIR" # This is the name of DSN set in odbc.ini. @@ -102,6 +124,58 @@ export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_ export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini export ODBCINI="${GOOGLE_ODBCINI}" +io::run cmake -B "$BUILD_DIR" \ + "${cmake_args[@]}" \ + -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ + -DCMAKE_CXX_STANDARD=17 \ + -DODBC_INTEGRATION_TESTING=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS=ON \ + -DODBC_DEMO_TESTING=OFF \ + -DODBC_EXAMPLES=OFF \ + -DODBC_UNIT_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF + +io::run cmake --build "${BUILD_DIR}" + +# ============================================================================ +# Publish Google driver .so for performance benchmarks +# ============================================================================ + +DRIVER_SO="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +if [[ ! -f "${DRIVER_SO}" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " ${DRIVER_SO}" + exit 1 +fi + +echo "Google driver found:" +ls -lh "${DRIVER_SO}" + +HAS_MAIN_DRIVER=false + +if [[ "${BRANCH_NAME}" == "main" ]]; then + echo "Main branch: uploading Google driver artifact..." + + gcloud storage cp \ + "${DRIVER_SO}" \ + "${MAIN_SO_GCS}" + + echo "Uploaded:" + echo " ${MAIN_SO_GCS}" +else + echo "Non-main branch: downloading Google Main driver..." + + if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true + else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + fi +fi + # ============================================================================ # Validate ODBC configuration # ============================================================================ @@ -118,35 +192,6 @@ if [[ ! -f "${EXISTING_ODBCINI}" ]]; then exit 1 fi -# --------------------------------------------------------------------------- -# Download Google Current driver -# --------------------------------------------------------------------------- - -echo -echo "Downloading Google Current driver" -echo "------------------------------------------------------------" - -gcloud storage cp \ - "${CURRENT_SO_GCS}" \ - "${CURRENT_SO}" - -# --------------------------------------------------------------------------- -# Download Google Main driver -# --------------------------------------------------------------------------- - -echo -echo "Downloading Google driver from main" -echo "------------------------------------------------------------" - -if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then - echo "Main Google driver downloaded successfully." - HAS_MAIN_DRIVER=true -else - echo "WARNING: Main Google driver was not found." - echo "WARNING: Google Main benchmark will be skipped." - HAS_MAIN_DRIVER=false -fi - # ============================================================================ # Run benchmark # ============================================================================ @@ -259,8 +304,8 @@ echo echo "Preparing Google Current" echo "------------------------------------------------------------" -cp "${CURRENT_SO}" "${DRIVER_PATH}" -ls -lh "${DRIVER_PATH}" +echo "Google Current driver:" +ls -lh "${DRIVER_SO}" run_benchmark \ "Google Current" \ @@ -278,6 +323,7 @@ echo "------------------------------------------------------------" if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then cp "${MAIN_SO}" "${DRIVER_PATH}" + ls -lh "${DRIVER_PATH}" run_benchmark \ @@ -355,4 +401,4 @@ echo " ${RESULTS_BUCKET}/" echo echo "============================================================" echo "Benchmark completed successfully" -echo "============================================================" +echo "============================================================" \ No newline at end of file diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 398beff18c..d541edeeb7 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -94,36 +94,8 @@ RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded -# files and any temporary artifacts after a successful build to keep -# the image smaller (and with fewer layers). - -WORKDIR /var/tmp/build/abseil-cpp -RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz | \ - tar -xzf - --strip-components=1 && \ - cmake \ - -DCMAKE_BUILD_TYPE="Release" \ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_BUILD_TESTING=OFF \ - -DABSL_PROPAGATE_CXX_STD=ON \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -S . -B cmake-out -GNinja && \ - cmake --build cmake-out --target install && \ - ldconfig && \ - cd /var/tmp && rm -fr build - -WORKDIR /var/tmp/build/googletest -RUN curl -fsSL https://github.com/google/googletest/archive/v1.15.2.tar.gz | \ - tar -xzf - --strip-components=1 && \ - cmake \ - -DCMAKE_BUILD_TYPE="Release" \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -S . -B cmake-out -GNinja && \ - cmake --build cmake-out --target install && \ - ldconfig && \ - cd /var/tmp && rm -fr build - +# files and any temporary artifacts after a successful build to keep the +# image smaller (and with fewer layers) # Install ctcache to speed up our clang-tidy build WORKDIR /var/tmp/build RUN curl -fsSL https://github.com/matus-chochlik/ctcache/archive/0ad2e227e8a981a9c1a6060ee6c8ec144bb976c6.tar.gz | \ diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index 089990fb8a..f0f0a06783 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -22,7 +22,7 @@ name: linux-bq-driver-benchmark-ci substitutions: _BUILD_NAME: linux-bq-driver-benchmark _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' - _DISTRO: ubuntu-22.04-install + _DISTRO: ubuntu-20.04-release _TRIGGER_TYPE: ci includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS tags: From 974fd972c4b817980f2a32bce3d30e3134863828 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 19:01:58 +0000 Subject: [PATCH 09/33] test --- .../integration-production-bq-driver-dm.sh | 13 +++++++++++ .../ubuntu-22.04-install.Dockerfile | 22 +++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..1a05127a98 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -50,6 +50,7 @@ mapfile -t secrets_bazel < <(secrets::bazel_args) io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... # Run the integration tests +rm -rf cmake-out mapfile -t cmake_args < <(cmake::common_args) BUILD_DIR="/opt/odbc-driver" @@ -77,9 +78,21 @@ io::run cmake -B "$BUILD_DIR" \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ -DODBC_DEMO_TESTING=ON \ -DODBC_EXAMPLES=ON \ + -DCMAKE_BUILD_TYPE=Release \ -DODBC_UNIT_TESTING=OFF \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +echo "===== DRIVER DEPENDENCIES =====" +ldd "$DRIVER_SO" || true + +echo "===== UNDEFINED ARROW SYMBOLS =====" +nm -D -C "$DRIVER_SO" | grep ' U .*arrow' || true + +echo "===== SHARED_PTR SYMBOL =====" +nm -D -C "$DRIVER_SO" | \ + grep 'operator==' || true # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index d541edeeb7..15ae4445e6 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -20,24 +20,22 @@ RUN apt-get update && \ automake \ autotools-dev \ build-essential \ - # Dependency for arrow bison \ clang-12 \ lld-12 \ cmake \ curl \ - # Dependency for arrow flex \ gawk \ git \ gcc \ g++ \ - # Required by Ubsan in Ubuntu 22.04 + gcc-11 \ + g++-11 \ libunwind-12-dev \ libc++-12-dev \ libc++abi-12-dev \ libcurl4-openssl-dev \ - # Needed to use autoreconf libltdl-dev \ libssl-dev \ libtool \ @@ -47,7 +45,6 @@ RUN apt-get update && \ make \ ninja-build \ patch \ - # Needed to use autoreconf perl \ pkg-config \ python3 \ @@ -70,11 +67,18 @@ ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 # Set clang as default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 +# Use GCC 11 / G++ 11 consistently. +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -ENV CC=clang -ENV CXX=clang++ +ENV CC=gcc +ENV CXX=g++ + + +RUN echo "gcc version:" && gcc --version && \ + echo "g++ version:" && g++ --version && \ + echo "cmake version:" && cmake --version && \ + echo "glibc version:" && ldd --version # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From c7d56d3cd0c27ce70c1d795398c54cee4e5d1d20 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 19:22:23 +0000 Subject: [PATCH 10/33] test --- .../dockerfiles/ubuntu-22.04-install.Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 15ae4445e6..2ae28cd2fe 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -67,12 +67,11 @@ ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 # Set clang as default -# Use GCC 11 / G++ 11 consistently. -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=gcc -ENV CXX=g++ +ENV CC=clang +ENV CXX=clang++ RUN echo "gcc version:" && gcc --version && \ From ce160202e8acf95c2ba1ae9549f6ecbbac0c0a8c Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 19:53:20 +0000 Subject: [PATCH 11/33] test --- .../integration-production-bq-driver-dm.sh | 7 ++++++ .../ubuntu-22.04-install.Dockerfile | 25 +++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 1a05127a98..6c2abc672a 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -42,6 +42,13 @@ git checkout "$VCPKG_VERSION" ./bootstrap-vcpkg.sh -disableMetrics cd "$WORKSPACE_DIR" +bazelisk clean --expunge || true +rm -rf "${HOME}/.cache/bazel" +rm -rf bazel-bin bazel-out bazel-testlogs +rm -rf cmake-out +rm -rf vcpkg_installed +rm -rf "${VCPKG_ROOT}/buildtrees" +rm -rf "${VCPKG_ROOT}/packages" # This runs all the unit tests mapfile -t args < <(bazel::common_args) mapfile -t unit_tests_args < <(unit_tests::bazel_args) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 2ae28cd2fe..ac1ca5ed74 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -67,17 +67,20 @@ ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 # Set clang as default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 - -ENV CC=clang -ENV CXX=clang++ - - -RUN echo "gcc version:" && gcc --version && \ - echo "g++ version:" && g++ --version && \ - echo "cmake version:" && cmake --version && \ - echo "glibc version:" && ldd --version +# Use GCC 11 consistently for the C++ build. +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 + +ENV CC=gcc +ENV CXX=g++ + +RUN echo "===== COMPILER CONFIGURATION =====" && \ + echo "CC=${CC}" && \ + echo "CXX=${CXX}" && \ + which gcc && gcc --version && \ + which g++ && g++ --version && \ + which clang && clang --version && \ + echo "==================================" # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From a671933e341c935536fde05c33aa6281427eb516 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 20:11:57 +0000 Subject: [PATCH 12/33] test --- .../builds/integration-production-bq-driver-dm.sh | 7 ------- ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 8 +------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 6c2abc672a..1a05127a98 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -42,13 +42,6 @@ git checkout "$VCPKG_VERSION" ./bootstrap-vcpkg.sh -disableMetrics cd "$WORKSPACE_DIR" -bazelisk clean --expunge || true -rm -rf "${HOME}/.cache/bazel" -rm -rf bazel-bin bazel-out bazel-testlogs -rm -rf cmake-out -rm -rf vcpkg_installed -rm -rf "${VCPKG_ROOT}/buildtrees" -rm -rf "${VCPKG_ROOT}/packages" # This runs all the unit tests mapfile -t args < <(bazel::common_args) mapfile -t unit_tests_args < <(unit_tests::bazel_args) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index ac1ca5ed74..c53ca68506 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -21,8 +21,6 @@ RUN apt-get update && \ autotools-dev \ build-essential \ bison \ - clang-12 \ - lld-12 \ cmake \ curl \ flex \ @@ -32,9 +30,6 @@ RUN apt-get update && \ g++ \ gcc-11 \ g++-11 \ - libunwind-12-dev \ - libc++-12-dev \ - libc++abi-12-dev \ libcurl4-openssl-dev \ libltdl-dev \ libssl-dev \ @@ -57,8 +52,7 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https \ - clang-tidy-12 + apt-transport-https # Needed for the existing driver v3.1.2.1004+ RUN locale-gen en_US.UTF-8 From 9cfc0cde0947cb62ac94be5f09add0defb5147e3 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 20:20:23 +0000 Subject: [PATCH 13/33] test --- ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index c53ca68506..3aeeed3f7c 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -68,13 +68,6 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ ENV CC=gcc ENV CXX=g++ -RUN echo "===== COMPILER CONFIGURATION =====" && \ - echo "CC=${CC}" && \ - echo "CXX=${CXX}" && \ - which gcc && gcc --version && \ - which g++ && g++ --version && \ - which clang && clang --version && \ - echo "==================================" # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From 99fc4fba51a91055a11115bef95d9032892eee59 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 20:39:14 +0000 Subject: [PATCH 14/33] test --- .../ubuntu-22.04-install.Dockerfile | 66 +++++++++++++++---- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 3aeeed3f7c..fe539f73dc 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -16,13 +16,19 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ + apt-get --no-install-recommends install -y \ + software-properties-common gnupg2 && \ + add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ + # Dependency for arrow bison \ cmake \ curl \ + # Dependency for arrow flex \ gawk \ git \ @@ -31,6 +37,7 @@ RUN apt-get update && \ gcc-11 \ g++-11 \ libcurl4-openssl-dev \ + # Needed to use autoreconf libltdl-dev \ libssl-dev \ libtool \ @@ -40,11 +47,10 @@ RUN apt-get update && \ make \ ninja-build \ patch \ + # Needed to use autoreconf perl \ pkg-config \ - python3 \ - python3-dev \ - python3-pip \ + libffi-dev \ tar \ unzip \ zip \ @@ -52,7 +58,7 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https + apt-transport-https # Needed for the existing driver v3.1.2.1004+ RUN locale-gen en_US.UTF-8 @@ -67,7 +73,7 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ ENV CC=gcc ENV CXX=g++ - +RUN ln -s /usr/bin/make /usr/bin/gmake # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -76,19 +82,55 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -# clang-tidy-cache needs python -RUN update-alternatives --install /usr/bin/python python $(which python3) 10 +RUN echo "ninja version: " && ninja --version +RUN echo "g++ version: " && g++ --version +RUN echo "cmake version: " && cmake --version +RUN echo "Glibc version" && ldd --version -COPY ./requirements.txt /var/tmp/ci/requirements.txt -WORKDIR /var/tmp/downloads -RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ - then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi -RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt +WORKDIR /usr/src +RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ + tar -xzf Python-3.10.14.tgz && \ + cd Python-3.10.14 && \ + ./configure --with-ensurepip=install && \ + make -j$(nproc) \ + && make altinstall + +# clang-tidy-cache needs python +RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ + ln -sf /usr/local/bin/python3.10 /usr/bin/python # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded # files and any temporary artifacts after a successful build to keep the # image smaller (and with fewer layers) + +WORKDIR /var/tmp/build/abseil-cpp +RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_BUILD_TESTING=OFF \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -S . -B cmake-out -GNinja && \ + cmake --build cmake-out --target install && \ + ldconfig && \ + cd /var/tmp && rm -fr build + +WORKDIR /var/tmp/build/googletest +RUN curl -fsSL https://github.com/google/googletest/archive/v1.15.2.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE="Release" \ + -DBUILD_SHARED_LIBS=yes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -S . -B cmake-out -GNinja && \ + cmake --build cmake-out --target install && \ + ldconfig && \ + cd /var/tmp && rm -fr build + # Install ctcache to speed up our clang-tidy build WORKDIR /var/tmp/build RUN curl -fsSL https://github.com/matus-chochlik/ctcache/archive/0ad2e227e8a981a9c1a6060ee6c8ec144bb976c6.tar.gz | \ From 734877e6e62127a2147649bef7a4119d99645468 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 20:57:16 +0000 Subject: [PATCH 15/33] test --- ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index fe539f73dc..b6ec9cfbd1 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -66,14 +66,12 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 -# Set clang as default # Use GCC 11 consistently for the C++ build. RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 ENV CC=gcc ENV CXX=g++ -RUN ln -s /usr/bin/make /usr/bin/gmake # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -163,6 +161,7 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. + COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini From db20f42d9cd3aacc3f902dd452e44355535dd14a Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 21:11:50 +0000 Subject: [PATCH 16/33] test --- .../builds/integration-production-bq-driver-dm.sh | 12 ++++++++++++ .../dockerfiles/ubuntu-22.04-install.Dockerfile | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 1a05127a98..6d31bb8050 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -43,6 +43,18 @@ git checkout "$VCPKG_VERSION" cd "$WORKSPACE_DIR" # This runs all the unit tests +echo "=== C/C++ TOOLCHAIN ===" +echo "PATH=${PATH}" +echo "CC=${CC}" +echo "CXX=${CXX}" +command -v gcc || true +command -v g++ || true +command -v gcc-11 || true +command -v g++-11 || true +ls -l /usr/bin/gcc /usr/bin/g++ /usr/bin/gcc-11 /usr/bin/g++-11 2>&1 || true +gcc --version 2>&1 || true +g++ --version 2>&1 || true +echo "=======================" mapfile -t args < <(bazel::common_args) mapfile -t unit_tests_args < <(unit_tests::bazel_args) mapfile -t secrets_bazel < <(secrets::bazel_args) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index b6ec9cfbd1..4d1b181bf1 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -70,8 +70,8 @@ ENV LC_ALL en_US.UTF-8 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -ENV CC=gcc -ENV CXX=g++ +ENV CC=/usr/bin/gcc +ENV CXX=/usr/bin/g++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From b9f95d55938902f2ecc6ab2cbeb693079d471515 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 21:39:28 +0000 Subject: [PATCH 17/33] tets --- google/cloud/odbc/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/odbc/BUILD.bazel b/google/cloud/odbc/BUILD.bazel index 2770ed27c5..eaaf88ceec 100644 --- a/google/cloud/odbc/BUILD.bazel +++ b/google/cloud/odbc/BUILD.bazel @@ -48,6 +48,7 @@ cc_library( copts = ["-std=c++17"], # TODO(sachinpro): remove this when we are able to compile arrow for bazel defines = ["NO_ARROW"], + alwayslink = True, visibility = ["//:__pkg__"], deps = [ "//google/cloud/odbc/bq_client_interface:odbc_bq_client_interface", From 2a22f28725efee738464ef125d5a14bb7e986cdf Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Tue, 25 Aug 2026 21:56:46 +0000 Subject: [PATCH 18/33] test --- .../integration-production-bq-driver-dm.sh | 55 +++++---- .../builds/linux-bq-driver-benchmark.sh | 108 +++++------------- .../ubuntu-22.04-install.Dockerfile | 82 ++++--------- .../linux-bq-driver-benchmark-ci.yaml | 2 +- google/cloud/odbc/BUILD.bazel | 1 - 5 files changed, 89 insertions(+), 159 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 6d31bb8050..bab81ef7ed 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -43,18 +43,6 @@ git checkout "$VCPKG_VERSION" cd "$WORKSPACE_DIR" # This runs all the unit tests -echo "=== C/C++ TOOLCHAIN ===" -echo "PATH=${PATH}" -echo "CC=${CC}" -echo "CXX=${CXX}" -command -v gcc || true -command -v g++ || true -command -v gcc-11 || true -command -v g++-11 || true -ls -l /usr/bin/gcc /usr/bin/g++ /usr/bin/gcc-11 /usr/bin/g++-11 2>&1 || true -gcc --version 2>&1 || true -g++ --version 2>&1 || true -echo "=======================" mapfile -t args < <(bazel::common_args) mapfile -t unit_tests_args < <(unit_tests::bazel_args) mapfile -t secrets_bazel < <(secrets::bazel_args) @@ -62,7 +50,6 @@ mapfile -t secrets_bazel < <(secrets::bazel_args) io::run bazel test "${args[@]}" "${secrets_bazel[@]}" "${unit_tests_args[@]}" --test_tag_filters=unit-tests ... # Run the integration tests -rm -rf cmake-out mapfile -t cmake_args < <(cmake::common_args) BUILD_DIR="/opt/odbc-driver" @@ -90,21 +77,45 @@ io::run cmake -B "$BUILD_DIR" \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ -DODBC_DEMO_TESTING=ON \ -DODBC_EXAMPLES=ON \ - -DCMAKE_BUILD_TYPE=Release \ -DODBC_UNIT_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out -DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" -echo "===== DRIVER DEPENDENCIES =====" -ldd "$DRIVER_SO" || true +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then + DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + + if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 + fi + + echo "Google driver found:" + ls -lh "$DRIVER_SO" -echo "===== UNDEFINED ARROW SYMBOLS =====" -nm -D -C "$DRIVER_SO" | grep ' U .*arrow' || true + # Sanitize branch name for use in GCS path. + SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' + ) -echo "===== SHARED_PTR SYMBOL =====" -nm -D -C "$DRIVER_SO" | \ - grep 'operator==' || true + PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + + echo "Uploading Google driver artifact..." + echo "Branch: ${SANITIZED_BRANCH}" + + gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + + echo "Google driver benchmark artifact uploaded:" + echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +fi # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 6b8430a306..75cbd68afd 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -88,28 +88,6 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -# Export VCPKG version. -VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) -export VCPKG_VERSION - -echo "Using VCPKG_VERSION=$VCPKG_VERSION" - -# ============================================================================ -# Vcpkg install and configure -# ============================================================================ - -export VCPKG_ROOT=/vcpkg - -git clone --branch "$VCPKG_VERSION" \ - https://github.com/microsoft/vcpkg.git \ - "$VCPKG_ROOT" - -cd "$VCPKG_ROOT" -git checkout "$VCPKG_VERSION" - -# Bootstrap. -./bootstrap-vcpkg.sh -disableMetrics - cd "$WORKSPACE_DIR" # This is the name of DSN set in odbc.ini. @@ -124,58 +102,6 @@ export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_ export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini export ODBCINI="${GOOGLE_ODBCINI}" -io::run cmake -B "$BUILD_DIR" \ - "${cmake_args[@]}" \ - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ - -DCMAKE_CXX_STANDARD=17 \ - -DODBC_INTEGRATION_TESTING=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=OFF \ - -DODBC_EXAMPLES=OFF \ - -DODBC_UNIT_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF - -io::run cmake --build "${BUILD_DIR}" - -# ============================================================================ -# Publish Google driver .so for performance benchmarks -# ============================================================================ - -DRIVER_SO="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -if [[ ! -f "${DRIVER_SO}" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" - echo " ${DRIVER_SO}" - exit 1 -fi - -echo "Google driver found:" -ls -lh "${DRIVER_SO}" - -HAS_MAIN_DRIVER=false - -if [[ "${BRANCH_NAME}" == "main" ]]; then - echo "Main branch: uploading Google driver artifact..." - - gcloud storage cp \ - "${DRIVER_SO}" \ - "${MAIN_SO_GCS}" - - echo "Uploaded:" - echo " ${MAIN_SO_GCS}" -else - echo "Non-main branch: downloading Google Main driver..." - - if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then - echo "Main Google driver downloaded successfully." - HAS_MAIN_DRIVER=true - else - echo "WARNING: Main Google driver was not found." - echo "WARNING: Google Main benchmark will be skipped." - fi -fi - # ============================================================================ # Validate ODBC configuration # ============================================================================ @@ -192,6 +118,35 @@ if [[ ! -f "${EXISTING_ODBCINI}" ]]; then exit 1 fi +# --------------------------------------------------------------------------- +# Download Google Current driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google Current driver" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + +# --------------------------------------------------------------------------- +# Download Google Main driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" + +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi + # ============================================================================ # Run benchmark # ============================================================================ @@ -304,8 +259,8 @@ echo echo "Preparing Google Current" echo "------------------------------------------------------------" -echo "Google Current driver:" -ls -lh "${DRIVER_SO}" +cp "${CURRENT_SO}" "${DRIVER_PATH}" +ls -lh "${DRIVER_PATH}" run_benchmark \ "Google Current" \ @@ -323,7 +278,6 @@ echo "------------------------------------------------------------" if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then cp "${MAIN_SO}" "${DRIVER_PATH}" - ls -lh "${DRIVER_PATH}" run_benchmark \ diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 4d1b181bf1..d541edeeb7 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,16 +16,14 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get --no-install-recommends install -y \ - software-properties-common gnupg2 && \ - add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ - apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ # Dependency for arrow bison \ + clang-12 \ + lld-12 \ cmake \ curl \ # Dependency for arrow @@ -34,8 +32,10 @@ RUN apt-get update && \ git \ gcc \ g++ \ - gcc-11 \ - g++-11 \ + # Required by Ubsan in Ubuntu 22.04 + libunwind-12-dev \ + libc++-12-dev \ + libc++abi-12-dev \ libcurl4-openssl-dev \ # Needed to use autoreconf libltdl-dev \ @@ -50,7 +50,9 @@ RUN apt-get update && \ # Needed to use autoreconf perl \ pkg-config \ - libffi-dev \ + python3 \ + python3-dev \ + python3-pip \ tar \ unzip \ zip \ @@ -58,7 +60,8 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https + apt-transport-https \ + clang-tidy-12 # Needed for the existing driver v3.1.2.1004+ RUN locale-gen en_US.UTF-8 @@ -66,12 +69,12 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 ENV LC_ALL en_US.UTF-8 -# Use GCC 11 consistently for the C++ build. -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 +# Set clang as default +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=/usr/bin/gcc -ENV CXX=/usr/bin/g++ +ENV CC=clang +ENV CXX=clang++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -80,55 +83,19 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -RUN echo "ninja version: " && ninja --version -RUN echo "g++ version: " && g++ --version -RUN echo "cmake version: " && cmake --version -RUN echo "Glibc version" && ldd --version - -WORKDIR /usr/src -RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ - tar -xzf Python-3.10.14.tgz && \ - cd Python-3.10.14 && \ - ./configure --with-ensurepip=install && \ - make -j$(nproc) \ - && make altinstall - # clang-tidy-cache needs python -RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ - ln -sf /usr/local/bin/python3.10 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python $(which python3) 10 + +COPY ./requirements.txt /var/tmp/ci/requirements.txt +WORKDIR /var/tmp/downloads +RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ + then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi +RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded # files and any temporary artifacts after a successful build to keep the # image smaller (and with fewer layers) - -WORKDIR /var/tmp/build/abseil-cpp -RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz | \ - tar -xzf - --strip-components=1 && \ - cmake \ - -DCMAKE_BUILD_TYPE="Release" \ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_BUILD_TESTING=OFF \ - -DABSL_PROPAGATE_CXX_STD=ON \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -S . -B cmake-out -GNinja && \ - cmake --build cmake-out --target install && \ - ldconfig && \ - cd /var/tmp && rm -fr build - -WORKDIR /var/tmp/build/googletest -RUN curl -fsSL https://github.com/google/googletest/archive/v1.15.2.tar.gz | \ - tar -xzf - --strip-components=1 && \ - cmake \ - -DCMAKE_BUILD_TYPE="Release" \ - -DBUILD_SHARED_LIBS=yes \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -S . -B cmake-out -GNinja && \ - cmake --build cmake-out --target install && \ - ldconfig && \ - cd /var/tmp && rm -fr build - # Install ctcache to speed up our clang-tidy build WORKDIR /var/tmp/build RUN curl -fsSL https://github.com/matus-chochlik/ctcache/archive/0ad2e227e8a981a9c1a6060ee6c8ec144bb976c6.tar.gz | \ @@ -161,7 +128,6 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. - COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index f0f0a06783..089990fb8a 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -22,7 +22,7 @@ name: linux-bq-driver-benchmark-ci substitutions: _BUILD_NAME: linux-bq-driver-benchmark _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' - _DISTRO: ubuntu-20.04-release + _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS tags: diff --git a/google/cloud/odbc/BUILD.bazel b/google/cloud/odbc/BUILD.bazel index eaaf88ceec..2770ed27c5 100644 --- a/google/cloud/odbc/BUILD.bazel +++ b/google/cloud/odbc/BUILD.bazel @@ -48,7 +48,6 @@ cc_library( copts = ["-std=c++17"], # TODO(sachinpro): remove this when we are able to compile arrow for bazel defines = ["NO_ARROW"], - alwayslink = True, visibility = ["//:__pkg__"], deps = [ "//google/cloud/odbc/bq_client_interface:odbc_bq_client_interface", From bcbc377b4251e35132e3a939cb73c66645dc153c Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 05:51:06 +0000 Subject: [PATCH 19/33] test --- ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index d541edeeb7..1553fc9015 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -74,7 +74,7 @@ RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 ENV CC=clang -ENV CXX=clang++ +ENV CXX="clang++ -stdlib=libstdc++" # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From 2cc10f9b4407ea3c2025079ba7f7b2fa84473df7 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 06:23:09 +0000 Subject: [PATCH 20/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 1 + ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index bab81ef7ed..384d362d9d 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -79,6 +79,7 @@ io::run cmake -B "$BUILD_DIR" \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ + -DVCPKG_TARGET_TRIPLET=x64-linux \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 1553fc9015..d541edeeb7 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -74,7 +74,7 @@ RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 ENV CC=clang -ENV CXX="clang++ -stdlib=libstdc++" +ENV CXX=clang++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From 721f175c228d76cdfc9ed16d0d1b0eafb5fb42f6 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 07:07:20 +0000 Subject: [PATCH 21/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 384d362d9d..f4dc0452d7 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -26,7 +26,7 @@ source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) - +rm -rf "$WORKSPACE_DIR"/vcpkg_installed # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION @@ -79,7 +79,7 @@ io::run cmake -B "$BUILD_DIR" \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DVCPKG_TARGET_TRIPLET=x64-linux \ + -DVCPKG_TARGET_TRIPLET=x64-linux-clang \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out From 7faad5c5918f097f1bcd0bea9e0bf2c74ac6e6a9 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 07:14:27 +0000 Subject: [PATCH 22/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index f4dc0452d7..54f9e4e15b 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -79,7 +79,7 @@ io::run cmake -B "$BUILD_DIR" \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DVCPKG_TARGET_TRIPLET=x64-linux-clang \ + -DVCPKG_TARGET_TRIPLET=x64-linux \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out From ab5ae4a35cfe5faf4bbe956a853f69665e0e585b Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 07:26:53 +0000 Subject: [PATCH 23/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 1 - ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 54f9e4e15b..15e8422bcd 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -79,7 +79,6 @@ io::run cmake -B "$BUILD_DIR" \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DVCPKG_TARGET_TRIPLET=x64-linux \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index d541edeeb7..bdf476a19b 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -73,8 +73,10 @@ ENV LC_ALL en_US.UTF-8 RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=clang -ENV CXX=clang++ +ENV CC=/usr/bin/clang-12 +ENV CXX=/usr/bin/clang++-12 +ENV CXXFLAGS="-stdlib=libstdc++" +ENV LDFLAGS="-stdlib=libstdc++" # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From 74c3b517117fa1d597271be053da1937b2ef4b77 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 07:46:03 +0000 Subject: [PATCH 24/33] test --- .../builds/integration-production-bq-driver-dm.sh | 2 ++ ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 15e8422bcd..6fe60da6ce 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -79,6 +79,8 @@ io::run cmake -B "$BUILD_DIR" \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=/usr/bin/gcc-11 \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++-11 \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index bdf476a19b..0b5bd2b718 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -32,6 +32,8 @@ RUN apt-get update && \ git \ gcc \ g++ \ + gcc-11 \ + g++-11 \ # Required by Ubsan in Ubuntu 22.04 libunwind-12-dev \ libc++-12-dev \ @@ -73,10 +75,8 @@ ENV LC_ALL en_US.UTF-8 RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=/usr/bin/clang-12 -ENV CXX=/usr/bin/clang++-12 -ENV CXXFLAGS="-stdlib=libstdc++" -ENV LDFLAGS="-stdlib=libstdc++" +ENV CC=clang +ENV CXX=clang++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ From f227603c95fced3840909fba9809d4a908d9da41 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 07:47:09 +0000 Subject: [PATCH 25/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 6fe60da6ce..805da056c5 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -27,6 +27,8 @@ source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) rm -rf "$WORKSPACE_DIR"/vcpkg_installed +rm -rf "$WORKSPACE_DIR"/cmake-out +rm -rf "/opt/odbc-driver" # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION From 2c4461062a667e7817d815c3fbc362e207928e9f Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 08:09:41 +0000 Subject: [PATCH 26/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 805da056c5..68adfce227 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -77,7 +77,7 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_CXX_STANDARD=17 \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=ON \ + -DODBC_DEMO_TESTING=OFF \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ From 370812c3dc47d16f1e97455d01751e157084453f Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 08:28:17 +0000 Subject: [PATCH 27/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 68adfce227..58e2f806c8 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -28,7 +28,6 @@ source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) rm -rf "$WORKSPACE_DIR"/vcpkg_installed rm -rf "$WORKSPACE_DIR"/cmake-out -rm -rf "/opt/odbc-driver" # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION From bf95242c396496f4eb87b35b6678b615244e8033 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 08:50:36 +0000 Subject: [PATCH 28/33] tset --- .../integration-production-bq-driver-dm.sh | 19 ++++++++++++++++++- google/cloud/odbc/CMakeLists.txt | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 58e2f806c8..cace35bb58 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -83,7 +83,24 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_C_COMPILER=/usr/bin/gcc-11 \ -DCMAKE_CXX_COMPILER=/usr/bin/g++-11 \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -io::run cmake --build cmake-out +io::run cmake --build "$BUILD_DIR" + +DRIVER_SO="$BUILD_DIR/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +echo "============================================================" +echo "Google ODBC driver dependencies" +echo "============================================================" +ldd "$DRIVER_SO" + +echo "============================================================" +echo "Unresolved symbols" +echo "============================================================" +ldd -r "$DRIVER_SO" || true + +echo "============================================================" +echo "Driver RPATH/RUNPATH" +echo "============================================================" +readelf -d "$DRIVER_SO" | grep -E 'NEEDED|RPATH|RUNPATH' || true # --------------------------------------------------------------------------- # Publish Google driver .so for performance benchmarks diff --git a/google/cloud/odbc/CMakeLists.txt b/google/cloud/odbc/CMakeLists.txt index ec04725342..a739f9e754 100644 --- a/google/cloud/odbc/CMakeLists.txt +++ b/google/cloud/odbc/CMakeLists.txt @@ -141,7 +141,7 @@ if (NOT google_cloud_cpp_bigquery_rest_FOUND OR NOT set(GOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND ON) set(GOOGLE_CLOUD_CPP_ENABLE_EXAMPLES OFF) set(BUILD_TESTING OFF) - set(CMAKE_POSITION_INDEPENDENT_CODE=ON) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(GOOGLE_CLOUD_CPP_ENABLE experimental-bigquery_rest oauth2 bigquery resourcemanager serviceusage) From 57ff77483f7e1ffef3d95498ed3d79d4a65d5fa5 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 09:25:57 +0000 Subject: [PATCH 29/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index cace35bb58..1c6f482b78 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -26,8 +26,6 @@ source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) -rm -rf "$WORKSPACE_DIR"/vcpkg_installed -rm -rf "$WORKSPACE_DIR"/cmake-out # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION From a1f75bf5ded86ad5c226bc15168b3741135db7a0 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 09:28:44 +0000 Subject: [PATCH 30/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 1c6f482b78..905ef5133a 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -81,7 +81,7 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_C_COMPILER=/usr/bin/gcc-11 \ -DCMAKE_CXX_COMPILER=/usr/bin/g++-11 \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF -io::run cmake --build "$BUILD_DIR" +io::run cmake --build cmake-out DRIVER_SO="$BUILD_DIR/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" From 0dc9c22f41704206fe6dacf697693fe1d3b34d54 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 09:50:49 +0000 Subject: [PATCH 31/33] test --- ci/cloudbuild/builds/integration-production-bq-driver-dm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 905ef5133a..40c4abce3a 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -26,6 +26,7 @@ source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) +rm -rf vcpkg_installed # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION @@ -80,10 +81,11 @@ io::run cmake -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=/usr/bin/gcc-11 \ -DCMAKE_CXX_COMPILER=/usr/bin/g++-11 \ + -DVCPKG_TARGET_TRIPLET=x64-linux \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out -DRIVER_SO="$BUILD_DIR/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" echo "============================================================" echo "Google ODBC driver dependencies" From 9f941f4ef035567efb9c6f4124025528a6fade89 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 10:26:32 +0000 Subject: [PATCH 32/33] test --- .../integration-production-bq-driver-dm.sh | 61 +------------------ .../ubuntu-22.04-install.Dockerfile | 2 - google/cloud/odbc/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 61 deletions(-) diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 40c4abce3a..02e625f9e9 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -26,7 +26,7 @@ source module ci/cloudbuild/builds/lib/unit-tests.sh source module ci/lib/io.sh WORKSPACE_DIR=$(pwd) -rm -rf vcpkg_installed + # Export as env variable VCPKG_VERSION=$(cat /tmp/vcpkg-version.txt) export VCPKG_VERSION @@ -73,70 +73,15 @@ io::run cmake -B "$BUILD_DIR" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ - -DODBC_DEMO_TESTING=OFF \ + -DODBC_DEMO_TESTING=ON \ -DODBC_EXAMPLES=ON \ -DODBC_UNIT_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=/usr/bin/gcc-11 \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++-11 \ - -DVCPKG_TARGET_TRIPLET=x64-linux \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out -DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - -echo "============================================================" -echo "Google ODBC driver dependencies" -echo "============================================================" -ldd "$DRIVER_SO" - -echo "============================================================" -echo "Unresolved symbols" -echo "============================================================" -ldd -r "$DRIVER_SO" || true - -echo "============================================================" -echo "Driver RPATH/RUNPATH" -echo "============================================================" -readelf -d "$DRIVER_SO" | grep -E 'NEEDED|RPATH|RUNPATH' || true - -# --------------------------------------------------------------------------- -# Publish Google driver .so for performance benchmarks -# --------------------------------------------------------------------------- - -if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then - DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" - - if [[ ! -f "$DRIVER_SO" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" - echo " $DRIVER_SO" - exit 1 - fi - - echo "Google driver found:" - ls -lh "$DRIVER_SO" - - # Sanitize branch name for use in GCS path. - SANITIZED_BRANCH=$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' - ) - - PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" - - echo "Uploading Google driver artifact..." - echo "Branch: ${SANITIZED_BRANCH}" - - gcloud storage cp \ - "$DRIVER_SO" \ - "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" - - echo "Google driver benchmark artifact uploaded:" - echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" -fi - # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 0b5bd2b718..d541edeeb7 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -32,8 +32,6 @@ RUN apt-get update && \ git \ gcc \ g++ \ - gcc-11 \ - g++-11 \ # Required by Ubsan in Ubuntu 22.04 libunwind-12-dev \ libc++-12-dev \ diff --git a/google/cloud/odbc/CMakeLists.txt b/google/cloud/odbc/CMakeLists.txt index a739f9e754..ec04725342 100644 --- a/google/cloud/odbc/CMakeLists.txt +++ b/google/cloud/odbc/CMakeLists.txt @@ -141,7 +141,7 @@ if (NOT google_cloud_cpp_bigquery_rest_FOUND OR NOT set(GOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND ON) set(GOOGLE_CLOUD_CPP_ENABLE_EXAMPLES OFF) set(BUILD_TESTING OFF) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(CMAKE_POSITION_INDEPENDENT_CODE=ON) set(GOOGLE_CLOUD_CPP_ENABLE experimental-bigquery_rest oauth2 bigquery resourcemanager serviceusage) From ff656b779b5b2cbb79cc573f577ffd99c2faabd5 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 10:55:43 +0000 Subject: [PATCH 33/33] test --- .../odbc/bq_driver/internal/odbc_sql_execute_utils.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc index e304737658..d3044f9b76 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc @@ -23,6 +23,14 @@ #endif #include +#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && \ + defined(__GLIBCXX__) +template bool std::operator==(const std::shared_ptr&, + std::nullptr_t) noexcept; +#endif // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && + // defined(__GLIBCXX__) + + ////////////////////////////////////////////////////////////////// // This file has query execution related utilities which can have // statement or descriptor handles as arguments. We have some utils