From 9f26d59d6172500b770d50103619eeeda53db0ec Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 12:19:41 +0000 Subject: [PATCH 1/2] impl(bq_driver): Adding performance pipeline for linux --- .../integration-production-bq-driver-dm.sh | 36 ++ ci/cloudbuild/builds/lib/benchmark_results.py | 417 ++++++++++++++++++ .../builds/linux-bq-driver-benchmark.sh | 358 +++++++++++++++ .../linux-bq-driver-benchmark-ci.yaml | 29 ++ .../internal/odbc_sql_execute_utils.cc | 7 + 5 files changed, 847 insertions(+) create mode 100644 ci/cloudbuild/builds/lib/benchmark_results.py create mode 100755 ci/cloudbuild/builds/linux-bq-driver-benchmark.sh create mode 100644 ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..91d98cfd76 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -73,6 +73,7 @@ io::run cmake -B "$BUILD_DIR" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ -DODBC_DEMO_TESTING=ON \ @@ -81,6 +82,41 @@ io::run cmake -B "$BUILD_DIR" \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then + DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + + if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 + fi + + echo "Google driver found:" + ls -lh "$DRIVER_SO" + + # Sanitize branch name for use in GCS path. + SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' + ) + + PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + + echo "Uploading Google driver artifact..." + echo "Branch: ${SANITIZED_BRANCH}" + + gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + + echo "Google driver benchmark artifact uploaded:" + echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +fi + # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..a26391a8f7 --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,417 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import re +import statistics +from pathlib import Path + +TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") + +FAILED_RE = re.compile( + r"\[\s*FAILED\s*\]\s+(.+?)(?:\s+\([\d.]+\s*(?:ns|us|ms|s)\))?\s*$" +) + + +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + name = name.strip().rstrip(",") + + # Ignore GTest summary lines such as: + # + # 19 tests, listed below: + # + if re.match(r"^\d+\s+tests?,\s+listed below:$", name): + return None + + # Remove GTest parameter description. + # + # Example: + # + # Benchmark/all_bq_types_2, where GetParam() = + # ("all_bq_types_2", "SELECT * FROM ...") + # + # becomes: + # + # Benchmark/all_bq_types_2 + # + name = re.sub( + r",\s*where\s+GetParam\(\)\s*=.*$", + "", + name, + ) + + # Remove everything before the first '.'. + # + # Example: + # + # Instantiation/TestSuite.TestCase/0 + # + # becomes: + # + # TestCase/0 + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy HTAPI suffixes. + name = re.sub( + r"/(?:With|Without)HTAPI$", + "", + name, + ) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + + if unit == "us": + return value / 1000.0 + + if unit == "ns": + return value / 1_000_000.0 + + return value + + +def format_ms(value): + """Format milliseconds for the benchmark table.""" + if value is None: + return "N/A" + + return f"{value:.0f}ms" + + +def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + A benchmark is run multiple times. + + Rules: + 1. Test passes in every iteration: + -> use median execution time. + + 2. Test fails in ANY iteration: + -> return None, displayed as N/A. + + 3. Test is completely missing: + -> return None, displayed as N/A. + """ + path = Path(path) + + if not path.exists(): + raise FileNotFoundError(f"Benchmark output not found: {path}") + + samples = {} + failed_tests = set() + all_tests = set() + + for line in path.read_text(errors="replace").splitlines(): + + # --------------------------------------------------------------- + # Successful test + # --------------------------------------------------------------- + + time_match = TIME_RE.search(line) + + if time_match: + test_name = clean_test_name(time_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + value = float(time_match.group(2)) + unit = time_match.group(3).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + test_name, + [], + ).append(value_ms) + + all_tests.add(test_name) + + continue + + # --------------------------------------------------------------- + # Failed test + # --------------------------------------------------------------- + + failed_match = FAILED_RE.search(line) + + if failed_match: + test_name = clean_test_name(failed_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + failed_tests.add(test_name) + all_tests.add(test_name) + + # ------------------------------------------------------------------- + # Build final result. + # ------------------------------------------------------------------- + + results = {} + + for test_name in all_tests: + + # If a test failed even once, report N/A. + if test_name in failed_tests: + results[test_name] = None + continue + + values = samples.get( + test_name, + [], + ) + + if not values: + results[test_name] = None + continue + + # Same behavior as the GitHub Actions implementation: + # median of all successful iterations. + results[test_name] = statistics.median(values) + + return results + + +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + + Negative = faster/improvement. + Positive = slower/degradation. + """ + if value_ms is None or reference_ms is None or reference_ms == 0: + return " (N/A)" + + pct = round(((value_ms - reference_ms) / reference_ms) * 100) + + if pct > 0: + return f" (+{pct}%)" + + if pct < 0: + return f" ({pct}%)" + + return " (0%)" + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--existing", + required=True, + help="Existing driver benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + + args = parser.parse_args() + + # ------------------------------------------------------------------- + # Parse benchmark outputs. + # ------------------------------------------------------------------- + + existing_data = parse_gtest_output(args.existing) + + current_data = parse_gtest_output(args.current) + + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found. " + "Existing Driver will be shown as N/A." + ) + + # ------------------------------------------------------------------- + # Union of all test names. + # ------------------------------------------------------------------- + + all_tests = ( + set(existing_data.keys()) | set(current_data.keys()) | set(main_data.keys()) + ) + + sorted_tests = sorted(all_tests) + + rows = [] + + for test_name in sorted_tests: + + existing_ms = existing_data.get(test_name) + + current_ms = current_data.get(test_name) + + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + + current_raw = format_ms(current_ms) + + main_raw = format_ms(main_ms) + + # --------------------------------------------------------------- + # Current vs Existing + # --------------------------------------------------------------- + + current_pct = "" + + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + # --------------------------------------------------------------- + # Main vs Current + # --------------------------------------------------------------- + + main_pct = "" + + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + + main_value = f"{main_raw}{main_pct}" + + rows.append( + ( + test_name, + existing_raw, + current_value, + main_value, + ) + ) + + # ------------------------------------------------------------------- + # Generate Markdown table. + # + # This intentionally follows the existing GHA table format. + # ------------------------------------------------------------------- + + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" + + w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) + + w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) + + w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) + + w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) + + table = ( + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*" + "\n\n" + ) + + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) + + table += ( + "|-" + + ("-" * w1) + + "-|-" + + ("-" * w2) + + "-|-" + + ("-" * w3) + + "-|-" + + ("-" * w4) + + "-|\n" + ) + + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" + ) + + # ------------------------------------------------------------------- + # Write output. + # ------------------------------------------------------------------- + + output_path = Path(args.output) + + output_path.write_text(table) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh new file mode 100755 index 0000000000..66031ffced --- /dev/null +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +# ============================================================================ +# Configuration +# ============================================================================ + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" + +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +# ============================================================================ +# Branch +# ============================================================================ + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +)" + +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo + +# ============================================================================ +# Result files +# ============================================================================ + +CURRENT_RESULT="${RESULTS_DIR}/current.txt" +MAIN_RESULT="${RESULTS_DIR}/main.txt" +EXISTING_RESULT="${RESULTS_DIR}/existing.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" + +# ============================================================================ +# Driver locations +# ============================================================================ + +DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" + +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" + +# ============================================================================ +# ODBC configuration +# ============================================================================ + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +cd "$WORKSPACE_DIR" + +# This is the name of DSN set in odbc.ini. +mapfile -t cmake_args < <(cmake::common_args) + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini +export ODBCINI="${GOOGLE_ODBCINI}" + +# ============================================================================ +# Validate ODBC configuration +# ============================================================================ + +if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 +fi + +if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Download Google Current driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google Current driver" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + +# --------------------------------------------------------------------------- +# Download Google Main driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" + +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi + +mkdir -p "$(dirname "${DRIVER_PATH}")" + +cp \ + /opt/odbc-driver/roots.pem \ + "${DRIVER_PATH%/*}/roots.pem" + +# ============================================================================ +# Run benchmark +# ============================================================================ + +run_benchmark() { + local name="$1" + local output_file="$2" + local dsn="$3" + local bq_tests_flag="$4" + + echo + echo "Reconfiguring performance_test for ${name}" + echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" + echo "------------------------------------------------------------" + + io::run cmake -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" + + io::run cmake --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)" + + PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + fi + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true + + return 1 + fi + + echo "performance_test:" + echo " ${PERFORMANCE_TEST}" + + : >"${output_file}" + + local failed_iterations=0 + + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo + echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" + + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >>"${output_file}" + + echo "=== ODBC tests DSN is====${ODBC_TESTS_DSN}===" + echo "=== ODBCINI is====${dsn}===" + + set +e + + ODBCINI="${dsn}" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "${PERFORMANCE_TEST}" \ + >>"${output_file}" 2>&1 + + run_exit=$? + + set -e + + if [[ "${run_exit}" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + echo "WARNING: Continuing with remaining iterations." + + echo "============================================================" + echo "Detailed failure for ${name}, iteration ${i}:" + echo "============================================================" + + grep -E -B 20 -A 20 \ + '\[ *FAILED *\]|Failure|FAILED|ERROR|Error|error|SQLSTATE|Diagnostic|diagnostic|SQL_ERROR|SQLConnect|SQLDriverConnect|Exception|exception' \ + "${output_file}" || true + + echo "============================================================" + + failed_iterations=$((failed_iterations + 1)) + fi + done + + echo + echo "${name} benchmark completed." + echo "Failed iterations: ${failed_iterations}/${BENCHMARK_ITERATIONS}" + echo "Raw result: ${output_file}" + + # Benchmark failures must not fail Cloud Build. + return 0 +} + +# ============================================================================ +# Google Current +# +# The existing Google DSN points to DRIVER_PATH. +# Replace the driver binary before running the benchmark. +# ============================================================================ + +echo +echo "Preparing Google Current" +echo "------------------------------------------------------------" + +cp "${CURRENT_SO}" "${DRIVER_PATH}" +ls -lh "${DRIVER_PATH}" + +run_benchmark \ + "Google Current" \ + "${CURRENT_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" + +# ============================================================================ +# Google Main +# ============================================================================ + +echo +echo "Preparing Google Main" +echo "------------------------------------------------------------" + +if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + cp "${MAIN_SO}" "${DRIVER_PATH}" + ls -lh "${DRIVER_PATH}" + + run_benchmark \ + "Google Main" \ + "${MAIN_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" +else + echo "Google Main benchmark skipped: main driver artifact unavailable." + : >"$MAIN_RESULT" +fi + +# ============================================================================ +# Existing +# ============================================================================ + +echo +echo "Preparing Existing Driver" +echo "------------------------------------------------------------" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBCINI="${EXISTING_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + +run_benchmark \ + "Existing" \ + "${EXISTING_RESULT}" \ + "${EXISTING_ODBCINI}" \ + "OFF" + +# ============================================================================ +# Generate comparison +# ============================================================================ + +echo +echo "============================================================" +echo "Generating benchmark comparison" +echo "============================================================" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --output "${SUMMARY_RESULT}" + +# ============================================================================ +# Upload results +# ============================================================================ + +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" + +echo +echo "Uploading benchmark results" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_RESULT}" \ + "${MAIN_RESULT}" \ + "${EXISTING_RESULT}" \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/" + +echo +echo "============================================================" +echo "Benchmark completed successfully" +echo "============================================================" diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..089990fb8a --- /dev/null +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + branch: ^main$ +name: linux-bq-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-bq-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc index fd3231ef9e..e5846848a8 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc @@ -23,6 +23,13 @@ #endif #include +#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && \ + defined(__GLIBCXX__) +template bool std::operator==(std::shared_ptr const&, + std::nullptr_t) noexcept; +#endif // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && + // defined(__GLIBCXX__) + ////////////////////////////////////////////////////////////////// // This file has query execution related utilities which can have // statement or descriptor handles as arguments. We have some utils From 06d5d7a26c818b7b441f365a7f4130ef1341276a Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 19:31:16 +0000 Subject: [PATCH 2/2] made changes after review comments --- ci/cloudbuild/builds/lib/benchmark_results.py | 67 ++------------ .../builds/linux-bq-driver-benchmark.sh | 89 ++++++++----------- .../linux-bq-driver-benchmark-ci.yaml | 2 +- 3 files changed, 47 insertions(+), 111 deletions(-) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index a26391a8f7..ab9caeaa56 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -19,10 +19,6 @@ TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") -FAILED_RE = re.compile( - r"\[\s*FAILED\s*\]\s+(.+?)(?:\s+\([\d.]+\s*(?:ns|us|ms|s)\))?\s*$" -) - def clean_test_name(name): """Normalize GTest test names for comparison.""" @@ -103,14 +99,16 @@ def parse_gtest_output(path): A benchmark is run multiple times. Rules: - 1. Test passes in every iteration: - -> use median execution time. - 2. Test fails in ANY iteration: - -> return None, displayed as N/A. + 1. Only successful iterations are used. + + 2. Failed iterations are ignored. + + 3. The median execution time is calculated from all successful + iterations. - 3. Test is completely missing: - -> return None, displayed as N/A. + 4. If all iterations fail or a test is completely missing, + it is displayed as N/A. """ path = Path(path) @@ -118,11 +116,8 @@ def parse_gtest_output(path): raise FileNotFoundError(f"Benchmark output not found: {path}") samples = {} - failed_tests = set() - all_tests = set() for line in path.read_text(errors="replace").splitlines(): - # --------------------------------------------------------------- # Successful test # --------------------------------------------------------------- @@ -149,50 +144,17 @@ def parse_gtest_output(path): [], ).append(value_ms) - all_tests.add(test_name) - - continue - - # --------------------------------------------------------------- - # Failed test - # --------------------------------------------------------------- - - failed_match = FAILED_RE.search(line) - - if failed_match: - test_name = clean_test_name(failed_match.group(1)) - - # Ignore GTest summary/non-test lines. - if test_name is None: - continue - - failed_tests.add(test_name) - all_tests.add(test_name) - # ------------------------------------------------------------------- # Build final result. # ------------------------------------------------------------------- results = {} - for test_name in all_tests: - - # If a test failed even once, report N/A. - if test_name in failed_tests: - results[test_name] = None - continue - - values = samples.get( - test_name, - [], - ) - + for test_name, values in samples.items(): if not values: results[test_name] = None continue - # Same behavior as the GitHub Actions implementation: - # median of all successful iterations. results[test_name] = statistics.median(values) return results @@ -261,9 +223,7 @@ def main(): # ------------------------------------------------------------------- existing_data = parse_gtest_output(args.existing) - current_data = parse_gtest_output(args.current) - main_data = parse_gtest_output(args.main) if not current_data: @@ -299,15 +259,11 @@ def main(): for test_name in sorted_tests: existing_ms = existing_data.get(test_name) - current_ms = current_data.get(test_name) - main_ms = main_data.get(test_name) existing_raw = format_ms(existing_ms) - current_raw = format_ms(current_ms) - main_raw = format_ms(main_ms) # --------------------------------------------------------------- @@ -335,7 +291,6 @@ def main(): ) current_value = f"{current_raw}{current_pct}" - main_value = f"{main_raw}{main_pct}" rows.append( @@ -359,11 +314,8 @@ def main(): h4 = "Google Driver (Main)" w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) - w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) - w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) - w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) table = ( @@ -407,7 +359,6 @@ def main(): # ------------------------------------------------------------------- output_path = Path(args.output) - output_path.write_text(table) return 0 diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 66031ffced..6f32b0bd33 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -30,7 +30,6 @@ WORKSPACE_DIR=$(pwd) # ============================================================================ BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" - PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" BUILD_DIR="${WORKSPACE_DIR}/cmake-out" @@ -39,25 +38,12 @@ RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" rm -rf "${RESULTS_DIR}" mkdir -p "${RESULTS_DIR}" -# ============================================================================ -# Branch -# ============================================================================ - BRANCH_NAME="${BRANCH_NAME:-main}" SANITIZED_BRANCH="$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9._-]/_/g' )" -echo "============================================================" -echo "Linux ODBC Performance Benchmark" -echo "============================================================" -echo "Branch : ${BRANCH_NAME}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Workspace : ${WORKSPACE_DIR}" -echo - # ============================================================================ # Result files # ============================================================================ @@ -71,7 +57,7 @@ SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" # Driver locations # ============================================================================ -DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" @@ -86,20 +72,13 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -cd "$WORKSPACE_DIR" - -# This is the name of DSN set in odbc.ini. -mapfile -t cmake_args < <(cmake::common_args) - -GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" -EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" - export ODBC_TESTS_DSN="SampleDSNGoogleDriver" - -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} -export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" +export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" export ODBCINI="${GOOGLE_ODBCINI}" +mapfile -t cmake_args < <(cmake::common_args) + # ============================================================================ # Validate ODBC configuration # ============================================================================ @@ -116,11 +95,22 @@ if [[ ! -f "${EXISTING_ODBCINI}" ]]; then exit 1 fi -# --------------------------------------------------------------------------- -# Download Google Current driver -# --------------------------------------------------------------------------- +# ============================================================================ +# Header +# ============================================================================ +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" echo + +# ============================================================================ +# Download drivers +# ============================================================================ + echo "Downloading Google Current driver" echo "------------------------------------------------------------" @@ -128,15 +118,11 @@ gcloud storage cp \ "${CURRENT_SO_GCS}" \ "${CURRENT_SO}" -# --------------------------------------------------------------------------- -# Download Google Main driver -# --------------------------------------------------------------------------- - echo echo "Downloading Google driver from main" echo "------------------------------------------------------------" -if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then +if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then echo "Main Google driver downloaded successfully." HAS_MAIN_DRIVER=true else @@ -166,23 +152,25 @@ run_benchmark() { echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" echo "------------------------------------------------------------" - io::run cmake -S "${WORKSPACE_DIR}" \ + io::run cmake \ + -S "${WORKSPACE_DIR}" \ -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_PERFORMANCE_TEST_ONLY=ON \ -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" - io::run cmake --build "${BUILD_DIR}" \ + io::run cmake \ + --build "${BUILD_DIR}" \ --target performance_test \ --parallel "$(nproc)" - PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + local performance_test="${BUILD_DIR}/integration_tests/performance_test" - if [[ ! -x "${PERFORMANCE_TEST}" ]]; then - PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + if [[ ! -x "${performance_test}" ]]; then + performance_test="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" fi - if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + if [[ ! -x "${performance_test}" ]]; then echo "ERROR: performance_test was not found." find "${BUILD_DIR}" \ @@ -194,13 +182,13 @@ run_benchmark() { fi echo "performance_test:" - echo " ${PERFORMANCE_TEST}" + echo " ${performance_test}" : >"${output_file}" local failed_iterations=0 - for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" @@ -214,10 +202,10 @@ run_benchmark() { ODBCINI="${dsn}" \ ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ - "${PERFORMANCE_TEST}" \ + "${performance_test}" \ >>"${output_file}" 2>&1 - run_exit=$? + local run_exit=$? set -e @@ -235,7 +223,7 @@ run_benchmark() { echo "============================================================" - failed_iterations=$((failed_iterations + 1)) + ((failed_iterations += 1)) fi done @@ -250,9 +238,6 @@ run_benchmark() { # ============================================================================ # Google Current -# -# The existing Google DSN points to DRIVER_PATH. -# Replace the driver binary before running the benchmark. # ============================================================================ echo @@ -276,7 +261,7 @@ echo echo "Preparing Google Main" echo "------------------------------------------------------------" -if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then +if [[ "${HAS_MAIN_DRIVER}" == "true" ]]; then cp "${MAIN_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" @@ -287,11 +272,11 @@ if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then "ON" else echo "Google Main benchmark skipped: main driver artifact unavailable." - : >"$MAIN_RESULT" + : >"${MAIN_RESULT}" fi # ============================================================================ -# Existing +# Existing Driver # ============================================================================ echo diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index 089990fb8a..bb6d04d0d5 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -1,4 +1,4 @@ -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.