From 8ada537612551bca8604539336113dfe6929e71d Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Mon, 27 Apr 2026 12:51:32 +0530 Subject: [PATCH 01/32] fix encoding --- google/cloud/odbc/bq_driver/internal/utils.cc | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index b9fb984502..8e532a7025 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -20,6 +20,7 @@ #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" #include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/internal/getenv.h" +#include #include #include #include @@ -825,6 +826,47 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( if (((in_str != nullptr) && (in_str[0] == '\0'))) { return std::string(); } + +#if !defined(_WIN32) + // The driver may be compiled against iODBC headers, where SQLWCHAR is + // wchar_t (4 bytes UCS-4LE on Linux/macOS), but loaded by unixODBC, which + // by default delivers 2-byte UTF-16LE wide chars. SAP HANA on Linux uses + // unixODBC, so without `IconvEncoding=UCS-4LE` in odbcinst.ini we receive + // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. Detect by inspecting the + // first two code units: ODBC connection strings and identifiers always + // start with ASCII (DSN=, DRIVER=, SELECT, etc.), so a UCS-4LE ASCII char + // looks like `XX 00 00 00` while UTF-16LE looks like `XX 00 YY 00`. + if (sizeof(SQLWCHAR) == 4) { + auto const* bytes = reinterpret_cast(in_str); + LOG(INFO) << "BqConvertSQLWCHARToString:: sizeof(SQLWCHAR)=4; " + << "first 4 bytes = " + << absl::StrFormat("%02X %02X %02X %02X", bytes[0], bytes[1], + bytes[2], bytes[3]) + << "; in_str_len=" << in_str_len; + if (bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && + bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { + auto const* utf16 = reinterpret_cast(in_str); + SQLINTEGER count = in_str_len; + if (count == SQL_NTS || count == NULL) { + count = 0; + while (utf16[count] != 0) ++count; + } + LOG(INFO) << "BqConvertSQLWCHARToString:: detected UTF-16LE wire format " + "(unixODBC under iODBC-built driver); reinterpreting as " + "uint16_t* with code-unit count=" + << count; + std::wstring wstr; + wstr.reserve(count); + for (SQLINTEGER i = 0; i < count; ++i) { + wstr.push_back(static_cast(utf16[i])); + } + return Utf16ToUtf8(wstr); + } + LOG(INFO) << "BqConvertSQLWCHARToString:: byte pattern does not match " + "UTF-16LE; falling through to default 4-byte SQLWCHAR path"; + } +#endif + if (in_str_len == SQL_NTS || in_str_len == NULL) { in_str_len = static_cast(std::char_traits::length(in_str)); From 302033d6f1a53e621766b96ea814ca9ec57b0e9a Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 28 Apr 2026 16:43:46 +0530 Subject: [PATCH 02/32] fix the encodings all places --- .../bq_driver/internal/data_translation.cc | 118 +++++++---------- .../internal/data_translation_inv.cc | 2 +- .../odbc/bq_driver/internal/odbc_desc_attr.cc | 3 +- .../bq_driver/internal/odbc_type_utils.cc | 21 +-- .../odbc/bq_driver/internal/odbc_type_utils.h | 28 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 83 +++++++++--- google/cloud/odbc/bq_driver/internal/utils.h | 29 ++++ google/cloud/odbc/bq_driver/odbc_api.cc | 124 +++++++++--------- .../odbc/bq_driver/odbc_driver_metadata.cc | 2 +- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 13 +- 10 files changed, 253 insertions(+), 170 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index eace909d6a..c330015380 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -104,7 +104,7 @@ odbc_internal::StatusRecord ConvertFromNumericDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; WStrToOutputBufferResponse(wstr.GetValue(), dest_data.buf, wchar_capacity, @@ -326,7 +326,7 @@ odbc_internal::StatusRecord ConvertFromStringDSValue(DSValue const& src_dsval, } auto src_len = static_cast(wide_str.length()); - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_str, dest_data.buf, wchar_capacity, src_len, required_chars, @@ -907,7 +907,7 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN required_chars = static_cast(wstr->length()) + 1; return WStrToOutputBufferResponse( wstr.GetValue(), dest_buf, wchar_capacity, k_time_src_len, @@ -1007,30 +1007,25 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - std::wstring wstr_val = wstr.GetValue(); - if (!wstr_val.empty() && wstr_val.back() == L'\0') { - wstr_val.pop_back(); - } - std::vector wstr_data(wstr_val.begin(), wstr_val.end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = k_timestamp_src_len * sizeof(SQLWCHAR); + *res_len = k_timestamp_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_timestamp_src_len) * sizeof(SQLWCHAR)); - dest[k_timestamp_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, k_timestamp_src_len); + write_terminator(k_timestamp_src_len); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); + write_terminator(wchar_capacity - 1); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1189,29 +1184,24 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - std::wstring wstr_val = wstr.GetValue(); - if (!wstr_val.empty() && wstr_val.back() == L'\0') { - wstr_val.pop_back(); - } - std::vector wstr_data(wstr_val.begin(), wstr_val.end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = k_datetime_src_len * sizeof(SQLWCHAR); + *res_len = k_datetime_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_datetime_src_len) * sizeof(SQLWCHAR)); - dest[k_datetime_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, k_datetime_src_len); + write_terminator(k_datetime_src_len); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); + write_terminator(wchar_capacity - 1); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1398,7 +1388,7 @@ odbc_internal::StatusRecord ConvertFromDateDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "DSValueToWchar Conversion Failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1434,7 +1424,7 @@ StatusRecord ConvertStringToJsonOutputBuffer(std::string const& src_str, return StatusRecord{SQLStates::k_HY000(), "Conversion to UTF-16 failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_string.GetValue(), dest_buf, @@ -1498,15 +1488,11 @@ StatusRecord ConvertFromArrayDSValue(DSValue const& src_dsval, if (!wide_string.Ok()) { return StatusRecord{SQLStates::k_HY000(), "Conversion Failed"}; } - std::wstring wide_val = wide_string.GetValue(); - if (!wide_val.empty() && wide_val.back() == L'\0') { - wide_val.pop_back(); - } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); - auto src_len = static_cast(wide_val.length()); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); + auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( - wide_val, dest_data.buf, wchar_capacity, src_len, required_chars, + *wide_string, dest_data.buf, wchar_capacity, src_len, required_chars, reinterpret_cast(dest_data.result_len)); } case SQL_C_BINARY: { @@ -1621,7 +1607,7 @@ odbc_internal::StatusRecord ConvertFromIntervalDSValue(DSValue const& src_dsval, StatusRecord{SQLStates::k_HY000(), wstr.GetStatusRecord().message}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto interval_char_length = static_cast(wstr.GetValue().length()); return WStrIntervalBufferResponse( @@ -1907,7 +1893,7 @@ StatusRecord ConvertFromGeographyDSValue(DSValue const& src_dsval, } std::memset(dest_data.buf, 0, buffer_length); std::wstring const& wide_str = wstr.GetValue(); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wide_str.length()); SQLLEN required_chars = src_len + 1; status_record = WStrToOutputBufferResponse( @@ -2054,20 +2040,20 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, "UTF-8 to UTF-16 conversion failed."}; } - std::wstring utf16_value = utf16_str.GetValue(); - if (!utf16_value.empty() && utf16_value.back() == L'\0') { - utf16_value.pop_back(); - } - size_t const required_size = utf16_value.length() * sizeof(SQLWCHAR); + std::wstring const& utf16_value = utf16_str.GetValue(); + size_t const required_size = utf16_value.length() * WireWcharSize(); - auto* buffer = reinterpret_cast(dest_data.buf); + auto write_terminator = [&](size_t char_index) { + auto* p = static_cast(dest_data.buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; // Handle truncation if buffer is insufficient - if (dest_data.buflen < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / sizeof(SQLWCHAR)) - 1; - std::memcpy(buffer, utf16_value.data(), - num_chars_to_copy * sizeof(SQLWCHAR)); - buffer[num_chars_to_copy] = L'\0'; + if (static_cast(dest_data.buflen) < required_size) { + size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; + WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy); + write_terminator(num_chars_to_copy); if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; @@ -2075,17 +2061,15 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { - buffer[i] = static_cast(utf16_str.GetValue()[i]); - } - size_t buffer_chars = dest_data.buflen / sizeof(SQLWCHAR); - if (utf16_str.GetValue().size() < buffer_chars) { - buffer[utf16_str.GetValue().size()] = L'\0'; + WriteWideToWireBuffer(utf16_value, dest_data.buf, utf16_value.size()); + size_t buffer_chars = dest_data.buflen / WireWcharSize(); + if (utf16_value.size() < buffer_chars) { + write_terminator(utf16_value.size()); } // Set output length if (dest_data.result_len) { - *dest_data.result_len = utf16_str.GetValue().size() * sizeof(SQLWCHAR); + *dest_data.result_len = utf16_value.size() * WireWcharSize(); } return status_record; } @@ -2253,7 +2237,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "Conversion to SQL_C_WCHAR failed."}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wstr->length()); SQLLEN required_chars = src_len + 1; return WStrToOutputBufferResponse( diff --git a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc index 76832234b3..c20ae95e00 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc @@ -53,7 +53,7 @@ StatusRecordOr ConvertFromCharBuffer(DataBuffer& src_data, auto* wchar_buf = static_cast(src_buf); if ((result_len > 0) || (result_len == SQL_NTS)) { if (result_len > 0) { - result_len /= sizeof(SQLWCHAR); + result_len /= WireWcharSize(); } auto utf8_res = BqConvertSQLWCHARToString( wchar_buf, static_cast(result_len)); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc index 8052921952..1a28dcb25d 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc @@ -14,6 +14,7 @@ #include "google/cloud/odbc/bq_driver/internal/odbc_desc_attr.h" #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/sql_state_constants.h" #include "google/cloud/odbc/internal/status_record_or.h" #include @@ -388,7 +389,7 @@ StatusRecord DescriptorRecord::SetOctetLength(SQLSMALLINT type, case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: - octet_length = value * sizeof(SQLWCHAR); + octet_length = value * WireWcharSize(); break; case SQL_DECIMAL: case SQL_NUMERIC: diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 993fa7374d..13d235d349 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,22 +45,25 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - std::vector wstr_data(wstr.begin(), wstr.end()); - wstr_data.emplace_back(L'\0'); + size_t const wire_sz = WireWcharSize(); + + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + (char_index * wire_sz); + std::memset(p, 0, wire_sz); + }; - auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = char_len * sizeof(SQLWCHAR); + *res_len = char_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (char_len) * sizeof(SQLWCHAR)); - dest[char_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, char_len); + write_terminator(char_len); } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, buffer_length); + write_terminator(buffer_length - 1); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 6630ef929f..c7515445d3 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -17,7 +17,9 @@ #include "google/cloud/odbc/internal/diagnostic_records.h" #include "google/cloud/odbc/internal/sql_state_constants.h" +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include +#include #include #include #include @@ -179,9 +181,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); + size_t const wire_sz = WireWcharSize(); + + // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset + // `byte_off` in dest_buf. + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + (char_index * wire_sz); + std::memset(p, 0, wire_sz); + }; + if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - reinterpret_cast(dest_buf)[0] = L'\0'; + write_terminator(0); } if (res_len) { *res_len = 0; @@ -189,21 +200,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - std::vector wstr_data(wstr.begin(), wstr.end()); - - auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { - *res_len = src_len * sizeof(SQLWCHAR); + *res_len = src_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (src_len) * sizeof(SQLWCHAR)); - dest[src_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, src_len); + write_terminator(src_len); } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, buffer_length); + write_terminator(buffer_length - 1); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 8e532a7025..84f8f15aaa 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -22,6 +22,7 @@ #include "google/cloud/internal/getenv.h" #include #include +#include #include #include #include @@ -40,6 +41,42 @@ bool g_suppress_dropdown = false; using ::google::cloud::odbc_internal::SQLStates; using ::google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; + +namespace { +// Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". +// Set inside BqConvertSQLWCHARToString once we observe the UTF-16LE byte +// pattern in an input buffer. The wire format never changes within a process, +// so once latched it stays latched. +std::atomic g_utf16le_wire_latched{false}; +} // namespace + +bool IsRuntimeWireUtf16Le() { +#if defined(_WIN32) + return false; +#else + if (sizeof(SQLWCHAR) != 4) return false; + return g_utf16le_wire_latched.load(std::memory_order_relaxed); +#endif +} + +size_t WireWcharSize() { + return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); +} + +void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { + if (count > src.size()) count = src.size(); + if (IsRuntimeWireUtf16Le()) { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); + } + } else { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); + } + } +} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; @@ -167,7 +204,7 @@ size_t BufferSizeForType(SQLSMALLINT type, size_t requested) { minimum_size = sizeof(SQL_TIMESTAMP_STRUCT); break; case SQL_C_WCHAR: - minimum_size = sizeof(SQLWCHAR); + minimum_size = WireWcharSize(); break; case SQL_C_SBIGINT: minimum_size = sizeof(SQLBIGINT); @@ -823,38 +860,50 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( if (in_str == nullptr) { return StatusRecord{SQLStates::k_HY000(), "in_str string is empty/Null"}; } - if (((in_str != nullptr) && (in_str[0] == '\0'))) { - return std::string(); - } #if !defined(_WIN32) // The driver may be compiled against iODBC headers, where SQLWCHAR is // wchar_t (4 bytes UCS-4LE on Linux/macOS), but loaded by unixODBC, which // by default delivers 2-byte UTF-16LE wide chars. SAP HANA on Linux uses // unixODBC, so without `IconvEncoding=UCS-4LE` in odbcinst.ini we receive - // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. Detect by inspecting the - // first two code units: ODBC connection strings and identifiers always - // start with ASCII (DSN=, DRIVER=, SELECT, etc.), so a UCS-4LE ASCII char - // looks like `XX 00 00 00` while UTF-16LE looks like `XX 00 YY 00`. + // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. + // + // Detection: a UCS-4LE ASCII char looks like `XX 00 00 00`, UTF-16LE + // looks like `XX 00 YY 00`. Single-char and empty strings are ambiguous + // so we *latch* the result the first time we get a clear signal — every + // later call (including the ambiguous ones from SQLTables(catalog="%")) + // honors the latch via IsRuntimeWireUtf16Le(). if (sizeof(SQLWCHAR) == 4) { + bool use_utf16le = + g_utf16le_wire_latched.load(std::memory_order_relaxed); + auto const* bytes = reinterpret_cast(in_str); LOG(INFO) << "BqConvertSQLWCHARToString:: sizeof(SQLWCHAR)=4; " << "first 4 bytes = " << absl::StrFormat("%02X %02X %02X %02X", bytes[0], bytes[1], bytes[2], bytes[3]) - << "; in_str_len=" << in_str_len; - if (bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && + << "; in_str_len=" << in_str_len + << "; latched=" << (use_utf16le ? "yes" : "no"); + + if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { + use_utf16le = true; + g_utf16le_wire_latched.store(true, std::memory_order_relaxed); + LOG(INFO) << "BqConvertSQLWCHARToString:: latched UTF-16LE wire format " + "for the rest of this process (unixODBC under iODBC-built " + "driver)"; + } + + if (use_utf16le) { auto const* utf16 = reinterpret_cast(in_str); + if (utf16[0] == 0) { + return std::string(); + } SQLINTEGER count = in_str_len; if (count == SQL_NTS || count == NULL) { count = 0; while (utf16[count] != 0) ++count; } - LOG(INFO) << "BqConvertSQLWCHARToString:: detected UTF-16LE wire format " - "(unixODBC under iODBC-built driver); reinterpreting as " - "uint16_t* with code-unit count=" - << count; std::wstring wstr; wstr.reserve(count); for (SQLINTEGER i = 0; i < count; ++i) { @@ -863,10 +912,14 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( return Utf16ToUtf8(wstr); } LOG(INFO) << "BqConvertSQLWCHARToString:: byte pattern does not match " - "UTF-16LE; falling through to default 4-byte SQLWCHAR path"; + "UTF-16LE and latch unset; falling through to default 4-byte " + "SQLWCHAR path"; } #endif + if (in_str[0] == '\0') { + return std::string(); + } if (in_str_len == SQL_NTS || in_str_len == NULL) { in_str_len = static_cast(std::char_traits::length(in_str)); diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index a557ce6725..26c123f597 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -226,6 +226,35 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( SQLWCHAR* in_str, SQLINTEGER in_str_len); +// True once the driver has detected that the loaded ODBC manager delivers +// wide-char buffers as 2-byte UTF-16LE even though the driver was compiled +// with sizeof(SQLWCHAR) == 4 (iODBC headers on Linux/macOS). This is the +// SAP HANA SDA case: HANA's bundled unixODBC speaks UTF-16LE, but the +// release-pipeline binary was built against iODBC. +// +// Latched (process-lifetime, atomic) inside BqConvertSQLWCHARToString the +// first time the wire-format heuristic fires. Always false on Windows and +// for builds where sizeof(SQLWCHAR) is already 2 — no adaptation needed. +bool IsRuntimeWireUtf16Le(); + +// Bytes per wide character on the wire between this driver and its loaded +// manager. Equals sizeof(SQLWCHAR) by default; equals 2 once the UTF-16LE +// wire format has been latched. Use this in *every* arithmetic expression +// that converts between byte counts and character counts on a buffer that +// crosses the driver/manager boundary — never use sizeof(SQLWCHAR) directly +// for that purpose. +size_t WireWcharSize(); + +// Copies up to `count` wide characters from `src` (a std::wstring whose +// element width is wchar_t — 4 bytes on Linux/macOS, 2 bytes on Windows) +// into `dest` using the wire format. On the UTF-16LE-on-wire path each +// wchar_t is narrowed to 16 bits — fine for ASCII / BMP content (which +// covers ODBC metadata strings); supplementary-plane characters would be +// truncated, but the same is true today for any code that does +// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement +// for that pattern. +void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count); + std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); bool IsDiagIdentifierString(SQLSMALLINT DiagIdentifier); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 92c4bf0cc1..710d122fbd 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -65,6 +65,8 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -280,7 +282,8 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString) { + if (SQL_SUCCEEDED(rc) && outConnectionString && + outConnectionStringBufferLen > 0) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -291,7 +294,13 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); + size_t buf_chars = static_cast(outConnectionStringBufferLen); + size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); + auto* term = static_cast(static_cast(outConnectionString)) + + (to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); + out_conn_str_len = static_cast(to_copy); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -393,10 +402,9 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, return utf16_out_conn_str.GetCalculatedReturnCode(); } std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * sizeof(SQLWCHAR)); - std::memcpy((SQLWCHAR*)outConnectionString, - ToSqlWChar(utf16_out_conn_str->data()), - utf16_out_conn_str->size() * sizeof(SQLWCHAR)); + outConnectionStringBufferLen * WireWcharSize()); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + utf16_out_conn_str->size()); } return rc; @@ -643,15 +651,19 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_info_val->begin(), - utf16_info_val->end()); - sql_w_str.emplace_back(L'\0'); std::size_t bytes_available = static_cast(infoValueBufferLen); - std::size_t bytes_to_copy = - std::min(sql_w_str.size() * sizeof(SQLWCHAR), bytes_available); - - std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); + // +1 to leave room for terminator within bytes_available, then + // shrink-to-fit at the wire char boundary. + std::size_t chars_available = bytes_available / WireWcharSize(); + std::size_t chars_to_copy = + std::min(utf16_info_val->size(), chars_available); + WriteWideToWireBuffer(*utf16_info_val, infoValue, chars_to_copy); + if (chars_to_copy < chars_available) { + auto* term = static_cast(infoValue) + + (chars_to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); + } } } else { if (info_val_buffer_len > 0) { @@ -662,7 +674,7 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, } } if (infoValueStringLen) - *infoValueStringLen = info_val_buffer_len * sizeof(SQLWCHAR); + *infoValueStringLen = info_val_buffer_len * WireWcharSize(); return rc; } @@ -806,7 +818,7 @@ SQLRETURN SQL_API SQLSetConnectAttrW(SQLHDBC connectionHandle, ConnectionValueType::kSqlChr) { if (valueStringLen && valueStringLen > 0) { updated_attrib_status = - ConvertSQLPointerToSQLChar(value, valueStringLen / sizeof(SQLWCHAR)); + ConvertSQLPointerToSQLChar(value, valueStringLen / WireWcharSize()); } else { updated_attrib_status = ConvertSQLPointerToSQLChar(value, valueStringLen); } @@ -912,14 +924,10 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - *valueStringLen = - wcslen(updated_out_attr_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_attr_status->c_str(), - updated_out_attr_status->c_str() + *valueStringLen); - sql_w_str.emplace_back(L'\0'); + size_t char_count = wcslen(updated_out_attr_status->data()); + *valueStringLen = char_count * WireWcharSize(); std::memset(value, '\0', valueBufferLen); - std::memcpy(value, sql_w_str.data(), sql_w_str.size()); + WriteWideToWireBuffer(*updated_out_attr_status, value, char_count); } return rc; @@ -1151,13 +1159,10 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - out_desc_val_string_len = - wcslen(utf16_out_desc_val->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str(utf16_out_desc_val->begin(), - utf16_out_desc_val->end()); - sql_w_str.emplace_back(L'\0'); + size_t char_count = utf16_out_desc_val->size(); + out_desc_val_string_len = char_count * WireWcharSize(); std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); + WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, char_count); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1238,9 +1243,9 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - std::memset(name, '\0', nameBufferLen * sizeof(SQLWCHAR)); + std::memset(name, '\0', nameBufferLen * WireWcharSize()); std::memcpy(name, ToSqlWChar(utf16_name->data()), - name_string_len * sizeof(SQLWCHAR)); + name_string_len * WireWcharSize()); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1526,11 +1531,11 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_cur_name->begin(), - utf16_cur_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memcpy(cursorName, sql_w_str.data(), - (sql_w_str.size() + 1) * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_cur_name, cursorName, + utf16_cur_name->size()); + auto* term = static_cast(static_cast(cursorName)) + + (utf16_cur_name->size() * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2103,14 +2108,16 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const bytes_to_copy = - std::min(static_cast(characterAttributeBufferLen), - wstr.size() * sizeof(SQLWCHAR)); - - std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); - if (characterAttributeBufferLen >= sizeof(SQLWCHAR)) { - SQLWCHAR* wchar_buf = static_cast(characterAttribute); - wchar_buf[bytes_to_copy / sizeof(SQLWCHAR)] = 0; + size_t const buf_chars = + static_cast(characterAttributeBufferLen) / WireWcharSize(); + size_t const chars_to_copy = std::min(wstr.size(), buf_chars); + + WriteWideToWireBuffer(wstr, characterAttribute, chars_to_copy); + if (characterAttributeBufferLen >= + static_cast(WireWcharSize())) { + auto* term = static_cast(characterAttribute) + + (chars_to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } character_attribute_string_len = static_cast(wstr.size()); @@ -2123,7 +2130,7 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, *characterAttributeStringLen = character_attribute_string_len; #ifdef WIN32 *characterAttributeStringLen = - character_attribute_string_len * sizeof(SQLWCHAR); + character_attribute_string_len * WireWcharSize(); #endif // WIN32 } @@ -2285,12 +2292,9 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_col_name->begin(), - utf16_col_name->end()); - sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - std::memcpy(columnName, sql_w_str.data(), - column_name_string_len * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_col_name, columnName, + static_cast(column_name_string_len)); } if (columnNameLen) { @@ -2495,13 +2499,13 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - diag_info_str_len = - wcslen(updated_out_diag_info_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_diag_info_status->c_str(), - updated_out_diag_info_status->c_str() + diag_info_str_len); - sql_w_str.emplace_back(L'\0'); - std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); + size_t char_count = updated_out_diag_info_status->size(); + diag_info_str_len = char_count * WireWcharSize(); + WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, + char_count); + auto* term = static_cast(diagInfo) + + (char_count * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2587,8 +2591,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), - utf16_sql_state->size() * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_sql_state, sqlState, + utf16_sql_state->size()); } if (messageText && message_text_buffer_len > 0) { @@ -2598,8 +2602,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), - utf16_msg_txt->size() * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_msg_txt, messageText, + utf16_msg_txt->size()); } if (messageTextLen) *messageTextLen = message_text_buffer_len; diff --git a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc index 5d17ce2085..566c948600 100644 --- a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc +++ b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc @@ -529,7 +529,7 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, LOG(ERROR) << "SQLTables::PopulateIrd:: " << ird_status.message; return LogAndReturnCode(handle, ird_status); } - + LOG(INFO) << "SQLTablesInternal:: end"; handle.SetResultSet(result_set); handle.SetStmtState(StmtStates::kStatementExecutedWithRs); return SQL_SUCCESS; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index c9f1bd4e4f..57bc814b9e 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -29,6 +29,7 @@ namespace google::cloud::odbc_bq_driver { using google::cloud::odbc_bq_driver_internal::BQDataType; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::CheckTargetType; using google::cloud::odbc_bq_driver_internal::ConnectionHandle; using google::cloud::odbc_bq_driver_internal::CreateDSRowFromTypeInfo; @@ -776,7 +777,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, // 3. If the data fits or is not a variable-length type, return it directly in // the caller’s buffer. SQLLEN target_buff_len = (target_c_type == SQL_C_WCHAR) - ? (target_value_buffer_len / sizeof(SQLWCHAR)) + ? (target_value_buffer_len / WireWcharSize()) : target_value_buffer_len; if (offset == 0) { if ((ds_val.size() > target_buff_len) && @@ -789,7 +790,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, size_t buffer_size = 0; if (target_c_type == SQL_C_WCHAR) { - buffer_size = (ds_val.size() + 1) * sizeof(SQLWCHAR); + buffer_size = (ds_val.size() + 1) * WireWcharSize(); } else { buffer_size = ds_val.size() + 1; } @@ -845,18 +846,18 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, result_set.translated_data.row_offset = offset + target_value_buffer_len; } else if (target_c_type == SQL_C_WCHAR) { auto data_size = result_set.translated_data.data.size(); - auto max_buff_chars = target_value_buffer_len / sizeof(SQLWCHAR); - auto offset_chars = offset / sizeof(SQLWCHAR); + auto max_buff_chars = target_value_buffer_len / WireWcharSize(); + auto offset_chars = offset / WireWcharSize(); auto remain_chars = (data_size > offset_chars) ? (data_size - offset_chars) : 0; auto copy_chars = (remain_chars >= max_buff_chars) ? (max_buff_chars - 1) : remain_chars; std::memcpy(target_value, result_set.translated_data.data.data() + offset, - copy_chars * sizeof(SQLWCHAR)); + copy_chars * WireWcharSize()); reinterpret_cast(target_value)[copy_chars] = 0; result_set.translated_data.row_offset = - offset + (copy_chars * sizeof(SQLWCHAR)); + offset + (copy_chars * WireWcharSize()); } else { std::memcpy(target_value, result_set.translated_data.data.data() + offset, target_value_buffer_len - 1); From a9c8a7bf5999e251667dcf2c17b377a0ab8a19c2 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Thu, 30 Apr 2026 12:30:35 +0530 Subject: [PATCH 03/32] upload on git --- ci/cloudbuild/builds/bq-driver-release.sh | 6 +- .../integration-production-bq-driver-dm.sh | 19 ++++- .../ubuntu-22.04-install.Dockerfile | 70 ++++++++----------- .../bq_driver/internal/odbc_sql_tables.cc | 19 ++++- .../odbc/bq_driver/odbc_driver_metadata.cc | 20 ++++++ 5 files changed, 88 insertions(+), 46 deletions(-) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index a18385d934..9316235856 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -81,9 +81,9 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" +# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +# mapfile -t ctest_args < <(ctest::common_args) +# io::run env -C cmake-out ctest "${ctest_args[@]}" io::log_h1 "Packaging and Uploading Driver" diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..abb968c31a 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -82,6 +82,19 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" +# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +# mapfile -t ctest_args < <(ctest::common_args) +# io::run env -C cmake-out ctest "${ctest_args[@]}" + +SO_FILE_PATH="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +GCS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc" + +if [[ -f "$SO_FILE_PATH" ]]; then + echo "Uploading $SO_FILE_PATH to $GCS_BUCKET" + gsutil cp "$SO_FILE_PATH" "$GCS_BUCKET" +else + echo "Error: $SO_FILE_PATH not found. Upload skipped." + echo "Listing contents of cmake-out directory for debugging:" + ls -R cmake-out + exit 1 +fi \ No newline at end of file diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 33e5f46c3f..6007e7512a 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:22.04 +FROM ubuntu:18.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ + apt-get --no-install-recommends install -y \ + software-properties-common gnupg2 && \ + add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ # Dependency for arrow bison \ - clang-12 \ - lld-12 \ cmake \ curl \ # Dependency for arrow @@ -32,10 +34,8 @@ RUN apt-get update && \ git \ gcc \ g++ \ - # Required by Ubsan in Ubuntu 22.04 - libunwind-12-dev \ - libc++-12-dev \ - libc++abi-12-dev \ + gcc-11 \ + g++-11 \ libcurl4-openssl-dev \ # Needed to use autoreconf libltdl-dev \ @@ -50,9 +50,7 @@ RUN apt-get update && \ # Needed to use autoreconf perl \ pkg-config \ - python3 \ - python3-dev \ - python3-pip \ + libffi-dev \ tar \ unzip \ zip \ @@ -60,21 +58,14 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https \ - clang-tidy-12 - -# Needed for the existing driver v3.1.2.1004+ -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 + apt-transport-https -# Set clang as default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -ENV CC=clang -ENV CXX=clang++ +ENV CC=gcc +ENV CXX=g++ +RUN ln -s /usr/bin/make /usr/bin/gmake # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -83,14 +74,22 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -# clang-tidy-cache needs python -RUN update-alternatives --install /usr/bin/python python $(which python3) 10 +RUN echo "ninja version: " && ninja --version +RUN echo "g++ version: " && g++ --version +RUN echo "cmake version: " && cmake --version +RUN echo "Glibc version" && ldd --version -COPY ./requirements.txt /var/tmp/ci/requirements.txt -WORKDIR /var/tmp/downloads -RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ - then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi -RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt +WORKDIR /usr/src +RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ + tar -xzf Python-3.10.14.tgz && \ + cd Python-3.10.14 && \ + ./configure --with-ensurepip=install && \ + make -j$(nproc) \ + && make altinstall + +# clang-tidy-cache needs python +RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ + ln -sf /usr/local/bin/python3.10 /usr/bin/python # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded @@ -128,6 +127,7 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. + COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini @@ -137,11 +137,3 @@ COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.goog COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini - -# glibc 2.17 or later -RUN echo 'Installing glibc...' -RUN apt-get install -y --no-install-recommends libc6 -RUN echo 'Verifying glibc version...' -RUN dpkg -l libc6 -RUN if [ $(ldd --version | grep GLIBC | awk '{print $5}') -lt 2.17 ] ; \ - then echo 'glibc version is < 2.17: exiting...' ; exit 1 ; fi diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 88e7f6db84..22c76ba224 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -82,11 +82,15 @@ StatusRecord ValidateInputParameters( StatusRecordOr> GetFilteredProjectIds( ODBCBQClient& bq_client, std::string const& projects_filter, SQLULEN metadata_id) { + LOG(INFO) << "GetFilteredProjectIds:: Start (filter='" << projects_filter + << "', metadata_id=" << metadata_id << ")"; std::vector project_ids; auto filter_regex = BuildRegex(projects_filter, metadata_id); // For now, we use default options. // We can set timeout here as needed later. Options options; + LOG(INFO) << "GetFilteredProjectIds:: calling ListAllProjects (filter='" + << projects_filter << "', metadata_id=" << metadata_id << ")"; StatusRecordOr> projects = bq_client.ListAllProjects(options); if (!projects) { @@ -94,12 +98,16 @@ StatusRecordOr> GetFilteredProjectIds( << projects.GetStatusRecord().message; return projects.GetStatusRecord(); } + LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned " + << projects->size() << " projects; applying filter"; for (auto const& project : *projects) { if ((!metadata_id && projects_filter == "%") || re2::RE2::FullMatch(project.id, *filter_regex)) { project_ids.push_back(project.id); } } + LOG(INFO) << "GetFilteredProjectIds:: kept " << project_ids.size() + << " projects after filter"; return project_ids; } @@ -347,6 +355,8 @@ ResultSet ProcessStringResults( StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, std::string const& additional_projects) { + LOG(INFO) << "GetResultSetForProjects:: Start (metadata_id=" << metadata_id + << ", additional_projects='" << additional_projects << "')"; auto project_ids_status = GetFilteredProjectIds(bq_client, kMatchAll, metadata_id); if (!project_ids_status) { @@ -356,12 +366,19 @@ StatusRecordOr GetResultSetForProjects( } std::vector project_list = *project_ids_status; + LOG(INFO) << "GetResultSetForProjects:: project_list.size()=" + << project_list.size(); if (!additional_projects.empty()) { project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), additional_projects); + LOG(INFO) << "GetResultSetForProjects:: after AppendAdditional: " + << project_list.size(); } - return CreateResultSetForProjects(project_list); + LOG(INFO) << "GetResultSetForProjects:: building result set"; + auto rs = CreateResultSetForProjects(project_list); + LOG(INFO) << "GetResultSetForProjects:: end"; + return rs; } StatusRecordOr GetResultSetForDatasets( diff --git a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc index 566c948600..b42dabeea0 100644 --- a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc +++ b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc @@ -423,6 +423,7 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, return handle_result.GetCalculatedReturnCode(); } StatementHandle& handle = *(*handle_result); + LOG(INFO) << "SQLTablesInternal:: validated stmt handle"; StatusRecordOr attr_status = handle.GetAttribute(SQL_ATTR_METADATA_ID); @@ -432,6 +433,7 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, return LogAndReturnCode(handle, attr_status); } SQLULEN metadata_id = *attr_status; + LOG(INFO) << "SQLTablesInternal:: metadata_id=" << metadata_id; auto input_param_status = ValidateInputParameters( catalog_name, catalog_name_len, schema_name, schema_name_len, table_name, @@ -441,11 +443,15 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, << input_param_status.message; return LogAndReturnCode(handle, input_param_status); } + LOG(INFO) << "SQLTablesInternal:: validated input parameters"; std::string project_filter = ToCharStr(catalog_name, kMatchAll); std::string dataset_filter = ToCharStr(schema_name, kMatchAll); std::string table_filter = ToCharStr(table_name, kMatchAll); std::string table_type_filter = ToCharStr(table_type, kMatchAll); + LOG(INFO) << "SQLTablesInternal:: filters: project='" << project_filter + << "' dataset='" << dataset_filter << "' table='" << table_filter + << "' table_type='" << table_type_filter << "'"; if (handle.GetConnectionHandle() == nullptr) { LOG(ERROR) << "SQLTables:: Internal connection handle is null"; @@ -458,6 +464,8 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, auto const dsn = conn_handle.GetDsn(); if (dsn.filter_tables_on_default_dataset && !dsn.default_dataset.empty()) { dataset_filter = dsn.default_dataset; + LOG(INFO) << "SQLTablesInternal:: defaulted dataset_filter to '" + << dataset_filter << "'"; } } if (!conn_handle.IsConnected()) { @@ -466,6 +474,7 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, handle, StatusRecord{SQLStates::k_08S01(), "Connection to the data source is broken"}); } + LOG(INFO) << "SQLTablesInternal:: connection healthy; getting bq client"; std::shared_ptr bq_client_ptr = conn_handle.GetClient(); if (!bq_client_ptr) { LOG(ERROR) << "SQLTables:: Error establishing Datasource connection"; @@ -478,21 +487,27 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, if (!metadata_id && project_filter == SQL_ALL_CATALOGS && dataset_filter.empty() && table_filter.empty()) { + LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForProjects"; result_set_status = GetResultSetForProjects( bq_client, metadata_id, conn_handle.GetDsn().additional_projects); } else if (!metadata_id && project_filter.empty() && dataset_filter == SQL_ALL_SCHEMAS && table_filter.empty()) { + LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForDatasets"; result_set_status = GetResultSetForDatasets(bq_client, metadata_id, kMatchAll, conn_handle.GetDsn().additional_projects); } else if (!metadata_id && project_filter.empty() && dataset_filter.empty() && table_filter.empty() && table_type_filter == SQL_ALL_TABLE_TYPES) { + LOG(INFO) << "SQLTablesInternal:: branch=CreateResultSetForTableTypes"; result_set_status = CreateResultSetForTableTypes(); } else { + LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForTables (network)"; result_set_status = GetResultSetForTables(handle, bq_client, project_filter, dataset_filter, table_filter, table_type_filter, metadata_id); } + LOG(INFO) << "SQLTablesInternal:: branch returned, ok=" + << static_cast(static_cast(result_set_status)); if (!result_set_status) { LOG(ERROR) << "SQLTables::ResultSet:: " << result_set_status.GetStatusRecord().message; @@ -508,12 +523,15 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, SQLULEN max_rows = *max_rows_status; ResultSet& result_set = *result_set_status; auto& rs_rows = result_set.rows; + LOG(INFO) << "SQLTablesInternal:: result_set rows=" << rs_rows.size() + << ", max_rows=" << max_rows; if (max_rows > 0 && max_rows < rs_rows.size()) { rs_rows.erase(rs_rows.begin() + max_rows, rs_rows.end()); } DescriptorHandle& ird = handle.GetDescriptorHandle(DescriptorType::kIRD); ird.SetConnectionHandle(&conn_handle); + LOG(INFO) << "SQLTablesInternal:: building table schema"; auto table_schema = BuildTableSchemaFromRowSchema(result_set.row_schema, kSchema); if (!table_schema) { @@ -523,12 +541,14 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, } TableReference table_fields; + LOG(INFO) << "SQLTablesInternal:: populating IRD"; auto ird_status = StatementHandle::PopulateIrd(ird, *table_schema, table_fields, true); if (!ird_status.ok()) { LOG(ERROR) << "SQLTables::PopulateIrd:: " << ird_status.message; return LogAndReturnCode(handle, ird_status); } + LOG(INFO) << "SQLTablesInternal:: IRD populated"; LOG(INFO) << "SQLTablesInternal:: end"; handle.SetResultSet(result_set); handle.SetStmtState(StmtStates::kStatementExecutedWithRs); From 7560d3fe2b29f17c58b368a57d23e039af268316 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Mon, 4 May 2026 14:21:53 +0530 Subject: [PATCH 04/32] more logging --- .../odbc/bq_driver/internal/odbc_sql_fetch.cc | 390 ++++++++++++------ .../bq_driver/internal/odbc_sql_tables.cc | 46 ++- .../odbc/bq_driver/internal/odbc_sql_tables.h | 12 +- .../bq_driver/internal/odbc_stmt_handle.cc | 20 + google/cloud/odbc/bq_driver/internal/utils.cc | 58 +++ .../odbc/bq_driver/odbc_driver_metadata.cc | 10 +- .../cloud/odbc/bq_driver/odbc_sql_requests.cc | 71 +++- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 38 +- 8 files changed, 479 insertions(+), 166 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc index 69618d66f8..f31e28b873 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc @@ -27,79 +27,137 @@ StatusRecord WriteToApplicationBuffer(DSValue const& ds_val, DescriptorRecord& app_desc_rec, SQLLEN bind_offset, SQLLEN bind_offset_ind) { - SQLSMALLINT target_c_type = app_desc_rec.concise_type; - SQLPOINTER app_buffer = app_desc_rec.data_ptr; - SQLLEN app_buffer_len = app_desc_rec.octet_length; - SQLLEN* indicator_ptr = app_desc_rec.indicator_ptr; - SQLLEN* octet_length_ptr = app_desc_rec.octet_length_ptr; + LOG(INFO) << "WriteToApplicationBuffer:: Start (bq_data_type=" + << static_cast(bq_data_type) + << ", target_c_type=" << app_desc_rec.concise_type + << ", bind_offset=" << bind_offset + << ", bind_offset_ind=" << bind_offset_ind + << ", data_ptr=" << static_cast(app_desc_rec.data_ptr) + << ", octet_length=" << app_desc_rec.octet_length + << ", ds_val.size()=" << ds_val.size() << ")"; + try { + SQLSMALLINT target_c_type = app_desc_rec.concise_type; + SQLPOINTER app_buffer = app_desc_rec.data_ptr; + SQLLEN app_buffer_len = app_desc_rec.octet_length; + SQLLEN* indicator_ptr = app_desc_rec.indicator_ptr; + SQLLEN* octet_length_ptr = app_desc_rec.octet_length_ptr; - app_buffer = reinterpret_cast(app_buffer) + bind_offset; - if (indicator_ptr) { - indicator_ptr = reinterpret_cast( - reinterpret_cast(indicator_ptr) + bind_offset_ind); - } - if (octet_length_ptr) { - octet_length_ptr = reinterpret_cast( - reinterpret_cast(octet_length_ptr) + bind_offset_ind); - } + if (app_buffer == nullptr) { + LOG(ERROR) << "WriteToApplicationBuffer:: data_ptr is NULL; column not " + "bound? Skipping."; + return StatusRecord::Ok(); + } - if (IsDSValueNull(ds_val)) { - LOG(ERROR) << "WriteToApplicationBuffer:: Indicator variable required but " - "not supplied for NULL data."; - if (indicator_ptr == nullptr) { - return {SQLStates::k_22002(), - "Indicator variable required but not supplied"}; + app_buffer = reinterpret_cast(app_buffer) + bind_offset; + if (indicator_ptr) { + indicator_ptr = reinterpret_cast( + reinterpret_cast(indicator_ptr) + bind_offset_ind); } - *indicator_ptr = SQL_NULL_DATA; - return StatusRecord::Ok(); - } - // We need to reset the indicator_ptr once it has been set to SQL_NULL_DATA - // for DSNullValues. - if (indicator_ptr) { - *indicator_ptr = ds_val.size(); - } + if (octet_length_ptr) { + octet_length_ptr = reinterpret_cast( + reinterpret_cast(octet_length_ptr) + bind_offset_ind); + } + LOG(INFO) << "WriteToApplicationBuffer:: pointers offset; " + "app_buffer (post-offset)=" << app_buffer + << ", indicator_ptr=" << static_cast(indicator_ptr) + << ", octet_length_ptr=" << static_cast(octet_length_ptr); - DataBuffer data = {target_c_type, app_buffer, app_buffer_len, - octet_length_ptr}; - StatusRecord status_record; - switch (bq_data_type) { - case BQDataType::kInt64: - return ConvertFromArithmeticDSValue(ds_val, data); - case BQDataType::kFloat64: - return ConvertFromArithmeticDSValue(ds_val, data); - case BQDataType::kString: - return ConvertFromStringDSValue(ds_val, data); - case BQDataType::kDate: - return ConvertFromDateDSValue(ds_val, data); - case BQDataType::kTime: - return ConvertFromTimeDSValue(ds_val, data); - case BQDataType::kJson: - return ConvertFromJsonDSValue(ds_val, data); - case BQDataType::kStruct: - return ConvertFromStructDSValue(ds_val, data); - case BQDataType::kArray: - return ConvertFromArrayDSValue(ds_val, data); - case BQDataType::kTimeStamp: - return ConvertFromTimestampDSValue(ds_val, data); - case BQDataType::kDatetime: - return ConvertFromDatetimeDSValue(ds_val, data); - case BQDataType::kInterval: - return ConvertFromIntervalDSValue(ds_val, data); - case BQDataType::kBool: - return ConvertFromBooleanDSValue(ds_val, data); - case BQDataType::kGeography: - return ConvertFromGeographyDSValue(ds_val, data); - case BQDataType::kBytes: - return ConvertFromBytesDSValue(ds_val, data); - case BQDataType::kRange: - return ConvertFromRangeDSValue(ds_val, data); - case BQDataType::kBigNumeric: - case BQDataType::kNumeric: - return ConvertFromNumericDSValue(ds_val, data); + if (IsDSValueNull(ds_val)) { + LOG(INFO) << "WriteToApplicationBuffer:: ds_val is NULL"; + if (indicator_ptr == nullptr) { + LOG(ERROR) << "WriteToApplicationBuffer:: Indicator variable required " + "but not supplied for NULL data."; + return {SQLStates::k_22002(), + "Indicator variable required but not supplied"}; + } + *indicator_ptr = SQL_NULL_DATA; + LOG(INFO) << "WriteToApplicationBuffer:: wrote SQL_NULL_DATA to " + "indicator; returning"; + return StatusRecord::Ok(); + } + // We need to reset the indicator_ptr once it has been set to SQL_NULL_DATA + // for DSNullValues. + if (indicator_ptr) { + LOG(INFO) << "WriteToApplicationBuffer:: writing ds_val.size()=" + << ds_val.size() << " to indicator_ptr"; + *indicator_ptr = ds_val.size(); + } + + DataBuffer data = {target_c_type, app_buffer, app_buffer_len, + octet_length_ptr}; + LOG(INFO) << "WriteToApplicationBuffer:: dispatching by bq_data_type=" + << static_cast(bq_data_type); + StatusRecord status_record; + switch (bq_data_type) { + case BQDataType::kInt64: + status_record = ConvertFromArithmeticDSValue(ds_val, data); + break; + case BQDataType::kFloat64: + status_record = ConvertFromArithmeticDSValue(ds_val, data); + break; + case BQDataType::kString: + status_record = ConvertFromStringDSValue(ds_val, data); + break; + case BQDataType::kDate: + status_record = ConvertFromDateDSValue(ds_val, data); + break; + case BQDataType::kTime: + status_record = ConvertFromTimeDSValue(ds_val, data); + break; + case BQDataType::kJson: + status_record = ConvertFromJsonDSValue(ds_val, data); + break; + case BQDataType::kStruct: + status_record = ConvertFromStructDSValue(ds_val, data); + break; + case BQDataType::kArray: + status_record = ConvertFromArrayDSValue(ds_val, data); + break; + case BQDataType::kTimeStamp: + status_record = ConvertFromTimestampDSValue(ds_val, data); + break; + case BQDataType::kDatetime: + status_record = ConvertFromDatetimeDSValue(ds_val, data); + break; + case BQDataType::kInterval: + status_record = ConvertFromIntervalDSValue(ds_val, data); + break; + case BQDataType::kBool: + status_record = ConvertFromBooleanDSValue(ds_val, data); + break; + case BQDataType::kGeography: + status_record = ConvertFromGeographyDSValue(ds_val, data); + break; + case BQDataType::kBytes: + status_record = ConvertFromBytesDSValue(ds_val, data); + break; + case BQDataType::kRange: + status_record = ConvertFromRangeDSValue(ds_val, data); + break; + case BQDataType::kBigNumeric: + case BQDataType::kNumeric: + status_record = ConvertFromNumericDSValue(ds_val, data); + break; + default: + LOG(ERROR) << "WriteToApplicationBuffer:: Data type not supported: " + << bq_data_type; + return {SQLStates::k_HYC00(), "Data type not supported"}; + } + LOG(INFO) << "WriteToApplicationBuffer:: end ok=" + << static_cast(status_record.ok()) + << ", message='" << status_record.message << "'"; + return status_record; + } catch (std::exception const& e) { + LOG(ERROR) << "WriteToApplicationBuffer:: std::exception caught: what='" + << e.what() << "'"; + return StatusRecord{ + SQLStates::k_HY000(), + std::string("exception in WriteToApplicationBuffer: ") + e.what()}; + } catch (...) { + LOG(ERROR) << "WriteToApplicationBuffer:: unknown exception caught"; + return StatusRecord{SQLStates::k_HY000(), + "Unknown exception in WriteToApplicationBuffer"}; } - LOG(ERROR) << "WriteToApplicationBuffer:: Data type not supported: " - << bq_data_type; - return {SQLStates::k_HYC00(), "Data type not supported"}; } // This is according to the spec: @@ -149,92 +207,152 @@ SQLLEN GetElemSize(DescriptorRecord& app_desc_rec) { StatusRecord WriteDSRow(DSRow const& ds_row, RowSchema const& schema, DescriptorHandle& ard, int row_num) { - SQLLEN* bind_offset_ptr = ard.GetHeaderRecord().bind_offset_ptr; - SQLLEN bind_offset = 0; - if (bind_offset_ptr) { - bind_offset = *bind_offset_ptr; - } - - for (ColumnSchema const& col_schema : schema) { - int col_index = col_schema.col_index; - DSValue const& ds_val = ds_row[col_index]; - // Column is not bound. - if (!ard.HasDescriptorRecord(col_index + 1)) { - continue; + LOG(INFO) << "WriteDSRow:: Start (row_num=" << row_num + << ", schema cols=" << schema.size() + << ", row vals=" << ds_row.size() << ")"; + try { + SQLLEN* bind_offset_ptr = ard.GetHeaderRecord().bind_offset_ptr; + SQLLEN bind_offset = 0; + if (bind_offset_ptr) { + bind_offset = *bind_offset_ptr; } - DescriptorRecord& col_desc = ard.GetDescriptorRecord(col_index + 1); + LOG(INFO) << "WriteDSRow:: bind_offset_ptr=" + << static_cast(bind_offset_ptr) + << ", bind_offset=" << bind_offset; - SQLLEN elem_size, elem_size_ind; - SQLINTEGER bind_type = ard.GetHeaderRecord().bind_type; - if (bind_type == SQL_BIND_BY_COLUMN) { - elem_size = GetElemSize(col_desc); - elem_size_ind = sizeof(SQLLEN); - } else { - elem_size = bind_type; - elem_size_ind = bind_type; - } - SQLLEN row_offset = row_num * elem_size; - SQLLEN row_offset_ind = row_num * elem_size_ind; + for (ColumnSchema const& col_schema : schema) { + int col_index = col_schema.col_index; + LOG(INFO) << "WriteDSRow:: col_index=" << col_index + << ", row size=" << ds_row.size(); + if (col_index < 0 || + static_cast(col_index) >= ds_row.size()) { + LOG(ERROR) << "WriteDSRow:: col_index out of range; skipping"; + continue; + } + DSValue const& ds_val = ds_row[col_index]; + // Column is not bound. + if (!ard.HasDescriptorRecord(col_index + 1)) { + LOG(INFO) << "WriteDSRow:: col_index=" << col_index + << " not bound; skipping"; + continue; + } + LOG(INFO) << "WriteDSRow:: col_index=" << col_index + << " is bound; getting descriptor"; + DescriptorRecord& col_desc = ard.GetDescriptorRecord(col_index + 1); - BQDataType bq_data_type = col_schema.col_type; - if (col_schema.is_mode_repeated) { - bq_data_type = BQDataType::kArray; - } + SQLLEN elem_size, elem_size_ind; + SQLINTEGER bind_type = ard.GetHeaderRecord().bind_type; + if (bind_type == SQL_BIND_BY_COLUMN) { + elem_size = GetElemSize(col_desc); + elem_size_ind = sizeof(SQLLEN); + } else { + elem_size = bind_type; + elem_size_ind = bind_type; + } + SQLLEN row_offset = row_num * elem_size; + SQLLEN row_offset_ind = row_num * elem_size_ind; + LOG(INFO) << "WriteDSRow:: col=" << col_index + << " elem_size=" << elem_size + << " bind_type=" << bind_type + << " row_offset=" << row_offset + << " row_offset_ind=" << row_offset_ind; - StatusRecord status_record = WriteToApplicationBuffer( - ds_val, bq_data_type, col_desc, bind_offset + row_offset, - bind_offset + row_offset_ind); - if (!status_record.ok()) { - LOG(ERROR) << "WriteDSRow::WriteToApplicationBuffer:: " - << status_record.message; - return status_record; + BQDataType bq_data_type = col_schema.col_type; + if (col_schema.is_mode_repeated) { + bq_data_type = BQDataType::kArray; + } + + LOG(INFO) << "WriteDSRow:: col=" << col_index + << " calling WriteToApplicationBuffer"; + StatusRecord status_record = WriteToApplicationBuffer( + ds_val, bq_data_type, col_desc, bind_offset + row_offset, + bind_offset + row_offset_ind); + LOG(INFO) << "WriteDSRow:: col=" << col_index + << " WriteToApplicationBuffer returned ok=" + << static_cast(status_record.ok()); + if (!status_record.ok()) { + LOG(ERROR) << "WriteDSRow::WriteToApplicationBuffer:: " + << status_record.message; + return status_record; + } } + LOG(INFO) << "WriteDSRow:: end"; + return StatusRecord::Ok(); + } catch (std::exception const& e) { + LOG(ERROR) << "WriteDSRow:: std::exception caught: what='" << e.what() + << "'"; + return StatusRecord{SQLStates::k_HY000(), + std::string("exception in WriteDSRow: ") + e.what()}; + } catch (...) { + LOG(ERROR) << "WriteDSRow:: unknown exception caught"; + return StatusRecord{SQLStates::k_HY000(), + "Unknown exception in WriteDSRow"}; } - return StatusRecord::Ok(); } StatusRecord WriteRowset(ResultSet const& result_set, int const rowset_size, DescriptorHandle& ard, DescriptorHandle& ird) { - if (rowset_size <= 0) { - LOG(ERROR) << "WriteRowset:: rowset_size should not be <= 0"; - StatusRecord status_record = {SQLStates::k_HY000(), - "rowset_size should not be <= 0"}; - return status_record; - } - int cursor = result_set.cursor; - int row_counter = 0; - SQLUSMALLINT* row_status_ptr = ird.GetHeaderRecord().array_status_ptr; - // We write 'rowset_size' rows from result_set.rows starting at the index - // 'cursor' - for (int i = cursor; i < cursor + rowset_size && i < result_set.rows.size(); - i++, row_counter++) { - StatusRecord status_record = - WriteDSRow(result_set.rows[i], result_set.row_schema, ard, i - cursor); - if (!status_record.ok()) { - LOG(ERROR) << "WriteRowset::WriteDSRow:: " << status_record.message; + LOG(INFO) << "WriteRowset:: Start (rowset_size=" << rowset_size + << ", cursor=" << result_set.cursor + << ", rows.size()=" << result_set.rows.size() << ")"; + try { + if (rowset_size <= 0) { + LOG(ERROR) << "WriteRowset:: rowset_size should not be <= 0"; + StatusRecord status_record = {SQLStates::k_HY000(), + "rowset_size should not be <= 0"}; return status_record; } + int cursor = result_set.cursor; + int row_counter = 0; + SQLUSMALLINT* row_status_ptr = ird.GetHeaderRecord().array_status_ptr; + LOG(INFO) << "WriteRowset:: row_status_ptr=" + << static_cast(row_status_ptr); + // We write 'rowset_size' rows from result_set.rows starting at the index + // 'cursor' + for (int i = cursor; i < cursor + rowset_size && i < result_set.rows.size(); + i++, row_counter++) { + LOG(INFO) << "WriteRowset:: writing row " << i << " (offset=" + << (i - cursor) << ")"; + StatusRecord status_record = WriteDSRow( + result_set.rows[i], result_set.row_schema, ard, i - cursor); + LOG(INFO) << "WriteRowset:: WriteDSRow row " << i << " returned ok=" + << static_cast(status_record.ok()); + if (!status_record.ok()) { + LOG(ERROR) << "WriteRowset::WriteDSRow:: " << status_record.message; + return status_record; + } - if (row_status_ptr) { - row_status_ptr[i - cursor] = SQL_ROW_SUCCESS; - } + if (row_status_ptr) { + row_status_ptr[i - cursor] = SQL_ROW_SUCCESS; + } - result_set.cursor = i; - } + result_set.cursor = i; + } + LOG(INFO) << "WriteRowset:: wrote " << row_counter << " rows"; - // Mark unused rows - if (row_status_ptr) { - for (int i = row_counter; i < rowset_size; i++) { - row_status_ptr[i] = SQL_ROW_NOROW; + // Mark unused rows + if (row_status_ptr) { + for (int i = row_counter; i < rowset_size; i++) { + row_status_ptr[i] = SQL_ROW_NOROW; + } } - } - SQLULEN* rows_processed_ptr = ird.GetHeaderRecord().rows_processed_ptr; - if (rows_processed_ptr) { - *rows_processed_ptr = row_counter; + SQLULEN* rows_processed_ptr = ird.GetHeaderRecord().rows_processed_ptr; + if (rows_processed_ptr) { + *rows_processed_ptr = row_counter; + } + LOG(INFO) << "WriteRowset:: end"; + return StatusRecord::Ok(); + } catch (std::exception const& e) { + LOG(ERROR) << "WriteRowset:: std::exception caught: what='" << e.what() + << "'"; + return StatusRecord{SQLStates::k_HY000(), + std::string("exception in WriteRowset: ") + e.what()}; + } catch (...) { + LOG(ERROR) << "WriteRowset:: unknown exception caught"; + return StatusRecord{SQLStates::k_HY000(), + "Unknown exception in WriteRowset"}; } - - return StatusRecord::Ok(); } StatusRecord FetchNextResultSet(StatementHandle& stmt_handle) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 22c76ba224..8214a2ccb5 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -354,11 +354,45 @@ ResultSet ProcessStringResults( StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, - std::string const& additional_projects) { + std::string const& additional_projects, + std::string const& connection_catalog) { LOG(INFO) << "GetResultSetForProjects:: Start (metadata_id=" << metadata_id - << ", additional_projects='" << additional_projects << "')"; + << ", additional_projects='" << additional_projects + << "', connection_catalog='" << connection_catalog << "')"; + + // Fast path: when the connection has a catalog bound via the DSN, skip the + // BigQuery ListAllProjects round-trip. Returning [catalog] + + // additional_projects is what HANA SDA's catalog-discovery probe actually + // needs, and it bypasses the failure in ListAllProjects that surfaces as + // "Cannot get remote source objects" when SDA calls CHECK_REMOTE_SOURCE. + if (!connection_catalog.empty()) { + LOG(INFO) << "GetResultSetForProjects:: fast-path entry; using catalog '" + << connection_catalog << "'"; + std::vector project_list = {connection_catalog}; + LOG(INFO) << "GetResultSetForProjects:: fast-path; project_list.size()=" + << project_list.size(); + if (!additional_projects.empty()) { + LOG(INFO) << "GetResultSetForProjects:: fast-path; appending additional"; + project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), + additional_projects); + LOG(INFO) + << "GetResultSetForProjects:: fast-path; after AppendAdditional: " + << project_list.size(); + } + LOG(INFO) << "GetResultSetForProjects:: fast-path; building result set"; + auto rs = CreateResultSetForProjects(project_list); + LOG(INFO) << "GetResultSetForProjects:: fast-path; end (" + << project_list.size() << " projects)"; + return rs; + } + + LOG(INFO) << "GetResultSetForProjects:: slow-path; calling " + "GetFilteredProjectIds"; auto project_ids_status = GetFilteredProjectIds(bq_client, kMatchAll, metadata_id); + LOG(INFO) << "GetResultSetForProjects:: slow-path; GetFilteredProjectIds " + "returned, ok=" + << static_cast(static_cast(project_ids_status)); if (!project_ids_status) { LOG(ERROR) << "GetResultSetForProjects::GetFilteredProjectIds:: " << project_ids_status.GetStatusRecord().message; @@ -366,18 +400,18 @@ StatusRecordOr GetResultSetForProjects( } std::vector project_list = *project_ids_status; - LOG(INFO) << "GetResultSetForProjects:: project_list.size()=" + LOG(INFO) << "GetResultSetForProjects:: slow-path; project_list.size()=" << project_list.size(); if (!additional_projects.empty()) { project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), additional_projects); - LOG(INFO) << "GetResultSetForProjects:: after AppendAdditional: " + LOG(INFO) << "GetResultSetForProjects:: slow-path; after AppendAdditional: " << project_list.size(); } - LOG(INFO) << "GetResultSetForProjects:: building result set"; + LOG(INFO) << "GetResultSetForProjects:: slow-path; building result set"; auto rs = CreateResultSetForProjects(project_list); - LOG(INFO) << "GetResultSetForProjects:: end"; + LOG(INFO) << "GetResultSetForProjects:: slow-path; end"; return rs; } diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h index ae0a45e756..ab167bd97c 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h @@ -106,9 +106,19 @@ ResultSet ProcessStringResults( std::vector> const& rows); // Search for all projects and populate ResultSet for it. +// +// `connection_catalog` (when non-empty) is the catalog the connection is bound +// to via the DSN's `Catalog=` field. If supplied, we skip the BigQuery +// `ListAllProjects` round-trip and return `[connection_catalog] + +// additional_projects` directly. This is what SAP HANA SDA needs for its +// catalog-discovery probe (`SQLTables(catalog="%", schema="", table="")`), +// and avoiding the network call keeps the probe well under HANA's ~30s +// discovery timeout — `ListAllProjects` can take many seconds when the +// service account has access to large numbers of GCP projects. odbc_internal::StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, - std::string const& additional_projects = ""); + std::string const& additional_projects = "", + std::string const& connection_catalog = ""); // Search for all datasets in all projects and populate ResultSet for it. odbc_internal::StatusRecordOr GetResultSetForDatasets( diff --git a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc index 701ba5fade..1a7c79117c 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc @@ -205,7 +205,10 @@ bool IsSelectQuery(std::string const& q) { // TODO(b/342044533) Sanitize query text to avoid potential SQL Injection // risk. StatusRecord StatementHandle::PrepareQuery(std::string const& query) { + LOG(INFO) << "PrepareQuery:: Start (query.length()=" << query.length() << ")"; StatusRecord transaction_status = BeginTransactionIfNeeded(*conn_handle_); + LOG(INFO) << "PrepareQuery:: BeginTransactionIfNeeded ok=" + << static_cast(transaction_status.ok()); if (!transaction_status.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::BeginTransactionIfNeeded:: " << transaction_status.message; @@ -213,6 +216,7 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { } ConnectionHandle& conn_handle = *GetConnectionHandle(); + LOG(INFO) << "PrepareQuery:: building Job request"; Job req; req.configuration.query.query = query; req.configuration.query.use_query_cache = conn_handle.GetDsn().is_query_cache; @@ -253,6 +257,8 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { if (has_named) { req.configuration.query.parameter_mode = "NAMED"; } + LOG(INFO) << "PrepareQuery:: positional=" << has_positional + << ", named=" << has_named; } std::vector combined_properties = @@ -269,21 +275,29 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { req.configuration.query.connection_properties = combined_properties; Options opt; opt.set(conn_handle.GetDsn().max_retries); + LOG(INFO) << "PrepareQuery:: calling InsertJob (catalog='" + << conn_handle.GetDsn().catalog << "', dry_run=true)"; auto response = conn_handle.GetClient()->InsertJob( conn_handle.GetDsn().catalog, req, opt); + LOG(INFO) << "PrepareQuery:: InsertJob returned ok=" + << static_cast(response.Ok()); if (!response.Ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::InsertJob:: " << response.GetStatusRecord().message; return response.GetStatusRecord(); } + LOG(INFO) << "PrepareQuery:: calling PopulateResultSet"; auto& schema = response.GetValue().statistics.job_query_stats.schema; auto pop_response = PopulateResultSet(schema); + LOG(INFO) << "PrepareQuery:: PopulateResultSet returned ok=" + << static_cast(pop_response.ok()); if (!pop_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateResultSet:: " << pop_response.message; return pop_response; } + LOG(INFO) << "PrepareQuery:: SetQueryParameters"; SetQueryParameters( response.GetValue() .statistics.job_query_stats.undeclared_query_parameters); @@ -304,22 +318,28 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { table_fields = response.GetValue().configuration.query.destination_table; } + LOG(INFO) << "PrepareQuery:: building IRD"; DescriptorHandle& desc_handle = this->GetDescriptorHandle(DescriptorType::kIRD); desc_handle.SetConnectionHandle(&conn_handle); desc_handle.ClearDescriptorRecordsMap(); StatusRecord ird_response = PopulateIrd(desc_handle, schema, table_fields); + LOG(INFO) << "PrepareQuery:: PopulateIrd returned ok=" + << static_cast(ird_response.ok()); if (!ird_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateIrd:: " << ird_response.message; return ird_response; } + LOG(INFO) << "PrepareQuery:: building IPD"; DescriptorHandle& ipd_desc_handle = this->GetDescriptorHandle(DescriptorType::kIPD); ipd_desc_handle.ClearDescriptorRecordsMap(); auto job_statistics = (*response).statistics; StatusRecord ipd_response = PopulateIpd(ipd_desc_handle, job_statistics); + LOG(INFO) << "PrepareQuery:: PopulateIpd returned ok=" + << static_cast(ipd_response.ok()); if (!ipd_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateIpd:: " << ipd_response.message; diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 84f8f15aaa..e4fb83a3f9 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -1036,6 +1036,64 @@ bool IsInfoTypeString(SQLUSMALLINT InfoType) { } } +// Translate an ODBC LIKE pattern into a C++ regex pattern. +// +// ODBC pattern semantics (per ODBC spec): +// '%' -> any sequence of characters (regex ".*") +// '_' -> any single character (regex ".") +// '\X' -> literal X (the backslash is the escape; consumed in output) +// any other char -> emitted as-is +// +// History: this used to be three chained `std::regex_replace` calls. That was +// fragile because it depended on the platform's back-end being able +// to compile small patterns like `^%|([^\\])%`. On hosts with stripped +// locale data (notably the SAP HANA SDA dpserver process when launched by +// adm with a minimal LC_* env), one of those `std::regex` constructions +// throws — and the exception unwinds across the ODBC ABI boundary, aborting +// the driver process with no error to the caller. We replace it with a +// manual single-pass loop that has no regex dependency, no allocations +// beyond the result string, and no failure modes other than std::bad_alloc +// (which is also caught below for safety). +std::string CastOdbcRegexToCppRegex(std::string const& str) { + LOG(INFO) << "CastOdbcRegexToCppRegex:: Start (input='" << str + << "', length=" << str.size() << ")"; + try { + std::string result; + // Worst case: every character becomes ".*" (2 chars). Reserve up front. + result.reserve(str.size() * 2); + for (size_t i = 0; i < str.size(); ++i) { + char c = str[i]; + if (c == '\\') { + if (i + 1 < str.size()) { + // Escape: emit the next char literally, consume both the + // backslash and the following char from the input. + result.push_back(str[i + 1]); + ++i; + } + // else: lone trailing backslash — drop it. The previous + // regex-based implementation removed all stray backslashes at the + // end of processing, so this matches that behavior. + } else if (c == '%') { + result.append(".*"); + } else if (c == '_') { + result.push_back('.'); + } else { + result.push_back(c); + } + } + LOG(INFO) << "CastOdbcRegexToCppRegex:: end; result='" << result + << "' (length=" << result.size() << ")"; + return result; + } catch (std::exception const& e) { + LOG(ERROR) << "CastOdbcRegexToCppRegex:: std::exception caught: what='" + << e.what() << "'"; + throw; + } catch (...) { + LOG(ERROR) << "CastOdbcRegexToCppRegex:: unknown exception caught"; + throw; + } +} + // Translate an ODBC LIKE pattern into a C++ regex pattern. // // ODBC pattern semantics (per ODBC spec): diff --git a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc index b42dabeea0..0abe7decad 100644 --- a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc +++ b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc @@ -487,9 +487,15 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, if (!metadata_id && project_filter == SQL_ALL_CATALOGS && dataset_filter.empty() && table_filter.empty()) { - LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForProjects"; + auto const& dsn = conn_handle.GetDsn(); + LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForProjects " + "(dsn.catalog='" + << dsn.catalog << "', additional_projects='" + << dsn.additional_projects << "')"; + // Pass dsn.catalog as the 4th arg so GetResultSetForProjects can take + // the fast path and avoid ListAllProjects (which crashes on HANA SDA). result_set_status = GetResultSetForProjects( - bq_client, metadata_id, conn_handle.GetDsn().additional_projects); + bq_client, metadata_id, dsn.additional_projects, dsn.catalog); } else if (!metadata_id && project_filter.empty() && dataset_filter == SQL_ALL_SCHEMAS && table_filter.empty()) { LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForDatasets"; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_requests.cc b/google/cloud/odbc/bq_driver/odbc_sql_requests.cc index 3c9db5a102..7761d4c82a 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_requests.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_requests.cc @@ -356,23 +356,48 @@ StatusRecord ActuallyProcessExecute(StatementHandle& stmt_handle, // This function synchronously processes current SQLExecDirect requests. StatusRecord ActuallyProcessExecDirect(StatementHandle& stmt_handle) { - stmt_handle.SetStmtState(StmtStates::kStatementStillExecuting); - - std::string query_str = stmt_handle.GetQueryString(); - // We need to call `PrepareQuery` because: - // 1) We need to get `statement_type` during `ActuallyProcessExecute` - // through - // `Job::statistics.job_query_stats.statement_type`. This is not possible - // through `PostQueryResults` - // 2) For positional params, we need to get `QueryParameter`s before - // SQLExecDirect is called. - StatusRecord prepare_status = stmt_handle.PrepareQuery(query_str); - if (!prepare_status.ok()) { - LOG(ERROR) << "ActuallyProcessExecDirect::PrepareQuery:: " - << prepare_status.message; - return prepare_status; - } - return ActuallyProcessExecute(stmt_handle, StmtStates::kStatementNotPrepared); + LOG(INFO) << "ActuallyProcessExecDirect:: Start"; + try { + stmt_handle.SetStmtState(StmtStates::kStatementStillExecuting); + LOG(INFO) << "ActuallyProcessExecDirect:: state set to StillExecuting"; + + std::string query_str = stmt_handle.GetQueryString(); + LOG(INFO) << "ActuallyProcessExecDirect:: query_str.length()=" + << query_str.length() + << ", first 200 chars='" << query_str.substr(0, 200) << "'"; + // We need to call `PrepareQuery` because: + // 1) We need to get `statement_type` during `ActuallyProcessExecute` + // through + // `Job::statistics.job_query_stats.statement_type`. This is not possible + // through `PostQueryResults` + // 2) For positional params, we need to get `QueryParameter`s before + // SQLExecDirect is called. + LOG(INFO) << "ActuallyProcessExecDirect:: calling PrepareQuery"; + StatusRecord prepare_status = stmt_handle.PrepareQuery(query_str); + LOG(INFO) << "ActuallyProcessExecDirect:: PrepareQuery returned, ok=" + << static_cast(prepare_status.ok()); + if (!prepare_status.ok()) { + LOG(ERROR) << "ActuallyProcessExecDirect::PrepareQuery:: " + << prepare_status.message; + return prepare_status; + } + LOG(INFO) << "ActuallyProcessExecDirect:: calling ActuallyProcessExecute"; + auto rs = ActuallyProcessExecute(stmt_handle, + StmtStates::kStatementNotPrepared); + LOG(INFO) << "ActuallyProcessExecDirect:: ActuallyProcessExecute returned, " + "ok=" + << static_cast(rs.ok()); + return rs; + } catch (std::exception const& e) { + LOG(ERROR) << "ActuallyProcessExecDirect:: std::exception caught: what='" + << e.what() << "'"; + return StatusRecord{SQLStates::k_HY000(), + std::string("exception: ") + e.what()}; + } catch (...) { + LOG(ERROR) << "ActuallyProcessExecDirect:: unknown exception caught"; + return StatusRecord{SQLStates::k_HY000(), + "Unknown exception in ActuallyProcessExecDirect"}; + } } SQLRETURN HandleAsyncGetResults(StatementHandle& stmt_handle, @@ -1034,7 +1059,8 @@ SQLRETURN SQLExecuteInternal(SQLHSTMT statement_handle) { SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, SQLCHAR* in_statement_text, SQLINTEGER in_text_length) { - LOG(INFO) << "SQLExecDirectInternal:: Start"; + LOG(INFO) << "SQLExecDirectInternal:: Start (in_text_length=" + << in_text_length << ")"; StatusRecordOr handle_result = ValidateStatementHandle(statement_handle); if (!handle_result) { @@ -1043,6 +1069,7 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, return handle_result.GetCalculatedReturnCode(); } StatementHandle& stmt_handle = *(*handle_result); + LOG(INFO) << "SQLExecDirectInternal:: stmt handle validated"; if ((in_text_length < 1) && (in_text_length != SQL_NTS)) { StatusRecord status_record = {SQLStates::k_HY090(), "Invalid query length"}; @@ -1065,6 +1092,8 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, } std::string query_str = ToCharStr(in_statement_text); + LOG(INFO) << "SQLExecDirectInternal:: query_str.length()=" + << query_str.length(); if (query_str.empty()) { auto status_record = StatusRecord{SQLStates::k_HY000(), "Query text is null or empty"}; @@ -1072,6 +1101,7 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, return LogAndReturnCode(stmt_handle, status_record); } stmt_handle.SetQueryString(query_str); + LOG(INFO) << "SQLExecDirectInternal:: query string stored on handle"; // Boolean to tell us if a ExecDirect Query was to be processed async and it // wasn't finished last time. We are not using a `StmtStates` here because @@ -1177,7 +1207,12 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, // ***************************************************************** // STEP 4: Synchronous execution // ***************************************************************** + LOG(INFO) << "SQLExecDirectInternal:: synchronous path; calling " + "ActuallyProcessExecDirect"; StatusRecord execute_status = ActuallyProcessExecDirect(stmt_handle); + LOG(INFO) << "SQLExecDirectInternal:: ActuallyProcessExecDirect returned, " + "ok=" + << static_cast(execute_status.ok()); if (execute_status.sql_state == SQLStates::k_SQL_NEED_DATA()) { stmt_handle.SetStmtState(StmtStates::kNeedsParams); return SQL_NEED_DATA; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index 57bc814b9e..a1ae93b846 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -220,7 +220,9 @@ SQLRETURN SQLFetchInternal(SQLHSTMT statement_handle) { SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, SQLSMALLINT fetch_orientation, SQLLEN /*fetch_offset*/) { - LOG(INFO) << "SQLFetchScrollInternal:: Start"; + LOG(INFO) << "SQLFetchScrollInternal:: Start (fetch_orientation=" + << fetch_orientation << ")"; + try { StatusRecordOr handle_result = ValidateStatementHandle(statement_handle); StatusRecord status_record; @@ -230,14 +232,18 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, return handle_result.GetCalculatedReturnCode(); } StatementHandle& handle = *(*handle_result); + LOG(INFO) << "SQLFetchScrollInternal:: stmt handle validated"; - if (handle.GetStmtState() == StmtStates::kStatementExecutedWithoutRs) { + auto stmt_state = handle.GetStmtState(); + LOG(INFO) << "SQLFetchScrollInternal:: stmt_state=" + << static_cast(stmt_state); + if (stmt_state == StmtStates::kStatementExecutedWithoutRs) { status_record = StatusRecord{SQLStates::k_24000(), "Invalid cursor state."}; LOG(ERROR) << "SQLFetchScroll:: " << status_record.message; return LogAndReturnCode(handle, status_record); } - if (handle.GetStmtState() != StmtStates::kStatementExecutedWithRs) { + if (stmt_state != StmtStates::kStatementExecutedWithRs) { status_record = {SQLStates::k_HY010(), "No statement has been executed with result set"}; LOG(ERROR) << "SQLFetchScroll:: " << status_record.message; @@ -257,13 +263,23 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, << status_record.message; return LogAndReturnCode(handle, status_record); } + LOG(INFO) << "SQLFetchScrollInternal:: fetch_orientation validated"; + LOG(INFO) << "SQLFetchScrollInternal:: getting ARD"; DescriptorHandle& ard = handle.GetDescriptorHandle(DescriptorType::kARD); + LOG(INFO) << "SQLFetchScrollInternal:: getting result set"; ResultSet& result_set = handle.GetResultSet(); + LOG(INFO) << "SQLFetchScrollInternal:: cursor=" << result_set.cursor + << ", rows.size()=" << result_set.rows.size() + << ", page_token.empty()=" + << handle.GetPagingInfo().page_token.empty(); if (result_set.cursor >= result_set.rows.size() - 1 && !handle.GetPagingInfo().page_token.empty()) { + LOG(INFO) << "SQLFetchScrollInternal:: calling FetchNextResultSet"; StatusRecord status_record = FetchNextResultSet(handle); + LOG(INFO) << "SQLFetchScrollInternal:: FetchNextResultSet returned ok=" + << static_cast(status_record.ok()); if (!status_record.ok()) { LOG(ERROR) << "SQLFetchScroll::FetchNextResultSet:: " << status_record.message; @@ -276,6 +292,9 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, switch (fetch_orientation) { case SQL_FETCH_NEXT: result_set.cursor++; + LOG(INFO) << "SQLFetchScrollInternal:: SQL_FETCH_NEXT advanced cursor " + "to " + << result_set.cursor; if (result_set.cursor >= result_set.rows.size() && handle.GetPagingInfo().page_token.empty()) { LOG(INFO) << "SQLFetch:: cursor: " << result_set.cursor @@ -300,9 +319,22 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, if (!rowset_size) { rowset_size = 1; } + LOG(INFO) << "SQLFetchScrollInternal:: rowset_size=" << rowset_size; DescriptorHandle& ird = handle.GetDescriptorHandle(DescriptorType::kIRD); + LOG(INFO) << "SQLFetchScrollInternal:: calling WriteRowset"; status_record = WriteRowset(result_set, rowset_size, ard, ird); + LOG(INFO) << "SQLFetchScrollInternal:: WriteRowset returned ok=" + << static_cast(status_record.ok()) + << ", message='" << status_record.message << "'"; return LogAndReturnCode(handle, status_record); + } catch (std::exception const& e) { + LOG(ERROR) << "SQLFetchScrollInternal:: std::exception caught: what='" + << e.what() << "'"; + return SQL_ERROR; + } catch (...) { + LOG(ERROR) << "SQLFetchScrollInternal:: unknown exception caught"; + return SQL_ERROR; + } } SQLRETURN SQLNumResultColsInternal(SQLHSTMT statement_handle, From 1b408ffa321c1d22c23af7907bd42869a32a35be Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 6 May 2026 13:14:19 +0530 Subject: [PATCH 05/32] cleaned logs --- .../bq_driver/internal/data_translation.cc | 4 +- .../odbc/bq_driver/internal/odbc_sql_fetch.cc | 390 ++++++------------ .../bq_driver/internal/odbc_sql_tables.cc | 113 +++-- .../bq_driver/internal/odbc_stmt_handle.cc | 20 - google/cloud/odbc/bq_driver/internal/utils.cc | 84 ++-- .../odbc/bq_driver/odbc_driver_metadata.cc | 28 +- .../cloud/odbc/bq_driver/odbc_sql_requests.cc | 71 +--- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 38 +- 8 files changed, 245 insertions(+), 503 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index c330015380..ce190a616f 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -2181,7 +2181,7 @@ void NormalizeDatetimeRange(std::string& src_str) { } } -namespace { + namespace { // Example: [2024-10-10, 2024-10-11) re2::RE2 const kDateRangeRegex(R"(\[\d{4}-\d{2}-\d{2}, \d{4}-\d{2}-\d{2}\))"); @@ -2209,7 +2209,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, auto status = ConvertRangeToTimestampFormat(src_str); if (!status.ok()) { return status; - } + } } switch (dest_data.type) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc index f31e28b873..69618d66f8 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_fetch.cc @@ -27,137 +27,79 @@ StatusRecord WriteToApplicationBuffer(DSValue const& ds_val, DescriptorRecord& app_desc_rec, SQLLEN bind_offset, SQLLEN bind_offset_ind) { - LOG(INFO) << "WriteToApplicationBuffer:: Start (bq_data_type=" - << static_cast(bq_data_type) - << ", target_c_type=" << app_desc_rec.concise_type - << ", bind_offset=" << bind_offset - << ", bind_offset_ind=" << bind_offset_ind - << ", data_ptr=" << static_cast(app_desc_rec.data_ptr) - << ", octet_length=" << app_desc_rec.octet_length - << ", ds_val.size()=" << ds_val.size() << ")"; - try { - SQLSMALLINT target_c_type = app_desc_rec.concise_type; - SQLPOINTER app_buffer = app_desc_rec.data_ptr; - SQLLEN app_buffer_len = app_desc_rec.octet_length; - SQLLEN* indicator_ptr = app_desc_rec.indicator_ptr; - SQLLEN* octet_length_ptr = app_desc_rec.octet_length_ptr; - - if (app_buffer == nullptr) { - LOG(ERROR) << "WriteToApplicationBuffer:: data_ptr is NULL; column not " - "bound? Skipping."; - return StatusRecord::Ok(); - } + SQLSMALLINT target_c_type = app_desc_rec.concise_type; + SQLPOINTER app_buffer = app_desc_rec.data_ptr; + SQLLEN app_buffer_len = app_desc_rec.octet_length; + SQLLEN* indicator_ptr = app_desc_rec.indicator_ptr; + SQLLEN* octet_length_ptr = app_desc_rec.octet_length_ptr; - app_buffer = reinterpret_cast(app_buffer) + bind_offset; - if (indicator_ptr) { - indicator_ptr = reinterpret_cast( - reinterpret_cast(indicator_ptr) + bind_offset_ind); - } - if (octet_length_ptr) { - octet_length_ptr = reinterpret_cast( - reinterpret_cast(octet_length_ptr) + bind_offset_ind); - } - LOG(INFO) << "WriteToApplicationBuffer:: pointers offset; " - "app_buffer (post-offset)=" << app_buffer - << ", indicator_ptr=" << static_cast(indicator_ptr) - << ", octet_length_ptr=" << static_cast(octet_length_ptr); + app_buffer = reinterpret_cast(app_buffer) + bind_offset; + if (indicator_ptr) { + indicator_ptr = reinterpret_cast( + reinterpret_cast(indicator_ptr) + bind_offset_ind); + } + if (octet_length_ptr) { + octet_length_ptr = reinterpret_cast( + reinterpret_cast(octet_length_ptr) + bind_offset_ind); + } - if (IsDSValueNull(ds_val)) { - LOG(INFO) << "WriteToApplicationBuffer:: ds_val is NULL"; - if (indicator_ptr == nullptr) { - LOG(ERROR) << "WriteToApplicationBuffer:: Indicator variable required " - "but not supplied for NULL data."; - return {SQLStates::k_22002(), - "Indicator variable required but not supplied"}; - } - *indicator_ptr = SQL_NULL_DATA; - LOG(INFO) << "WriteToApplicationBuffer:: wrote SQL_NULL_DATA to " - "indicator; returning"; - return StatusRecord::Ok(); - } - // We need to reset the indicator_ptr once it has been set to SQL_NULL_DATA - // for DSNullValues. - if (indicator_ptr) { - LOG(INFO) << "WriteToApplicationBuffer:: writing ds_val.size()=" - << ds_val.size() << " to indicator_ptr"; - *indicator_ptr = ds_val.size(); + if (IsDSValueNull(ds_val)) { + LOG(ERROR) << "WriteToApplicationBuffer:: Indicator variable required but " + "not supplied for NULL data."; + if (indicator_ptr == nullptr) { + return {SQLStates::k_22002(), + "Indicator variable required but not supplied"}; } + *indicator_ptr = SQL_NULL_DATA; + return StatusRecord::Ok(); + } + // We need to reset the indicator_ptr once it has been set to SQL_NULL_DATA + // for DSNullValues. + if (indicator_ptr) { + *indicator_ptr = ds_val.size(); + } - DataBuffer data = {target_c_type, app_buffer, app_buffer_len, - octet_length_ptr}; - LOG(INFO) << "WriteToApplicationBuffer:: dispatching by bq_data_type=" - << static_cast(bq_data_type); - StatusRecord status_record; - switch (bq_data_type) { - case BQDataType::kInt64: - status_record = ConvertFromArithmeticDSValue(ds_val, data); - break; - case BQDataType::kFloat64: - status_record = ConvertFromArithmeticDSValue(ds_val, data); - break; - case BQDataType::kString: - status_record = ConvertFromStringDSValue(ds_val, data); - break; - case BQDataType::kDate: - status_record = ConvertFromDateDSValue(ds_val, data); - break; - case BQDataType::kTime: - status_record = ConvertFromTimeDSValue(ds_val, data); - break; - case BQDataType::kJson: - status_record = ConvertFromJsonDSValue(ds_val, data); - break; - case BQDataType::kStruct: - status_record = ConvertFromStructDSValue(ds_val, data); - break; - case BQDataType::kArray: - status_record = ConvertFromArrayDSValue(ds_val, data); - break; - case BQDataType::kTimeStamp: - status_record = ConvertFromTimestampDSValue(ds_val, data); - break; - case BQDataType::kDatetime: - status_record = ConvertFromDatetimeDSValue(ds_val, data); - break; - case BQDataType::kInterval: - status_record = ConvertFromIntervalDSValue(ds_val, data); - break; - case BQDataType::kBool: - status_record = ConvertFromBooleanDSValue(ds_val, data); - break; - case BQDataType::kGeography: - status_record = ConvertFromGeographyDSValue(ds_val, data); - break; - case BQDataType::kBytes: - status_record = ConvertFromBytesDSValue(ds_val, data); - break; - case BQDataType::kRange: - status_record = ConvertFromRangeDSValue(ds_val, data); - break; - case BQDataType::kBigNumeric: - case BQDataType::kNumeric: - status_record = ConvertFromNumericDSValue(ds_val, data); - break; - default: - LOG(ERROR) << "WriteToApplicationBuffer:: Data type not supported: " - << bq_data_type; - return {SQLStates::k_HYC00(), "Data type not supported"}; - } - LOG(INFO) << "WriteToApplicationBuffer:: end ok=" - << static_cast(status_record.ok()) - << ", message='" << status_record.message << "'"; - return status_record; - } catch (std::exception const& e) { - LOG(ERROR) << "WriteToApplicationBuffer:: std::exception caught: what='" - << e.what() << "'"; - return StatusRecord{ - SQLStates::k_HY000(), - std::string("exception in WriteToApplicationBuffer: ") + e.what()}; - } catch (...) { - LOG(ERROR) << "WriteToApplicationBuffer:: unknown exception caught"; - return StatusRecord{SQLStates::k_HY000(), - "Unknown exception in WriteToApplicationBuffer"}; + DataBuffer data = {target_c_type, app_buffer, app_buffer_len, + octet_length_ptr}; + StatusRecord status_record; + switch (bq_data_type) { + case BQDataType::kInt64: + return ConvertFromArithmeticDSValue(ds_val, data); + case BQDataType::kFloat64: + return ConvertFromArithmeticDSValue(ds_val, data); + case BQDataType::kString: + return ConvertFromStringDSValue(ds_val, data); + case BQDataType::kDate: + return ConvertFromDateDSValue(ds_val, data); + case BQDataType::kTime: + return ConvertFromTimeDSValue(ds_val, data); + case BQDataType::kJson: + return ConvertFromJsonDSValue(ds_val, data); + case BQDataType::kStruct: + return ConvertFromStructDSValue(ds_val, data); + case BQDataType::kArray: + return ConvertFromArrayDSValue(ds_val, data); + case BQDataType::kTimeStamp: + return ConvertFromTimestampDSValue(ds_val, data); + case BQDataType::kDatetime: + return ConvertFromDatetimeDSValue(ds_val, data); + case BQDataType::kInterval: + return ConvertFromIntervalDSValue(ds_val, data); + case BQDataType::kBool: + return ConvertFromBooleanDSValue(ds_val, data); + case BQDataType::kGeography: + return ConvertFromGeographyDSValue(ds_val, data); + case BQDataType::kBytes: + return ConvertFromBytesDSValue(ds_val, data); + case BQDataType::kRange: + return ConvertFromRangeDSValue(ds_val, data); + case BQDataType::kBigNumeric: + case BQDataType::kNumeric: + return ConvertFromNumericDSValue(ds_val, data); } + LOG(ERROR) << "WriteToApplicationBuffer:: Data type not supported: " + << bq_data_type; + return {SQLStates::k_HYC00(), "Data type not supported"}; } // This is according to the spec: @@ -207,152 +149,92 @@ SQLLEN GetElemSize(DescriptorRecord& app_desc_rec) { StatusRecord WriteDSRow(DSRow const& ds_row, RowSchema const& schema, DescriptorHandle& ard, int row_num) { - LOG(INFO) << "WriteDSRow:: Start (row_num=" << row_num - << ", schema cols=" << schema.size() - << ", row vals=" << ds_row.size() << ")"; - try { - SQLLEN* bind_offset_ptr = ard.GetHeaderRecord().bind_offset_ptr; - SQLLEN bind_offset = 0; - if (bind_offset_ptr) { - bind_offset = *bind_offset_ptr; - } - LOG(INFO) << "WriteDSRow:: bind_offset_ptr=" - << static_cast(bind_offset_ptr) - << ", bind_offset=" << bind_offset; + SQLLEN* bind_offset_ptr = ard.GetHeaderRecord().bind_offset_ptr; + SQLLEN bind_offset = 0; + if (bind_offset_ptr) { + bind_offset = *bind_offset_ptr; + } - for (ColumnSchema const& col_schema : schema) { - int col_index = col_schema.col_index; - LOG(INFO) << "WriteDSRow:: col_index=" << col_index - << ", row size=" << ds_row.size(); - if (col_index < 0 || - static_cast(col_index) >= ds_row.size()) { - LOG(ERROR) << "WriteDSRow:: col_index out of range; skipping"; - continue; - } - DSValue const& ds_val = ds_row[col_index]; - // Column is not bound. - if (!ard.HasDescriptorRecord(col_index + 1)) { - LOG(INFO) << "WriteDSRow:: col_index=" << col_index - << " not bound; skipping"; - continue; - } - LOG(INFO) << "WriteDSRow:: col_index=" << col_index - << " is bound; getting descriptor"; - DescriptorRecord& col_desc = ard.GetDescriptorRecord(col_index + 1); + for (ColumnSchema const& col_schema : schema) { + int col_index = col_schema.col_index; + DSValue const& ds_val = ds_row[col_index]; + // Column is not bound. + if (!ard.HasDescriptorRecord(col_index + 1)) { + continue; + } + DescriptorRecord& col_desc = ard.GetDescriptorRecord(col_index + 1); - SQLLEN elem_size, elem_size_ind; - SQLINTEGER bind_type = ard.GetHeaderRecord().bind_type; - if (bind_type == SQL_BIND_BY_COLUMN) { - elem_size = GetElemSize(col_desc); - elem_size_ind = sizeof(SQLLEN); - } else { - elem_size = bind_type; - elem_size_ind = bind_type; - } - SQLLEN row_offset = row_num * elem_size; - SQLLEN row_offset_ind = row_num * elem_size_ind; - LOG(INFO) << "WriteDSRow:: col=" << col_index - << " elem_size=" << elem_size - << " bind_type=" << bind_type - << " row_offset=" << row_offset - << " row_offset_ind=" << row_offset_ind; + SQLLEN elem_size, elem_size_ind; + SQLINTEGER bind_type = ard.GetHeaderRecord().bind_type; + if (bind_type == SQL_BIND_BY_COLUMN) { + elem_size = GetElemSize(col_desc); + elem_size_ind = sizeof(SQLLEN); + } else { + elem_size = bind_type; + elem_size_ind = bind_type; + } + SQLLEN row_offset = row_num * elem_size; + SQLLEN row_offset_ind = row_num * elem_size_ind; - BQDataType bq_data_type = col_schema.col_type; - if (col_schema.is_mode_repeated) { - bq_data_type = BQDataType::kArray; - } + BQDataType bq_data_type = col_schema.col_type; + if (col_schema.is_mode_repeated) { + bq_data_type = BQDataType::kArray; + } - LOG(INFO) << "WriteDSRow:: col=" << col_index - << " calling WriteToApplicationBuffer"; - StatusRecord status_record = WriteToApplicationBuffer( - ds_val, bq_data_type, col_desc, bind_offset + row_offset, - bind_offset + row_offset_ind); - LOG(INFO) << "WriteDSRow:: col=" << col_index - << " WriteToApplicationBuffer returned ok=" - << static_cast(status_record.ok()); - if (!status_record.ok()) { - LOG(ERROR) << "WriteDSRow::WriteToApplicationBuffer:: " - << status_record.message; - return status_record; - } + StatusRecord status_record = WriteToApplicationBuffer( + ds_val, bq_data_type, col_desc, bind_offset + row_offset, + bind_offset + row_offset_ind); + if (!status_record.ok()) { + LOG(ERROR) << "WriteDSRow::WriteToApplicationBuffer:: " + << status_record.message; + return status_record; } - LOG(INFO) << "WriteDSRow:: end"; - return StatusRecord::Ok(); - } catch (std::exception const& e) { - LOG(ERROR) << "WriteDSRow:: std::exception caught: what='" << e.what() - << "'"; - return StatusRecord{SQLStates::k_HY000(), - std::string("exception in WriteDSRow: ") + e.what()}; - } catch (...) { - LOG(ERROR) << "WriteDSRow:: unknown exception caught"; - return StatusRecord{SQLStates::k_HY000(), - "Unknown exception in WriteDSRow"}; } + return StatusRecord::Ok(); } StatusRecord WriteRowset(ResultSet const& result_set, int const rowset_size, DescriptorHandle& ard, DescriptorHandle& ird) { - LOG(INFO) << "WriteRowset:: Start (rowset_size=" << rowset_size - << ", cursor=" << result_set.cursor - << ", rows.size()=" << result_set.rows.size() << ")"; - try { - if (rowset_size <= 0) { - LOG(ERROR) << "WriteRowset:: rowset_size should not be <= 0"; - StatusRecord status_record = {SQLStates::k_HY000(), - "rowset_size should not be <= 0"}; + if (rowset_size <= 0) { + LOG(ERROR) << "WriteRowset:: rowset_size should not be <= 0"; + StatusRecord status_record = {SQLStates::k_HY000(), + "rowset_size should not be <= 0"}; + return status_record; + } + int cursor = result_set.cursor; + int row_counter = 0; + SQLUSMALLINT* row_status_ptr = ird.GetHeaderRecord().array_status_ptr; + // We write 'rowset_size' rows from result_set.rows starting at the index + // 'cursor' + for (int i = cursor; i < cursor + rowset_size && i < result_set.rows.size(); + i++, row_counter++) { + StatusRecord status_record = + WriteDSRow(result_set.rows[i], result_set.row_schema, ard, i - cursor); + if (!status_record.ok()) { + LOG(ERROR) << "WriteRowset::WriteDSRow:: " << status_record.message; return status_record; } - int cursor = result_set.cursor; - int row_counter = 0; - SQLUSMALLINT* row_status_ptr = ird.GetHeaderRecord().array_status_ptr; - LOG(INFO) << "WriteRowset:: row_status_ptr=" - << static_cast(row_status_ptr); - // We write 'rowset_size' rows from result_set.rows starting at the index - // 'cursor' - for (int i = cursor; i < cursor + rowset_size && i < result_set.rows.size(); - i++, row_counter++) { - LOG(INFO) << "WriteRowset:: writing row " << i << " (offset=" - << (i - cursor) << ")"; - StatusRecord status_record = WriteDSRow( - result_set.rows[i], result_set.row_schema, ard, i - cursor); - LOG(INFO) << "WriteRowset:: WriteDSRow row " << i << " returned ok=" - << static_cast(status_record.ok()); - if (!status_record.ok()) { - LOG(ERROR) << "WriteRowset::WriteDSRow:: " << status_record.message; - return status_record; - } - if (row_status_ptr) { - row_status_ptr[i - cursor] = SQL_ROW_SUCCESS; - } - - result_set.cursor = i; - } - LOG(INFO) << "WriteRowset:: wrote " << row_counter << " rows"; - - // Mark unused rows if (row_status_ptr) { - for (int i = row_counter; i < rowset_size; i++) { - row_status_ptr[i] = SQL_ROW_NOROW; - } + row_status_ptr[i - cursor] = SQL_ROW_SUCCESS; } - SQLULEN* rows_processed_ptr = ird.GetHeaderRecord().rows_processed_ptr; - if (rows_processed_ptr) { - *rows_processed_ptr = row_counter; + result_set.cursor = i; + } + + // Mark unused rows + if (row_status_ptr) { + for (int i = row_counter; i < rowset_size; i++) { + row_status_ptr[i] = SQL_ROW_NOROW; } - LOG(INFO) << "WriteRowset:: end"; - return StatusRecord::Ok(); - } catch (std::exception const& e) { - LOG(ERROR) << "WriteRowset:: std::exception caught: what='" << e.what() - << "'"; - return StatusRecord{SQLStates::k_HY000(), - std::string("exception in WriteRowset: ") + e.what()}; - } catch (...) { - LOG(ERROR) << "WriteRowset:: unknown exception caught"; - return StatusRecord{SQLStates::k_HY000(), - "Unknown exception in WriteRowset"}; } + + SQLULEN* rows_processed_ptr = ird.GetHeaderRecord().rows_processed_ptr; + if (rows_processed_ptr) { + *rows_processed_ptr = row_counter; + } + + return StatusRecord::Ok(); } StatusRecord FetchNextResultSet(StatementHandle& stmt_handle) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 8214a2ccb5..247c5781b9 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -83,32 +83,59 @@ StatusRecordOr> GetFilteredProjectIds( ODBCBQClient& bq_client, std::string const& projects_filter, SQLULEN metadata_id) { LOG(INFO) << "GetFilteredProjectIds:: Start (filter='" << projects_filter - << "', metadata_id=" << metadata_id << ")"; - std::vector project_ids; - auto filter_regex = BuildRegex(projects_filter, metadata_id); - // For now, we use default options. - // We can set timeout here as needed later. - Options options; - LOG(INFO) << "GetFilteredProjectIds:: calling ListAllProjects (filter='" - << projects_filter << "', metadata_id=" << metadata_id << ")"; - StatusRecordOr> projects = - bq_client.ListAllProjects(options); - if (!projects) { - LOG(ERROR) << "GetFilteredProjectIds::ListAllProjects:: " - << projects.GetStatusRecord().message; - return projects.GetStatusRecord(); - } - LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned " - << projects->size() << " projects; applying filter"; - for (auto const& project : *projects) { - if ((!metadata_id && projects_filter == "%") || - re2::RE2::FullMatch(project.id, *filter_regex)) { - project_ids.push_back(project.id); + << "', filter.size()=" << projects_filter.size() + << ", metadata_id=" << metadata_id << ")"; + // Wrap the entire body in try/catch so any thrown exception (regex_error, + // bad_alloc, anything from the BigQuery client) becomes a logged error + // instead of an unhandled exception that abort()s the driver process. + try { + LOG(INFO) << "GetFilteredProjectIds:: declaring project_ids vector"; + std::vector project_ids; + LOG(INFO) << "GetFilteredProjectIds:: about to call BuildRegex"; + std::regex filter_regex = BuildRegex(projects_filter, metadata_id); + LOG(INFO) << "GetFilteredProjectIds:: BuildRegex returned ok"; + // For now, we use default options. + // We can set timeout here as needed later. + LOG(INFO) << "GetFilteredProjectIds:: constructing Options"; + Options options; + LOG(INFO) << "GetFilteredProjectIds:: Options constructed"; + LOG(INFO) << "GetFilteredProjectIds:: calling ListAllProjects (filter='" + << projects_filter << "', metadata_id=" << metadata_id << ")"; + StatusRecordOr> projects = + bq_client.ListAllProjects(options); + LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned, ok=" + << static_cast(static_cast(projects)); + if (!projects) { + LOG(ERROR) << "GetFilteredProjectIds::ListAllProjects:: " + << projects.GetStatusRecord().message; + return projects.GetStatusRecord(); } + LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned " + << projects->size() << " projects; applying filter"; + for (auto const& project : *projects) { + if ((!metadata_id && projects_filter == "%") || + std::regex_match(project.id, filter_regex)) { + project_ids.push_back(project.id); + } + } + LOG(INFO) << "GetFilteredProjectIds:: kept " << project_ids.size() + << " projects after filter"; + return project_ids; + } catch (std::regex_error const& e) { + LOG(ERROR) << "GetFilteredProjectIds:: std::regex_error caught: code=" + << static_cast(e.code()) << " what='" << e.what() << "'"; + return StatusRecord{SQLStates::k_HY000(), + std::string("regex_error: ") + e.what()}; + } catch (std::exception const& e) { + LOG(ERROR) << "GetFilteredProjectIds:: std::exception caught: what='" + << e.what() << "'"; + return StatusRecord{SQLStates::k_HY000(), + std::string("exception: ") + e.what()}; + } catch (...) { + LOG(ERROR) << "GetFilteredProjectIds:: unknown exception caught"; + return StatusRecord{SQLStates::k_HY000(), + "Unknown exception in GetFilteredProjectIds"}; } - LOG(INFO) << "GetFilteredProjectIds:: kept " << project_ids.size() - << " projects after filter"; - return project_ids; } StatusRecordOr> GetFilteredDatasetIds( @@ -356,43 +383,22 @@ StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, std::string const& additional_projects, std::string const& connection_catalog) { - LOG(INFO) << "GetResultSetForProjects:: Start (metadata_id=" << metadata_id - << ", additional_projects='" << additional_projects - << "', connection_catalog='" << connection_catalog << "')"; - // Fast path: when the connection has a catalog bound via the DSN, skip the // BigQuery ListAllProjects round-trip. Returning [catalog] + - // additional_projects is what HANA SDA's catalog-discovery probe actually - // needs, and it bypasses the failure in ListAllProjects that surfaces as - // "Cannot get remote source objects" when SDA calls CHECK_REMOTE_SOURCE. + // additional_projects is what HANA SDA's catalog-discovery probe needs, + // and avoiding ListAllProjects sidesteps a failure mode that has surfaced + // as "Cannot get remote source objects" under SAP HANA SDA. if (!connection_catalog.empty()) { - LOG(INFO) << "GetResultSetForProjects:: fast-path entry; using catalog '" - << connection_catalog << "'"; std::vector project_list = {connection_catalog}; - LOG(INFO) << "GetResultSetForProjects:: fast-path; project_list.size()=" - << project_list.size(); if (!additional_projects.empty()) { - LOG(INFO) << "GetResultSetForProjects:: fast-path; appending additional"; project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), additional_projects); - LOG(INFO) - << "GetResultSetForProjects:: fast-path; after AppendAdditional: " - << project_list.size(); } - LOG(INFO) << "GetResultSetForProjects:: fast-path; building result set"; - auto rs = CreateResultSetForProjects(project_list); - LOG(INFO) << "GetResultSetForProjects:: fast-path; end (" - << project_list.size() << " projects)"; - return rs; + return CreateResultSetForProjects(project_list); } - LOG(INFO) << "GetResultSetForProjects:: slow-path; calling " - "GetFilteredProjectIds"; auto project_ids_status = GetFilteredProjectIds(bq_client, kMatchAll, metadata_id); - LOG(INFO) << "GetResultSetForProjects:: slow-path; GetFilteredProjectIds " - "returned, ok=" - << static_cast(static_cast(project_ids_status)); if (!project_ids_status) { LOG(ERROR) << "GetResultSetForProjects::GetFilteredProjectIds:: " << project_ids_status.GetStatusRecord().message; @@ -400,19 +406,12 @@ StatusRecordOr GetResultSetForProjects( } std::vector project_list = *project_ids_status; - LOG(INFO) << "GetResultSetForProjects:: slow-path; project_list.size()=" - << project_list.size(); if (!additional_projects.empty()) { project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), additional_projects); - LOG(INFO) << "GetResultSetForProjects:: slow-path; after AppendAdditional: " - << project_list.size(); } - LOG(INFO) << "GetResultSetForProjects:: slow-path; building result set"; - auto rs = CreateResultSetForProjects(project_list); - LOG(INFO) << "GetResultSetForProjects:: slow-path; end"; - return rs; + return CreateResultSetForProjects(project_list); } StatusRecordOr GetResultSetForDatasets( diff --git a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc index 1a7c79117c..701ba5fade 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc @@ -205,10 +205,7 @@ bool IsSelectQuery(std::string const& q) { // TODO(b/342044533) Sanitize query text to avoid potential SQL Injection // risk. StatusRecord StatementHandle::PrepareQuery(std::string const& query) { - LOG(INFO) << "PrepareQuery:: Start (query.length()=" << query.length() << ")"; StatusRecord transaction_status = BeginTransactionIfNeeded(*conn_handle_); - LOG(INFO) << "PrepareQuery:: BeginTransactionIfNeeded ok=" - << static_cast(transaction_status.ok()); if (!transaction_status.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::BeginTransactionIfNeeded:: " << transaction_status.message; @@ -216,7 +213,6 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { } ConnectionHandle& conn_handle = *GetConnectionHandle(); - LOG(INFO) << "PrepareQuery:: building Job request"; Job req; req.configuration.query.query = query; req.configuration.query.use_query_cache = conn_handle.GetDsn().is_query_cache; @@ -257,8 +253,6 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { if (has_named) { req.configuration.query.parameter_mode = "NAMED"; } - LOG(INFO) << "PrepareQuery:: positional=" << has_positional - << ", named=" << has_named; } std::vector combined_properties = @@ -275,29 +269,21 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { req.configuration.query.connection_properties = combined_properties; Options opt; opt.set(conn_handle.GetDsn().max_retries); - LOG(INFO) << "PrepareQuery:: calling InsertJob (catalog='" - << conn_handle.GetDsn().catalog << "', dry_run=true)"; auto response = conn_handle.GetClient()->InsertJob( conn_handle.GetDsn().catalog, req, opt); - LOG(INFO) << "PrepareQuery:: InsertJob returned ok=" - << static_cast(response.Ok()); if (!response.Ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::InsertJob:: " << response.GetStatusRecord().message; return response.GetStatusRecord(); } - LOG(INFO) << "PrepareQuery:: calling PopulateResultSet"; auto& schema = response.GetValue().statistics.job_query_stats.schema; auto pop_response = PopulateResultSet(schema); - LOG(INFO) << "PrepareQuery:: PopulateResultSet returned ok=" - << static_cast(pop_response.ok()); if (!pop_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateResultSet:: " << pop_response.message; return pop_response; } - LOG(INFO) << "PrepareQuery:: SetQueryParameters"; SetQueryParameters( response.GetValue() .statistics.job_query_stats.undeclared_query_parameters); @@ -318,28 +304,22 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { table_fields = response.GetValue().configuration.query.destination_table; } - LOG(INFO) << "PrepareQuery:: building IRD"; DescriptorHandle& desc_handle = this->GetDescriptorHandle(DescriptorType::kIRD); desc_handle.SetConnectionHandle(&conn_handle); desc_handle.ClearDescriptorRecordsMap(); StatusRecord ird_response = PopulateIrd(desc_handle, schema, table_fields); - LOG(INFO) << "PrepareQuery:: PopulateIrd returned ok=" - << static_cast(ird_response.ok()); if (!ird_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateIrd:: " << ird_response.message; return ird_response; } - LOG(INFO) << "PrepareQuery:: building IPD"; DescriptorHandle& ipd_desc_handle = this->GetDescriptorHandle(DescriptorType::kIPD); ipd_desc_handle.ClearDescriptorRecordsMap(); auto job_statistics = (*response).statistics; StatusRecord ipd_response = PopulateIpd(ipd_desc_handle, job_statistics); - LOG(INFO) << "PrepareQuery:: PopulateIpd returned ok=" - << static_cast(ipd_response.ok()); if (!ipd_response.ok()) { LOG(ERROR) << "StatementHandle::PrepareQuery::PopulateIpd:: " << ipd_response.message; diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index e4fb83a3f9..85cf5b9045 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -20,7 +20,6 @@ #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" #include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/internal/getenv.h" -#include #include #include #include @@ -878,20 +877,12 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( g_utf16le_wire_latched.load(std::memory_order_relaxed); auto const* bytes = reinterpret_cast(in_str); - LOG(INFO) << "BqConvertSQLWCHARToString:: sizeof(SQLWCHAR)=4; " - << "first 4 bytes = " - << absl::StrFormat("%02X %02X %02X %02X", bytes[0], bytes[1], - bytes[2], bytes[3]) - << "; in_str_len=" << in_str_len - << "; latched=" << (use_utf16le ? "yes" : "no"); - if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { use_utf16le = true; g_utf16le_wire_latched.store(true, std::memory_order_relaxed); LOG(INFO) << "BqConvertSQLWCHARToString:: latched UTF-16LE wire format " - "for the rest of this process (unixODBC under iODBC-built " - "driver)"; + "for this process (unixODBC under iODBC-built driver)"; } if (use_utf16le) { @@ -911,9 +902,6 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( } return Utf16ToUtf8(wstr); } - LOG(INFO) << "BqConvertSQLWCHARToString:: byte pattern does not match " - "UTF-16LE and latch unset; falling through to default 4-byte " - "SQLWCHAR path"; } #endif @@ -1044,54 +1032,36 @@ bool IsInfoTypeString(SQLUSMALLINT InfoType) { // '\X' -> literal X (the backslash is the escape; consumed in output) // any other char -> emitted as-is // -// History: this used to be three chained `std::regex_replace` calls. That was -// fragile because it depended on the platform's back-end being able -// to compile small patterns like `^%|([^\\])%`. On hosts with stripped -// locale data (notably the SAP HANA SDA dpserver process when launched by -// adm with a minimal LC_* env), one of those `std::regex` constructions -// throws — and the exception unwinds across the ODBC ABI boundary, aborting -// the driver process with no error to the caller. We replace it with a -// manual single-pass loop that has no regex dependency, no allocations -// beyond the result string, and no failure modes other than std::bad_alloc -// (which is also caught below for safety). +// Implemented as a manual single-pass loop rather than chained +// std::regex_replace calls: on some host libstdc++/libc++ builds the +// regex DFA initialization could throw and unwind across the ODBC ABI +// boundary, aborting the driver. The manual loop has no +// dependency. std::string CastOdbcRegexToCppRegex(std::string const& str) { - LOG(INFO) << "CastOdbcRegexToCppRegex:: Start (input='" << str - << "', length=" << str.size() << ")"; - try { - std::string result; - // Worst case: every character becomes ".*" (2 chars). Reserve up front. - result.reserve(str.size() * 2); - for (size_t i = 0; i < str.size(); ++i) { - char c = str[i]; - if (c == '\\') { - if (i + 1 < str.size()) { - // Escape: emit the next char literally, consume both the - // backslash and the following char from the input. - result.push_back(str[i + 1]); - ++i; - } - // else: lone trailing backslash — drop it. The previous - // regex-based implementation removed all stray backslashes at the - // end of processing, so this matches that behavior. - } else if (c == '%') { - result.append(".*"); - } else if (c == '_') { - result.push_back('.'); - } else { - result.push_back(c); + std::string result; + // Worst case: every character becomes ".*" (2 chars). + result.reserve(str.size() * 2); + for (size_t i = 0; i < str.size(); ++i) { + char c = str[i]; + if (c == '\\') { + if (i + 1 < str.size()) { + // Escape: emit the next char literally, consume both the + // backslash and the following char from the input. + result.push_back(str[i + 1]); + ++i; } + // else: lone trailing backslash — drop it (matches the previous + // regex-based implementation, which stripped all stray backslashes + // at the end of processing). + } else if (c == '%') { + result.append(".*"); + } else if (c == '_') { + result.push_back('.'); + } else { + result.push_back(c); } - LOG(INFO) << "CastOdbcRegexToCppRegex:: end; result='" << result - << "' (length=" << result.size() << ")"; - return result; - } catch (std::exception const& e) { - LOG(ERROR) << "CastOdbcRegexToCppRegex:: std::exception caught: what='" - << e.what() << "'"; - throw; - } catch (...) { - LOG(ERROR) << "CastOdbcRegexToCppRegex:: unknown exception caught"; - throw; } + return result; } // Translate an ODBC LIKE pattern into a C++ regex pattern. diff --git a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc index 0abe7decad..dd6ffd6c05 100644 --- a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc +++ b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc @@ -423,7 +423,6 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, return handle_result.GetCalculatedReturnCode(); } StatementHandle& handle = *(*handle_result); - LOG(INFO) << "SQLTablesInternal:: validated stmt handle"; StatusRecordOr attr_status = handle.GetAttribute(SQL_ATTR_METADATA_ID); @@ -433,7 +432,6 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, return LogAndReturnCode(handle, attr_status); } SQLULEN metadata_id = *attr_status; - LOG(INFO) << "SQLTablesInternal:: metadata_id=" << metadata_id; auto input_param_status = ValidateInputParameters( catalog_name, catalog_name_len, schema_name, schema_name_len, table_name, @@ -443,15 +441,11 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, << input_param_status.message; return LogAndReturnCode(handle, input_param_status); } - LOG(INFO) << "SQLTablesInternal:: validated input parameters"; std::string project_filter = ToCharStr(catalog_name, kMatchAll); std::string dataset_filter = ToCharStr(schema_name, kMatchAll); std::string table_filter = ToCharStr(table_name, kMatchAll); std::string table_type_filter = ToCharStr(table_type, kMatchAll); - LOG(INFO) << "SQLTablesInternal:: filters: project='" << project_filter - << "' dataset='" << dataset_filter << "' table='" << table_filter - << "' table_type='" << table_type_filter << "'"; if (handle.GetConnectionHandle() == nullptr) { LOG(ERROR) << "SQLTables:: Internal connection handle is null"; @@ -464,8 +458,6 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, auto const dsn = conn_handle.GetDsn(); if (dsn.filter_tables_on_default_dataset && !dsn.default_dataset.empty()) { dataset_filter = dsn.default_dataset; - LOG(INFO) << "SQLTablesInternal:: defaulted dataset_filter to '" - << dataset_filter << "'"; } } if (!conn_handle.IsConnected()) { @@ -474,7 +466,6 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, handle, StatusRecord{SQLStates::k_08S01(), "Connection to the data source is broken"}); } - LOG(INFO) << "SQLTablesInternal:: connection healthy; getting bq client"; std::shared_ptr bq_client_ptr = conn_handle.GetClient(); if (!bq_client_ptr) { LOG(ERROR) << "SQLTables:: Error establishing Datasource connection"; @@ -487,33 +478,25 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, if (!metadata_id && project_filter == SQL_ALL_CATALOGS && dataset_filter.empty() && table_filter.empty()) { - auto const& dsn = conn_handle.GetDsn(); - LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForProjects " - "(dsn.catalog='" - << dsn.catalog << "', additional_projects='" - << dsn.additional_projects << "')"; // Pass dsn.catalog as the 4th arg so GetResultSetForProjects can take - // the fast path and avoid ListAllProjects (which crashes on HANA SDA). + // the fast path and avoid the BigQuery ListAllProjects round-trip; + // SAP HANA SDA's CHECK_REMOTE_SOURCE catalog probe lands here. + auto const& dsn = conn_handle.GetDsn(); result_set_status = GetResultSetForProjects( bq_client, metadata_id, dsn.additional_projects, dsn.catalog); } else if (!metadata_id && project_filter.empty() && dataset_filter == SQL_ALL_SCHEMAS && table_filter.empty()) { - LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForDatasets"; result_set_status = GetResultSetForDatasets(bq_client, metadata_id, kMatchAll, conn_handle.GetDsn().additional_projects); } else if (!metadata_id && project_filter.empty() && dataset_filter.empty() && table_filter.empty() && table_type_filter == SQL_ALL_TABLE_TYPES) { - LOG(INFO) << "SQLTablesInternal:: branch=CreateResultSetForTableTypes"; result_set_status = CreateResultSetForTableTypes(); } else { - LOG(INFO) << "SQLTablesInternal:: branch=GetResultSetForTables (network)"; result_set_status = GetResultSetForTables(handle, bq_client, project_filter, dataset_filter, table_filter, table_type_filter, metadata_id); } - LOG(INFO) << "SQLTablesInternal:: branch returned, ok=" - << static_cast(static_cast(result_set_status)); if (!result_set_status) { LOG(ERROR) << "SQLTables::ResultSet:: " << result_set_status.GetStatusRecord().message; @@ -529,15 +512,12 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, SQLULEN max_rows = *max_rows_status; ResultSet& result_set = *result_set_status; auto& rs_rows = result_set.rows; - LOG(INFO) << "SQLTablesInternal:: result_set rows=" << rs_rows.size() - << ", max_rows=" << max_rows; if (max_rows > 0 && max_rows < rs_rows.size()) { rs_rows.erase(rs_rows.begin() + max_rows, rs_rows.end()); } DescriptorHandle& ird = handle.GetDescriptorHandle(DescriptorType::kIRD); ird.SetConnectionHandle(&conn_handle); - LOG(INFO) << "SQLTablesInternal:: building table schema"; auto table_schema = BuildTableSchemaFromRowSchema(result_set.row_schema, kSchema); if (!table_schema) { @@ -547,14 +527,12 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, } TableReference table_fields; - LOG(INFO) << "SQLTablesInternal:: populating IRD"; auto ird_status = StatementHandle::PopulateIrd(ird, *table_schema, table_fields, true); if (!ird_status.ok()) { LOG(ERROR) << "SQLTables::PopulateIrd:: " << ird_status.message; return LogAndReturnCode(handle, ird_status); } - LOG(INFO) << "SQLTablesInternal:: IRD populated"; LOG(INFO) << "SQLTablesInternal:: end"; handle.SetResultSet(result_set); handle.SetStmtState(StmtStates::kStatementExecutedWithRs); diff --git a/google/cloud/odbc/bq_driver/odbc_sql_requests.cc b/google/cloud/odbc/bq_driver/odbc_sql_requests.cc index 7761d4c82a..3c9db5a102 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_requests.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_requests.cc @@ -356,48 +356,23 @@ StatusRecord ActuallyProcessExecute(StatementHandle& stmt_handle, // This function synchronously processes current SQLExecDirect requests. StatusRecord ActuallyProcessExecDirect(StatementHandle& stmt_handle) { - LOG(INFO) << "ActuallyProcessExecDirect:: Start"; - try { - stmt_handle.SetStmtState(StmtStates::kStatementStillExecuting); - LOG(INFO) << "ActuallyProcessExecDirect:: state set to StillExecuting"; - - std::string query_str = stmt_handle.GetQueryString(); - LOG(INFO) << "ActuallyProcessExecDirect:: query_str.length()=" - << query_str.length() - << ", first 200 chars='" << query_str.substr(0, 200) << "'"; - // We need to call `PrepareQuery` because: - // 1) We need to get `statement_type` during `ActuallyProcessExecute` - // through - // `Job::statistics.job_query_stats.statement_type`. This is not possible - // through `PostQueryResults` - // 2) For positional params, we need to get `QueryParameter`s before - // SQLExecDirect is called. - LOG(INFO) << "ActuallyProcessExecDirect:: calling PrepareQuery"; - StatusRecord prepare_status = stmt_handle.PrepareQuery(query_str); - LOG(INFO) << "ActuallyProcessExecDirect:: PrepareQuery returned, ok=" - << static_cast(prepare_status.ok()); - if (!prepare_status.ok()) { - LOG(ERROR) << "ActuallyProcessExecDirect::PrepareQuery:: " - << prepare_status.message; - return prepare_status; - } - LOG(INFO) << "ActuallyProcessExecDirect:: calling ActuallyProcessExecute"; - auto rs = ActuallyProcessExecute(stmt_handle, - StmtStates::kStatementNotPrepared); - LOG(INFO) << "ActuallyProcessExecDirect:: ActuallyProcessExecute returned, " - "ok=" - << static_cast(rs.ok()); - return rs; - } catch (std::exception const& e) { - LOG(ERROR) << "ActuallyProcessExecDirect:: std::exception caught: what='" - << e.what() << "'"; - return StatusRecord{SQLStates::k_HY000(), - std::string("exception: ") + e.what()}; - } catch (...) { - LOG(ERROR) << "ActuallyProcessExecDirect:: unknown exception caught"; - return StatusRecord{SQLStates::k_HY000(), - "Unknown exception in ActuallyProcessExecDirect"}; - } + stmt_handle.SetStmtState(StmtStates::kStatementStillExecuting); + + std::string query_str = stmt_handle.GetQueryString(); + // We need to call `PrepareQuery` because: + // 1) We need to get `statement_type` during `ActuallyProcessExecute` + // through + // `Job::statistics.job_query_stats.statement_type`. This is not possible + // through `PostQueryResults` + // 2) For positional params, we need to get `QueryParameter`s before + // SQLExecDirect is called. + StatusRecord prepare_status = stmt_handle.PrepareQuery(query_str); + if (!prepare_status.ok()) { + LOG(ERROR) << "ActuallyProcessExecDirect::PrepareQuery:: " + << prepare_status.message; + return prepare_status; + } + return ActuallyProcessExecute(stmt_handle, StmtStates::kStatementNotPrepared); } SQLRETURN HandleAsyncGetResults(StatementHandle& stmt_handle, @@ -1059,8 +1034,7 @@ SQLRETURN SQLExecuteInternal(SQLHSTMT statement_handle) { SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, SQLCHAR* in_statement_text, SQLINTEGER in_text_length) { - LOG(INFO) << "SQLExecDirectInternal:: Start (in_text_length=" - << in_text_length << ")"; + LOG(INFO) << "SQLExecDirectInternal:: Start"; StatusRecordOr handle_result = ValidateStatementHandle(statement_handle); if (!handle_result) { @@ -1069,7 +1043,6 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, return handle_result.GetCalculatedReturnCode(); } StatementHandle& stmt_handle = *(*handle_result); - LOG(INFO) << "SQLExecDirectInternal:: stmt handle validated"; if ((in_text_length < 1) && (in_text_length != SQL_NTS)) { StatusRecord status_record = {SQLStates::k_HY090(), "Invalid query length"}; @@ -1092,8 +1065,6 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, } std::string query_str = ToCharStr(in_statement_text); - LOG(INFO) << "SQLExecDirectInternal:: query_str.length()=" - << query_str.length(); if (query_str.empty()) { auto status_record = StatusRecord{SQLStates::k_HY000(), "Query text is null or empty"}; @@ -1101,7 +1072,6 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, return LogAndReturnCode(stmt_handle, status_record); } stmt_handle.SetQueryString(query_str); - LOG(INFO) << "SQLExecDirectInternal:: query string stored on handle"; // Boolean to tell us if a ExecDirect Query was to be processed async and it // wasn't finished last time. We are not using a `StmtStates` here because @@ -1207,12 +1177,7 @@ SQLRETURN SQLExecDirectInternal(SQLHSTMT statement_handle, // ***************************************************************** // STEP 4: Synchronous execution // ***************************************************************** - LOG(INFO) << "SQLExecDirectInternal:: synchronous path; calling " - "ActuallyProcessExecDirect"; StatusRecord execute_status = ActuallyProcessExecDirect(stmt_handle); - LOG(INFO) << "SQLExecDirectInternal:: ActuallyProcessExecDirect returned, " - "ok=" - << static_cast(execute_status.ok()); if (execute_status.sql_state == SQLStates::k_SQL_NEED_DATA()) { stmt_handle.SetStmtState(StmtStates::kNeedsParams); return SQL_NEED_DATA; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index a1ae93b846..57bc814b9e 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -220,9 +220,7 @@ SQLRETURN SQLFetchInternal(SQLHSTMT statement_handle) { SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, SQLSMALLINT fetch_orientation, SQLLEN /*fetch_offset*/) { - LOG(INFO) << "SQLFetchScrollInternal:: Start (fetch_orientation=" - << fetch_orientation << ")"; - try { + LOG(INFO) << "SQLFetchScrollInternal:: Start"; StatusRecordOr handle_result = ValidateStatementHandle(statement_handle); StatusRecord status_record; @@ -232,18 +230,14 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, return handle_result.GetCalculatedReturnCode(); } StatementHandle& handle = *(*handle_result); - LOG(INFO) << "SQLFetchScrollInternal:: stmt handle validated"; - auto stmt_state = handle.GetStmtState(); - LOG(INFO) << "SQLFetchScrollInternal:: stmt_state=" - << static_cast(stmt_state); - if (stmt_state == StmtStates::kStatementExecutedWithoutRs) { + if (handle.GetStmtState() == StmtStates::kStatementExecutedWithoutRs) { status_record = StatusRecord{SQLStates::k_24000(), "Invalid cursor state."}; LOG(ERROR) << "SQLFetchScroll:: " << status_record.message; return LogAndReturnCode(handle, status_record); } - if (stmt_state != StmtStates::kStatementExecutedWithRs) { + if (handle.GetStmtState() != StmtStates::kStatementExecutedWithRs) { status_record = {SQLStates::k_HY010(), "No statement has been executed with result set"}; LOG(ERROR) << "SQLFetchScroll:: " << status_record.message; @@ -263,23 +257,13 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, << status_record.message; return LogAndReturnCode(handle, status_record); } - LOG(INFO) << "SQLFetchScrollInternal:: fetch_orientation validated"; - LOG(INFO) << "SQLFetchScrollInternal:: getting ARD"; DescriptorHandle& ard = handle.GetDescriptorHandle(DescriptorType::kARD); - LOG(INFO) << "SQLFetchScrollInternal:: getting result set"; ResultSet& result_set = handle.GetResultSet(); - LOG(INFO) << "SQLFetchScrollInternal:: cursor=" << result_set.cursor - << ", rows.size()=" << result_set.rows.size() - << ", page_token.empty()=" - << handle.GetPagingInfo().page_token.empty(); if (result_set.cursor >= result_set.rows.size() - 1 && !handle.GetPagingInfo().page_token.empty()) { - LOG(INFO) << "SQLFetchScrollInternal:: calling FetchNextResultSet"; StatusRecord status_record = FetchNextResultSet(handle); - LOG(INFO) << "SQLFetchScrollInternal:: FetchNextResultSet returned ok=" - << static_cast(status_record.ok()); if (!status_record.ok()) { LOG(ERROR) << "SQLFetchScroll::FetchNextResultSet:: " << status_record.message; @@ -292,9 +276,6 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, switch (fetch_orientation) { case SQL_FETCH_NEXT: result_set.cursor++; - LOG(INFO) << "SQLFetchScrollInternal:: SQL_FETCH_NEXT advanced cursor " - "to " - << result_set.cursor; if (result_set.cursor >= result_set.rows.size() && handle.GetPagingInfo().page_token.empty()) { LOG(INFO) << "SQLFetch:: cursor: " << result_set.cursor @@ -319,22 +300,9 @@ SQLRETURN SQLFetchScrollInternal(SQLHSTMT statement_handle, if (!rowset_size) { rowset_size = 1; } - LOG(INFO) << "SQLFetchScrollInternal:: rowset_size=" << rowset_size; DescriptorHandle& ird = handle.GetDescriptorHandle(DescriptorType::kIRD); - LOG(INFO) << "SQLFetchScrollInternal:: calling WriteRowset"; status_record = WriteRowset(result_set, rowset_size, ard, ird); - LOG(INFO) << "SQLFetchScrollInternal:: WriteRowset returned ok=" - << static_cast(status_record.ok()) - << ", message='" << status_record.message << "'"; return LogAndReturnCode(handle, status_record); - } catch (std::exception const& e) { - LOG(ERROR) << "SQLFetchScrollInternal:: std::exception caught: what='" - << e.what() << "'"; - return SQL_ERROR; - } catch (...) { - LOG(ERROR) << "SQLFetchScrollInternal:: unknown exception caught"; - return SQL_ERROR; - } } SQLRETURN SQLNumResultColsInternal(SQLHSTMT statement_handle, From a619f117cd625bcc6fe945b6b5733843477d060e Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 6 May 2026 13:16:51 +0530 Subject: [PATCH 06/32] removed redundant code --- .../odbc/bq_driver/internal/odbc_sql_tables.cc | 17 +---------------- .../odbc/bq_driver/internal/odbc_sql_tables.h | 12 +----------- .../odbc/bq_driver/odbc_driver_metadata.cc | 8 ++------ 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 247c5781b9..28d64f59e9 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -381,22 +381,7 @@ ResultSet ProcessStringResults( StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, - std::string const& additional_projects, - std::string const& connection_catalog) { - // Fast path: when the connection has a catalog bound via the DSN, skip the - // BigQuery ListAllProjects round-trip. Returning [catalog] + - // additional_projects is what HANA SDA's catalog-discovery probe needs, - // and avoiding ListAllProjects sidesteps a failure mode that has surfaced - // as "Cannot get remote source objects" under SAP HANA SDA. - if (!connection_catalog.empty()) { - std::vector project_list = {connection_catalog}; - if (!additional_projects.empty()) { - project_list = AppendAdditionalProjectsIfMissing(std::move(project_list), - additional_projects); - } - return CreateResultSetForProjects(project_list); - } - + std::string const& additional_projects) { auto project_ids_status = GetFilteredProjectIds(bq_client, kMatchAll, metadata_id); if (!project_ids_status) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h index ab167bd97c..ae0a45e756 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.h @@ -106,19 +106,9 @@ ResultSet ProcessStringResults( std::vector> const& rows); // Search for all projects and populate ResultSet for it. -// -// `connection_catalog` (when non-empty) is the catalog the connection is bound -// to via the DSN's `Catalog=` field. If supplied, we skip the BigQuery -// `ListAllProjects` round-trip and return `[connection_catalog] + -// additional_projects` directly. This is what SAP HANA SDA needs for its -// catalog-discovery probe (`SQLTables(catalog="%", schema="", table="")`), -// and avoiding the network call keeps the probe well under HANA's ~30s -// discovery timeout — `ListAllProjects` can take many seconds when the -// service account has access to large numbers of GCP projects. odbc_internal::StatusRecordOr GetResultSetForProjects( ODBCBQClient& bq_client, SQLULEN metadata_id, - std::string const& additional_projects = "", - std::string const& connection_catalog = ""); + std::string const& additional_projects = ""); // Search for all datasets in all projects and populate ResultSet for it. odbc_internal::StatusRecordOr GetResultSetForDatasets( diff --git a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc index dd6ffd6c05..5d17ce2085 100644 --- a/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc +++ b/google/cloud/odbc/bq_driver/odbc_driver_metadata.cc @@ -478,12 +478,8 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, if (!metadata_id && project_filter == SQL_ALL_CATALOGS && dataset_filter.empty() && table_filter.empty()) { - // Pass dsn.catalog as the 4th arg so GetResultSetForProjects can take - // the fast path and avoid the BigQuery ListAllProjects round-trip; - // SAP HANA SDA's CHECK_REMOTE_SOURCE catalog probe lands here. - auto const& dsn = conn_handle.GetDsn(); result_set_status = GetResultSetForProjects( - bq_client, metadata_id, dsn.additional_projects, dsn.catalog); + bq_client, metadata_id, conn_handle.GetDsn().additional_projects); } else if (!metadata_id && project_filter.empty() && dataset_filter == SQL_ALL_SCHEMAS && table_filter.empty()) { result_set_status = @@ -533,7 +529,7 @@ SQLRETURN SQLTablesInternal(SQLHSTMT stmt_handle, SQLCHAR* catalog_name, LOG(ERROR) << "SQLTables::PopulateIrd:: " << ird_status.message; return LogAndReturnCode(handle, ird_status); } - LOG(INFO) << "SQLTablesInternal:: end"; + handle.SetResultSet(result_set); handle.SetStmtState(StmtStates::kStatementExecutedWithRs); return SQL_SUCCESS; From 2d6334a6d4d0426e9897a1cf4e6f54b662324a4d Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 6 May 2026 16:48:34 +0530 Subject: [PATCH 07/32] test --- .../bq_driver/internal/data_translation.cc | 99 +++++++------- .../internal/data_translation_inv.cc | 2 +- .../odbc/bq_driver/internal/odbc_desc_attr.cc | 3 +- .../bq_driver/internal/odbc_type_utils.cc | 21 ++- .../odbc/bq_driver/internal/odbc_type_utils.h | 28 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 26 +--- google/cloud/odbc/bq_driver/internal/utils.h | 28 ---- google/cloud/odbc/bq_driver/odbc_api.cc | 124 +++++++++--------- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 13 +- 9 files changed, 141 insertions(+), 203 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index ce190a616f..d50ad15d6a 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -104,7 +104,7 @@ odbc_internal::StatusRecord ConvertFromNumericDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); + SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; WStrToOutputBufferResponse(wstr.GetValue(), dest_data.buf, wchar_capacity, @@ -326,7 +326,7 @@ odbc_internal::StatusRecord ConvertFromStringDSValue(DSValue const& src_dsval, } auto src_len = static_cast(wide_str.length()); - SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); + SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_str, dest_data.buf, wchar_capacity, src_len, required_chars, @@ -907,7 +907,7 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); SQLLEN required_chars = static_cast(wstr->length()) + 1; return WStrToOutputBufferResponse( wstr.GetValue(), dest_buf, wchar_capacity, k_time_src_len, @@ -1007,25 +1007,26 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = k_timestamp_src_len * WireWcharSize(); + *res_len = k_timestamp_src_len * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(*wstr, dest_buf, k_timestamp_src_len); - write_terminator(k_timestamp_src_len); + std::memcpy(dest, wstr_data.data(), + (k_timestamp_src_len) * sizeof(SQLWCHAR)); + dest[k_timestamp_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { - *res_len = wchar_capacity * WireWcharSize(); + *res_len = wchar_capacity * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); - write_terminator(wchar_capacity - 1); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * sizeof(SQLWCHAR)); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1184,24 +1185,25 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = k_datetime_src_len * WireWcharSize(); + *res_len = k_datetime_src_len * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(*wstr, dest_buf, k_datetime_src_len); - write_terminator(k_datetime_src_len); + std::memcpy(dest, wstr_data.data(), + (k_datetime_src_len) * sizeof(SQLWCHAR)); + dest[k_datetime_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { - *res_len = wchar_capacity * WireWcharSize(); + *res_len = wchar_capacity * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); - write_terminator(wchar_capacity - 1); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * sizeof(SQLWCHAR)); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1388,7 +1390,7 @@ odbc_internal::StatusRecord ConvertFromDateDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "DSValueToWchar Conversion Failed"}; } - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1424,7 +1426,7 @@ StatusRecord ConvertStringToJsonOutputBuffer(std::string const& src_str, return StatusRecord{SQLStates::k_HY000(), "Conversion to UTF-16 failed"}; } - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_string.GetValue(), dest_buf, @@ -1488,7 +1490,7 @@ StatusRecord ConvertFromArrayDSValue(DSValue const& src_dsval, if (!wide_string.Ok()) { return StatusRecord{SQLStates::k_HY000(), "Conversion Failed"}; } - SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); + SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1607,7 +1609,7 @@ odbc_internal::StatusRecord ConvertFromIntervalDSValue(DSValue const& src_dsval, StatusRecord{SQLStates::k_HY000(), wstr.GetStatusRecord().message}; break; } - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); auto interval_char_length = static_cast(wstr.GetValue().length()); return WStrIntervalBufferResponse( @@ -1893,7 +1895,7 @@ StatusRecord ConvertFromGeographyDSValue(DSValue const& src_dsval, } std::memset(dest_data.buf, 0, buffer_length); std::wstring const& wide_str = wstr.GetValue(); - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); SQLLEN src_len = static_cast(wide_str.length()); SQLLEN required_chars = src_len + 1; status_record = WStrToOutputBufferResponse( @@ -2041,19 +2043,16 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, } std::wstring const& utf16_value = utf16_str.GetValue(); - size_t const required_size = utf16_value.length() * WireWcharSize(); + size_t const required_size = utf16_str.GetValue().length() * sizeof(SQLWCHAR); - auto write_terminator = [&](size_t char_index) { - auto* p = static_cast(dest_data.buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; + auto* buffer = reinterpret_cast(dest_data.buf); // Handle truncation if buffer is insufficient - if (static_cast(dest_data.buflen) < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; - WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy); - write_terminator(num_chars_to_copy); + if (dest_data.buflen < required_size) { + size_t num_chars_to_copy = (dest_data.buflen / sizeof(SQLWCHAR)) - 1; + std::memcpy(buffer, utf16_value.data(), + num_chars_to_copy * sizeof(SQLWCHAR)); + buffer[num_chars_to_copy] = L'\0'; if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; @@ -2061,15 +2060,17 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - WriteWideToWireBuffer(utf16_value, dest_data.buf, utf16_value.size()); - size_t buffer_chars = dest_data.buflen / WireWcharSize(); - if (utf16_value.size() < buffer_chars) { - write_terminator(utf16_value.size()); + for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { + buffer[i] = static_cast(utf16_str.GetValue()[i]); + } + size_t buffer_chars = dest_data.buflen / sizeof(SQLWCHAR); + if (utf16_str.GetValue().size() < buffer_chars) { + buffer[utf16_str.GetValue().size()] = L'\0'; } // Set output length if (dest_data.result_len) { - *dest_data.result_len = utf16_value.size() * WireWcharSize(); + *dest_data.result_len = utf16_str.GetValue().size() * sizeof(SQLWCHAR); } return status_record; } @@ -2209,7 +2210,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, auto status = ConvertRangeToTimestampFormat(src_str); if (!status.ok()) { return status; - } + } } switch (dest_data.type) { @@ -2237,7 +2238,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "Conversion to SQL_C_WCHAR failed."}; } - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); SQLLEN src_len = static_cast(wstr->length()); SQLLEN required_chars = src_len + 1; return WStrToOutputBufferResponse( diff --git a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc index c20ae95e00..76832234b3 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc @@ -53,7 +53,7 @@ StatusRecordOr ConvertFromCharBuffer(DataBuffer& src_data, auto* wchar_buf = static_cast(src_buf); if ((result_len > 0) || (result_len == SQL_NTS)) { if (result_len > 0) { - result_len /= WireWcharSize(); + result_len /= sizeof(SQLWCHAR); } auto utf8_res = BqConvertSQLWCHARToString( wchar_buf, static_cast(result_len)); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc index 1a28dcb25d..8052921952 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc @@ -14,7 +14,6 @@ #include "google/cloud/odbc/bq_driver/internal/odbc_desc_attr.h" #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" -#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/sql_state_constants.h" #include "google/cloud/odbc/internal/status_record_or.h" #include @@ -389,7 +388,7 @@ StatusRecord DescriptorRecord::SetOctetLength(SQLSMALLINT type, case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: - octet_length = value * WireWcharSize(); + octet_length = value * sizeof(SQLWCHAR); break; case SQL_DECIMAL: case SQL_NUMERIC: diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 13d235d349..993fa7374d 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,25 +45,22 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); - - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + (char_index * wire_sz); - std::memset(p, 0, wire_sz); - }; + std::vector wstr_data(wstr.begin(), wstr.end()); + wstr_data.emplace_back(L'\0'); + auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = char_len * wire_sz; + *res_len = char_len * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(wstr, dest_buf, char_len); - write_terminator(char_len); + std::memcpy(dest, wstr_data.data(), (char_len) * sizeof(SQLWCHAR)); + dest[char_len] = L'\0'; } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * wire_sz; + *res_len = buffer_length * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(wstr, dest_buf, buffer_length); - write_terminator(buffer_length - 1); + std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index c7515445d3..6630ef929f 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -17,9 +17,7 @@ #include "google/cloud/odbc/internal/diagnostic_records.h" #include "google/cloud/odbc/internal/sql_state_constants.h" -#include "google/cloud/odbc/bq_driver/internal/utils.h" #include -#include #include #include #include @@ -181,18 +179,9 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); - - // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset - // `byte_off` in dest_buf. - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + (char_index * wire_sz); - std::memset(p, 0, wire_sz); - }; - if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - write_terminator(0); + reinterpret_cast(dest_buf)[0] = L'\0'; } if (res_len) { *res_len = 0; @@ -200,18 +189,21 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } + std::vector wstr_data(wstr.begin(), wstr.end()); + + auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { - *res_len = src_len * wire_sz; + *res_len = src_len * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(wstr, dest_buf, src_len); - write_terminator(src_len); + std::memcpy(dest, wstr_data.data(), (src_len) * sizeof(SQLWCHAR)); + dest[src_len] = L'\0'; } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { - *res_len = buffer_length * wire_sz; + *res_len = buffer_length * sizeof(SQLWCHAR); } - WriteWideToWireBuffer(wstr, dest_buf, buffer_length); - write_terminator(buffer_length - 1); + std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 85cf5b9045..690f1f3cfa 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -42,7 +42,8 @@ using ::google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; namespace { -// Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". + + // Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". // Set inside BqConvertSQLWCHARToString once we observe the UTF-16LE byte // pattern in an input buffer. The wire format never changes within a process, // so once latched it stays latched. @@ -58,24 +59,6 @@ bool IsRuntimeWireUtf16Le() { #endif } -size_t WireWcharSize() { - return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); -} - -void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { - if (count > src.size()) count = src.size(); - if (IsRuntimeWireUtf16Le()) { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); - } - } else { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); - } - } -} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; @@ -203,7 +186,7 @@ size_t BufferSizeForType(SQLSMALLINT type, size_t requested) { minimum_size = sizeof(SQL_TIMESTAMP_STRUCT); break; case SQL_C_WCHAR: - minimum_size = WireWcharSize(); + minimum_size = sizeof(SQLWCHAR); break; case SQL_C_SBIGINT: minimum_size = sizeof(SQLBIGINT); @@ -873,8 +856,7 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( // later call (including the ambiguous ones from SQLTables(catalog="%")) // honors the latch via IsRuntimeWireUtf16Le(). if (sizeof(SQLWCHAR) == 4) { - bool use_utf16le = - g_utf16le_wire_latched.load(std::memory_order_relaxed); + bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); auto const* bytes = reinterpret_cast(in_str); if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 26c123f597..3485cf6a56 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -226,34 +226,6 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( SQLWCHAR* in_str, SQLINTEGER in_str_len); -// True once the driver has detected that the loaded ODBC manager delivers -// wide-char buffers as 2-byte UTF-16LE even though the driver was compiled -// with sizeof(SQLWCHAR) == 4 (iODBC headers on Linux/macOS). This is the -// SAP HANA SDA case: HANA's bundled unixODBC speaks UTF-16LE, but the -// release-pipeline binary was built against iODBC. -// -// Latched (process-lifetime, atomic) inside BqConvertSQLWCHARToString the -// first time the wire-format heuristic fires. Always false on Windows and -// for builds where sizeof(SQLWCHAR) is already 2 — no adaptation needed. -bool IsRuntimeWireUtf16Le(); - -// Bytes per wide character on the wire between this driver and its loaded -// manager. Equals sizeof(SQLWCHAR) by default; equals 2 once the UTF-16LE -// wire format has been latched. Use this in *every* arithmetic expression -// that converts between byte counts and character counts on a buffer that -// crosses the driver/manager boundary — never use sizeof(SQLWCHAR) directly -// for that purpose. -size_t WireWcharSize(); - -// Copies up to `count` wide characters from `src` (a std::wstring whose -// element width is wchar_t — 4 bytes on Linux/macOS, 2 bytes on Windows) -// into `dest` using the wire format. On the UTF-16LE-on-wire path each -// wchar_t is narrowed to 16 bits — fine for ASCII / BMP content (which -// covers ODBC metadata strings); supplementary-plane characters would be -// truncated, but the same is true today for any code that does -// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement -// for that pattern. -void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count); std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 710d122fbd..92c4bf0cc1 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -65,8 +65,6 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; -using google::cloud::odbc_bq_driver_internal::WireWcharSize; -using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -282,8 +280,7 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString && - outConnectionStringBufferLen > 0) { + if (SQL_SUCCEEDED(rc) && outConnectionString) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -294,13 +291,7 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - size_t buf_chars = static_cast(outConnectionStringBufferLen); - size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); - auto* term = static_cast(static_cast(outConnectionString)) + - (to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); - out_conn_str_len = static_cast(to_copy); + outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -402,9 +393,10 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, return utf16_out_conn_str.GetCalculatedReturnCode(); } std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * WireWcharSize()); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, - utf16_out_conn_str->size()); + outConnectionStringBufferLen * sizeof(SQLWCHAR)); + std::memcpy((SQLWCHAR*)outConnectionString, + ToSqlWChar(utf16_out_conn_str->data()), + utf16_out_conn_str->size() * sizeof(SQLWCHAR)); } return rc; @@ -651,19 +643,15 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_info_val->begin(), + utf16_info_val->end()); + sql_w_str.emplace_back(L'\0'); std::size_t bytes_available = static_cast(infoValueBufferLen); - // +1 to leave room for terminator within bytes_available, then - // shrink-to-fit at the wire char boundary. - std::size_t chars_available = bytes_available / WireWcharSize(); - std::size_t chars_to_copy = - std::min(utf16_info_val->size(), chars_available); - WriteWideToWireBuffer(*utf16_info_val, infoValue, chars_to_copy); - if (chars_to_copy < chars_available) { - auto* term = static_cast(infoValue) + - (chars_to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); - } + std::size_t bytes_to_copy = + std::min(sql_w_str.size() * sizeof(SQLWCHAR), bytes_available); + + std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); } } else { if (info_val_buffer_len > 0) { @@ -674,7 +662,7 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, } } if (infoValueStringLen) - *infoValueStringLen = info_val_buffer_len * WireWcharSize(); + *infoValueStringLen = info_val_buffer_len * sizeof(SQLWCHAR); return rc; } @@ -818,7 +806,7 @@ SQLRETURN SQL_API SQLSetConnectAttrW(SQLHDBC connectionHandle, ConnectionValueType::kSqlChr) { if (valueStringLen && valueStringLen > 0) { updated_attrib_status = - ConvertSQLPointerToSQLChar(value, valueStringLen / WireWcharSize()); + ConvertSQLPointerToSQLChar(value, valueStringLen / sizeof(SQLWCHAR)); } else { updated_attrib_status = ConvertSQLPointerToSQLChar(value, valueStringLen); } @@ -924,10 +912,14 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - size_t char_count = wcslen(updated_out_attr_status->data()); - *valueStringLen = char_count * WireWcharSize(); + *valueStringLen = + wcslen(updated_out_attr_status->data()) * sizeof(SQLWCHAR); + std::vector sql_w_str( + updated_out_attr_status->c_str(), + updated_out_attr_status->c_str() + *valueStringLen); + sql_w_str.emplace_back(L'\0'); std::memset(value, '\0', valueBufferLen); - WriteWideToWireBuffer(*updated_out_attr_status, value, char_count); + std::memcpy(value, sql_w_str.data(), sql_w_str.size()); } return rc; @@ -1159,10 +1151,13 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - size_t char_count = utf16_out_desc_val->size(); - out_desc_val_string_len = char_count * WireWcharSize(); + out_desc_val_string_len = + wcslen(utf16_out_desc_val->data()) * sizeof(SQLWCHAR); + std::vector sql_w_str(utf16_out_desc_val->begin(), + utf16_out_desc_val->end()); + sql_w_str.emplace_back(L'\0'); std::memset(outDescValue, '\0', outDescValueBufferLen); - WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, char_count); + std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1243,9 +1238,9 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - std::memset(name, '\0', nameBufferLen * WireWcharSize()); + std::memset(name, '\0', nameBufferLen * sizeof(SQLWCHAR)); std::memcpy(name, ToSqlWChar(utf16_name->data()), - name_string_len * WireWcharSize()); + name_string_len * sizeof(SQLWCHAR)); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1531,11 +1526,11 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_cur_name, cursorName, - utf16_cur_name->size()); - auto* term = static_cast(static_cast(cursorName)) + - (utf16_cur_name->size() * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + std::vector sql_w_str(utf16_cur_name->begin(), + utf16_cur_name->end()); + sql_w_str.emplace_back(L'\0'); + std::memcpy(cursorName, sql_w_str.data(), + (sql_w_str.size() + 1) * sizeof(SQLWCHAR)); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2108,16 +2103,14 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const buf_chars = - static_cast(characterAttributeBufferLen) / WireWcharSize(); - size_t const chars_to_copy = std::min(wstr.size(), buf_chars); - - WriteWideToWireBuffer(wstr, characterAttribute, chars_to_copy); - if (characterAttributeBufferLen >= - static_cast(WireWcharSize())) { - auto* term = static_cast(characterAttribute) + - (chars_to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + size_t const bytes_to_copy = + std::min(static_cast(characterAttributeBufferLen), + wstr.size() * sizeof(SQLWCHAR)); + + std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); + if (characterAttributeBufferLen >= sizeof(SQLWCHAR)) { + SQLWCHAR* wchar_buf = static_cast(characterAttribute); + wchar_buf[bytes_to_copy / sizeof(SQLWCHAR)] = 0; } character_attribute_string_len = static_cast(wstr.size()); @@ -2130,7 +2123,7 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, *characterAttributeStringLen = character_attribute_string_len; #ifdef WIN32 *characterAttributeStringLen = - character_attribute_string_len * WireWcharSize(); + character_attribute_string_len * sizeof(SQLWCHAR); #endif // WIN32 } @@ -2292,9 +2285,12 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_col_name->begin(), + utf16_col_name->end()); + sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - WriteWideToWireBuffer(*utf16_col_name, columnName, - static_cast(column_name_string_len)); + std::memcpy(columnName, sql_w_str.data(), + column_name_string_len * sizeof(SQLWCHAR)); } if (columnNameLen) { @@ -2499,13 +2495,13 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - size_t char_count = updated_out_diag_info_status->size(); - diag_info_str_len = char_count * WireWcharSize(); - WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, - char_count); - auto* term = static_cast(diagInfo) + - (char_count * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + diag_info_str_len = + wcslen(updated_out_diag_info_status->data()) * sizeof(SQLWCHAR); + std::vector sql_w_str( + updated_out_diag_info_status->c_str(), + updated_out_diag_info_status->c_str() + diag_info_str_len); + sql_w_str.emplace_back(L'\0'); + std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2591,8 +2587,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_sql_state, sqlState, - utf16_sql_state->size()); + std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), + utf16_sql_state->size() * sizeof(SQLWCHAR)); } if (messageText && message_text_buffer_len > 0) { @@ -2602,8 +2598,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - WriteWideToWireBuffer(*utf16_msg_txt, messageText, - utf16_msg_txt->size()); + std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), + utf16_msg_txt->size() * sizeof(SQLWCHAR)); } if (messageTextLen) *messageTextLen = message_text_buffer_len; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index 57bc814b9e..c9f1bd4e4f 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -29,7 +29,6 @@ namespace google::cloud::odbc_bq_driver { using google::cloud::odbc_bq_driver_internal::BQDataType; -using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::CheckTargetType; using google::cloud::odbc_bq_driver_internal::ConnectionHandle; using google::cloud::odbc_bq_driver_internal::CreateDSRowFromTypeInfo; @@ -777,7 +776,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, // 3. If the data fits or is not a variable-length type, return it directly in // the caller’s buffer. SQLLEN target_buff_len = (target_c_type == SQL_C_WCHAR) - ? (target_value_buffer_len / WireWcharSize()) + ? (target_value_buffer_len / sizeof(SQLWCHAR)) : target_value_buffer_len; if (offset == 0) { if ((ds_val.size() > target_buff_len) && @@ -790,7 +789,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, size_t buffer_size = 0; if (target_c_type == SQL_C_WCHAR) { - buffer_size = (ds_val.size() + 1) * WireWcharSize(); + buffer_size = (ds_val.size() + 1) * sizeof(SQLWCHAR); } else { buffer_size = ds_val.size() + 1; } @@ -846,18 +845,18 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, result_set.translated_data.row_offset = offset + target_value_buffer_len; } else if (target_c_type == SQL_C_WCHAR) { auto data_size = result_set.translated_data.data.size(); - auto max_buff_chars = target_value_buffer_len / WireWcharSize(); - auto offset_chars = offset / WireWcharSize(); + auto max_buff_chars = target_value_buffer_len / sizeof(SQLWCHAR); + auto offset_chars = offset / sizeof(SQLWCHAR); auto remain_chars = (data_size > offset_chars) ? (data_size - offset_chars) : 0; auto copy_chars = (remain_chars >= max_buff_chars) ? (max_buff_chars - 1) : remain_chars; std::memcpy(target_value, result_set.translated_data.data.data() + offset, - copy_chars * WireWcharSize()); + copy_chars * sizeof(SQLWCHAR)); reinterpret_cast(target_value)[copy_chars] = 0; result_set.translated_data.row_offset = - offset + (copy_chars * WireWcharSize()); + offset + (copy_chars * sizeof(SQLWCHAR)); } else { std::memcpy(target_value, result_set.translated_data.data.data() + offset, target_value_buffer_len - 1); From 0231e48fa236589e48c72fca14e29cd4984d73dd Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 12 May 2026 10:34:11 +0530 Subject: [PATCH 08/32] added logs --- google/cloud/odbc/bq_driver/internal/utils.cc | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 690f1f3cfa..7855479145 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -20,6 +20,7 @@ #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" #include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/internal/getenv.h" +#include #include #include #include @@ -843,40 +844,38 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( return StatusRecord{SQLStates::k_HY000(), "in_str string is empty/Null"}; } + if (((in_str != nullptr) && (in_str[0] == '\0'))) { + return std::string(); + } + #if !defined(_WIN32) // The driver may be compiled against iODBC headers, where SQLWCHAR is // wchar_t (4 bytes UCS-4LE on Linux/macOS), but loaded by unixODBC, which // by default delivers 2-byte UTF-16LE wide chars. SAP HANA on Linux uses // unixODBC, so without `IconvEncoding=UCS-4LE` in odbcinst.ini we receive - // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. - // - // Detection: a UCS-4LE ASCII char looks like `XX 00 00 00`, UTF-16LE - // looks like `XX 00 YY 00`. Single-char and empty strings are ambiguous - // so we *latch* the result the first time we get a clear signal — every - // later call (including the ambiguous ones from SQLTables(catalog="%")) - // honors the latch via IsRuntimeWireUtf16Le(). + // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. Detect by inspecting the + // first two code units: ODBC connection strings and identifiers always + // start with ASCII (DSN=, DRIVER=, SELECT, etc.), so a UCS-4LE ASCII char + // looks like `XX 00 00 00` while UTF-16LE looks like `XX 00 YY 00`. if (sizeof(SQLWCHAR) == 4) { - bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); - auto const* bytes = reinterpret_cast(in_str); - if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && + LOG(INFO) << "BqConvertSQLWCHARToString:: sizeof(SQLWCHAR)=4; " + << "first 4 bytes = " + << absl::StrFormat("%02X %02X %02X %02X", bytes[0], bytes[1], + bytes[2], bytes[3]) + << "; in_str_len=" << in_str_len; + if (bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { - use_utf16le = true; - g_utf16le_wire_latched.store(true, std::memory_order_relaxed); - LOG(INFO) << "BqConvertSQLWCHARToString:: latched UTF-16LE wire format " - "for this process (unixODBC under iODBC-built driver)"; - } - - if (use_utf16le) { auto const* utf16 = reinterpret_cast(in_str); - if (utf16[0] == 0) { - return std::string(); - } SQLINTEGER count = in_str_len; if (count == SQL_NTS || count == NULL) { count = 0; while (utf16[count] != 0) ++count; } + LOG(INFO) << "BqConvertSQLWCHARToString:: detected UTF-16LE wire format " + "(unixODBC under iODBC-built driver); reinterpreting as " + "uint16_t* with code-unit count=" + << count; std::wstring wstr; wstr.reserve(count); for (SQLINTEGER i = 0; i < count; ++i) { @@ -884,12 +883,12 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( } return Utf16ToUtf8(wstr); } + LOG(INFO) << "BqConvertSQLWCHARToString:: byte pattern does not match " + "UTF-16LE; falling through to default 4-byte SQLWCHAR path"; } #endif - if (in_str[0] == '\0') { - return std::string(); - } + if (in_str_len == SQL_NTS || in_str_len == NULL) { in_str_len = static_cast(std::char_traits::length(in_str)); @@ -1020,6 +1019,8 @@ bool IsInfoTypeString(SQLUSMALLINT InfoType) { // boundary, aborting the driver. The manual loop has no // dependency. std::string CastOdbcRegexToCppRegex(std::string const& str) { + LOG(INFO) << "CastOdbcRegexToCppRegex: " << str + << std::endl; std::string result; // Worst case: every character becomes ".*" (2 chars). result.reserve(str.size() * 2); From 507960061c15603c31d2100ff5a5975c85853b1c Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 12 May 2026 14:59:38 +0530 Subject: [PATCH 09/32] Revert "added logs" This reverts commit 1c7187d2247aaf0c6e853e6699e4306be450683d. --- google/cloud/odbc/bq_driver/internal/utils.cc | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 7855479145..690f1f3cfa 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -20,7 +20,6 @@ #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" #include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/internal/getenv.h" -#include #include #include #include @@ -844,38 +843,40 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( return StatusRecord{SQLStates::k_HY000(), "in_str string is empty/Null"}; } - if (((in_str != nullptr) && (in_str[0] == '\0'))) { - return std::string(); - } - #if !defined(_WIN32) // The driver may be compiled against iODBC headers, where SQLWCHAR is // wchar_t (4 bytes UCS-4LE on Linux/macOS), but loaded by unixODBC, which // by default delivers 2-byte UTF-16LE wide chars. SAP HANA on Linux uses // unixODBC, so without `IconvEncoding=UCS-4LE` in odbcinst.ini we receive - // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. Detect by inspecting the - // first two code units: ODBC connection strings and identifiers always - // start with ASCII (DSN=, DRIVER=, SELECT, etc.), so a UCS-4LE ASCII char - // looks like `XX 00 00 00` while UTF-16LE looks like `XX 00 YY 00`. + // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. + // + // Detection: a UCS-4LE ASCII char looks like `XX 00 00 00`, UTF-16LE + // looks like `XX 00 YY 00`. Single-char and empty strings are ambiguous + // so we *latch* the result the first time we get a clear signal — every + // later call (including the ambiguous ones from SQLTables(catalog="%")) + // honors the latch via IsRuntimeWireUtf16Le(). if (sizeof(SQLWCHAR) == 4) { + bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); + auto const* bytes = reinterpret_cast(in_str); - LOG(INFO) << "BqConvertSQLWCHARToString:: sizeof(SQLWCHAR)=4; " - << "first 4 bytes = " - << absl::StrFormat("%02X %02X %02X %02X", bytes[0], bytes[1], - bytes[2], bytes[3]) - << "; in_str_len=" << in_str_len; - if (bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && + if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { + use_utf16le = true; + g_utf16le_wire_latched.store(true, std::memory_order_relaxed); + LOG(INFO) << "BqConvertSQLWCHARToString:: latched UTF-16LE wire format " + "for this process (unixODBC under iODBC-built driver)"; + } + + if (use_utf16le) { auto const* utf16 = reinterpret_cast(in_str); + if (utf16[0] == 0) { + return std::string(); + } SQLINTEGER count = in_str_len; if (count == SQL_NTS || count == NULL) { count = 0; while (utf16[count] != 0) ++count; } - LOG(INFO) << "BqConvertSQLWCHARToString:: detected UTF-16LE wire format " - "(unixODBC under iODBC-built driver); reinterpreting as " - "uint16_t* with code-unit count=" - << count; std::wstring wstr; wstr.reserve(count); for (SQLINTEGER i = 0; i < count; ++i) { @@ -883,12 +884,12 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( } return Utf16ToUtf8(wstr); } - LOG(INFO) << "BqConvertSQLWCHARToString:: byte pattern does not match " - "UTF-16LE; falling through to default 4-byte SQLWCHAR path"; } #endif - + if (in_str[0] == '\0') { + return std::string(); + } if (in_str_len == SQL_NTS || in_str_len == NULL) { in_str_len = static_cast(std::char_traits::length(in_str)); @@ -1019,8 +1020,6 @@ bool IsInfoTypeString(SQLUSMALLINT InfoType) { // boundary, aborting the driver. The manual loop has no // dependency. std::string CastOdbcRegexToCppRegex(std::string const& str) { - LOG(INFO) << "CastOdbcRegexToCppRegex: " << str - << std::endl; std::string result; // Worst case: every character becomes ".*" (2 chars). result.reserve(str.size() * 2); From 3cc65b56ae14a9537c36f93823d057ab8336c075 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 12 May 2026 15:00:02 +0530 Subject: [PATCH 10/32] Revert "test" This reverts commit c350b18e3ec09dd510d990bc374e6c9f14f93495. --- .../bq_driver/internal/data_translation.cc | 99 +++++++------- .../internal/data_translation_inv.cc | 2 +- .../odbc/bq_driver/internal/odbc_desc_attr.cc | 3 +- .../bq_driver/internal/odbc_type_utils.cc | 21 +-- .../odbc/bq_driver/internal/odbc_type_utils.h | 28 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 26 +++- google/cloud/odbc/bq_driver/internal/utils.h | 28 ++++ google/cloud/odbc/bq_driver/odbc_api.cc | 124 +++++++++--------- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 13 +- 9 files changed, 203 insertions(+), 141 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index d50ad15d6a..ce190a616f 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -104,7 +104,7 @@ odbc_internal::StatusRecord ConvertFromNumericDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; WStrToOutputBufferResponse(wstr.GetValue(), dest_data.buf, wchar_capacity, @@ -326,7 +326,7 @@ odbc_internal::StatusRecord ConvertFromStringDSValue(DSValue const& src_dsval, } auto src_len = static_cast(wide_str.length()); - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_str, dest_data.buf, wchar_capacity, src_len, required_chars, @@ -907,7 +907,7 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN required_chars = static_cast(wstr->length()) + 1; return WStrToOutputBufferResponse( wstr.GetValue(), dest_buf, wchar_capacity, k_time_src_len, @@ -1007,26 +1007,25 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = k_timestamp_src_len * sizeof(SQLWCHAR); + *res_len = k_timestamp_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_timestamp_src_len) * sizeof(SQLWCHAR)); - dest[k_timestamp_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, k_timestamp_src_len); + write_terminator(k_timestamp_src_len); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); + write_terminator(wchar_capacity - 1); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1185,25 +1184,24 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = k_datetime_src_len * sizeof(SQLWCHAR); + *res_len = k_datetime_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_datetime_src_len) * sizeof(SQLWCHAR)); - dest[k_datetime_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, k_datetime_src_len); + write_terminator(k_datetime_src_len); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { - *res_len = wchar_capacity * sizeof(SQLWCHAR); + *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * sizeof(SQLWCHAR)); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); + write_terminator(wchar_capacity - 1); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1390,7 +1388,7 @@ odbc_internal::StatusRecord ConvertFromDateDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "DSValueToWchar Conversion Failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wstr->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1426,7 +1424,7 @@ StatusRecord ConvertStringToJsonOutputBuffer(std::string const& src_str, return StatusRecord{SQLStates::k_HY000(), "Conversion to UTF-16 failed"}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse(wide_string.GetValue(), dest_buf, @@ -1490,7 +1488,7 @@ StatusRecord ConvertFromArrayDSValue(DSValue const& src_dsval, if (!wide_string.Ok()) { return StatusRecord{SQLStates::k_HY000(), "Conversion Failed"}; } - SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize(); auto src_len = static_cast(wide_string->length()); SQLINTEGER required_chars = src_len + 1; return WStrToOutputBufferResponse( @@ -1609,7 +1607,7 @@ odbc_internal::StatusRecord ConvertFromIntervalDSValue(DSValue const& src_dsval, StatusRecord{SQLStates::k_HY000(), wstr.GetStatusRecord().message}; break; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); auto interval_char_length = static_cast(wstr.GetValue().length()); return WStrIntervalBufferResponse( @@ -1895,7 +1893,7 @@ StatusRecord ConvertFromGeographyDSValue(DSValue const& src_dsval, } std::memset(dest_data.buf, 0, buffer_length); std::wstring const& wide_str = wstr.GetValue(); - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wide_str.length()); SQLLEN required_chars = src_len + 1; status_record = WStrToOutputBufferResponse( @@ -2043,16 +2041,19 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, } std::wstring const& utf16_value = utf16_str.GetValue(); - size_t const required_size = utf16_str.GetValue().length() * sizeof(SQLWCHAR); + size_t const required_size = utf16_value.length() * WireWcharSize(); - auto* buffer = reinterpret_cast(dest_data.buf); + auto write_terminator = [&](size_t char_index) { + auto* p = static_cast(dest_data.buf) + + (char_index * WireWcharSize()); + std::memset(p, 0, WireWcharSize()); + }; // Handle truncation if buffer is insufficient - if (dest_data.buflen < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / sizeof(SQLWCHAR)) - 1; - std::memcpy(buffer, utf16_value.data(), - num_chars_to_copy * sizeof(SQLWCHAR)); - buffer[num_chars_to_copy] = L'\0'; + if (static_cast(dest_data.buflen) < required_size) { + size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; + WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy); + write_terminator(num_chars_to_copy); if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; @@ -2060,17 +2061,15 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { - buffer[i] = static_cast(utf16_str.GetValue()[i]); - } - size_t buffer_chars = dest_data.buflen / sizeof(SQLWCHAR); - if (utf16_str.GetValue().size() < buffer_chars) { - buffer[utf16_str.GetValue().size()] = L'\0'; + WriteWideToWireBuffer(utf16_value, dest_data.buf, utf16_value.size()); + size_t buffer_chars = dest_data.buflen / WireWcharSize(); + if (utf16_value.size() < buffer_chars) { + write_terminator(utf16_value.size()); } // Set output length if (dest_data.result_len) { - *dest_data.result_len = utf16_str.GetValue().size() * sizeof(SQLWCHAR); + *dest_data.result_len = utf16_value.size() * WireWcharSize(); } return status_record; } @@ -2210,7 +2209,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, auto status = ConvertRangeToTimestampFormat(src_str); if (!status.ok()) { return status; - } + } } switch (dest_data.type) { @@ -2238,7 +2237,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, return StatusRecord{SQLStates::k_HY000(), "Conversion to SQL_C_WCHAR failed."}; } - SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR); + SQLLEN wchar_capacity = buffer_length / WireWcharSize(); SQLLEN src_len = static_cast(wstr->length()); SQLLEN required_chars = src_len + 1; return WStrToOutputBufferResponse( diff --git a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc index 76832234b3..c20ae95e00 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation_inv.cc @@ -53,7 +53,7 @@ StatusRecordOr ConvertFromCharBuffer(DataBuffer& src_data, auto* wchar_buf = static_cast(src_buf); if ((result_len > 0) || (result_len == SQL_NTS)) { if (result_len > 0) { - result_len /= sizeof(SQLWCHAR); + result_len /= WireWcharSize(); } auto utf8_res = BqConvertSQLWCHARToString( wchar_buf, static_cast(result_len)); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc index 8052921952..1a28dcb25d 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc @@ -14,6 +14,7 @@ #include "google/cloud/odbc/bq_driver/internal/odbc_desc_attr.h" #include "google/cloud/odbc/bq_driver/internal/trace_utils.h" +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/sql_state_constants.h" #include "google/cloud/odbc/internal/status_record_or.h" #include @@ -388,7 +389,7 @@ StatusRecord DescriptorRecord::SetOctetLength(SQLSMALLINT type, case SQL_WCHAR: case SQL_WVARCHAR: case SQL_WLONGVARCHAR: - octet_length = value * sizeof(SQLWCHAR); + octet_length = value * WireWcharSize(); break; case SQL_DECIMAL: case SQL_NUMERIC: diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 993fa7374d..13d235d349 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,22 +45,25 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - std::vector wstr_data(wstr.begin(), wstr.end()); - wstr_data.emplace_back(L'\0'); + size_t const wire_sz = WireWcharSize(); + + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + (char_index * wire_sz); + std::memset(p, 0, wire_sz); + }; - auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = char_len * sizeof(SQLWCHAR); + *res_len = char_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (char_len) * sizeof(SQLWCHAR)); - dest[char_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, char_len); + write_terminator(char_len); } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, buffer_length); + write_terminator(buffer_length - 1); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 6630ef929f..c7515445d3 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -17,7 +17,9 @@ #include "google/cloud/odbc/internal/diagnostic_records.h" #include "google/cloud/odbc/internal/sql_state_constants.h" +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include +#include #include #include #include @@ -179,9 +181,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); + size_t const wire_sz = WireWcharSize(); + + // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset + // `byte_off` in dest_buf. + auto write_terminator = [&](SQLLEN char_index) { + auto* p = static_cast(dest_buf) + (char_index * wire_sz); + std::memset(p, 0, wire_sz); + }; + if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - reinterpret_cast(dest_buf)[0] = L'\0'; + write_terminator(0); } if (res_len) { *res_len = 0; @@ -189,21 +200,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - std::vector wstr_data(wstr.begin(), wstr.end()); - - auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { - *res_len = src_len * sizeof(SQLWCHAR); + *res_len = src_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (src_len) * sizeof(SQLWCHAR)); - dest[src_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, src_len); + write_terminator(src_len); } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { - *res_len = buffer_length * sizeof(SQLWCHAR); + *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR)); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, buffer_length); + write_terminator(buffer_length - 1); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 690f1f3cfa..85cf5b9045 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -42,8 +42,7 @@ using ::google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; namespace { - - // Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". +// Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". // Set inside BqConvertSQLWCHARToString once we observe the UTF-16LE byte // pattern in an input buffer. The wire format never changes within a process, // so once latched it stays latched. @@ -59,6 +58,24 @@ bool IsRuntimeWireUtf16Le() { #endif } +size_t WireWcharSize() { + return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); +} + +void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { + if (count > src.size()) count = src.size(); + if (IsRuntimeWireUtf16Le()) { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); + } + } else { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); + } + } +} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; @@ -186,7 +203,7 @@ size_t BufferSizeForType(SQLSMALLINT type, size_t requested) { minimum_size = sizeof(SQL_TIMESTAMP_STRUCT); break; case SQL_C_WCHAR: - minimum_size = sizeof(SQLWCHAR); + minimum_size = WireWcharSize(); break; case SQL_C_SBIGINT: minimum_size = sizeof(SQLBIGINT); @@ -856,7 +873,8 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( // later call (including the ambiguous ones from SQLTables(catalog="%")) // honors the latch via IsRuntimeWireUtf16Le(). if (sizeof(SQLWCHAR) == 4) { - bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); + bool use_utf16le = + g_utf16le_wire_latched.load(std::memory_order_relaxed); auto const* bytes = reinterpret_cast(in_str); if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 3485cf6a56..26c123f597 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -226,6 +226,34 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( SQLWCHAR* in_str, SQLINTEGER in_str_len); +// True once the driver has detected that the loaded ODBC manager delivers +// wide-char buffers as 2-byte UTF-16LE even though the driver was compiled +// with sizeof(SQLWCHAR) == 4 (iODBC headers on Linux/macOS). This is the +// SAP HANA SDA case: HANA's bundled unixODBC speaks UTF-16LE, but the +// release-pipeline binary was built against iODBC. +// +// Latched (process-lifetime, atomic) inside BqConvertSQLWCHARToString the +// first time the wire-format heuristic fires. Always false on Windows and +// for builds where sizeof(SQLWCHAR) is already 2 — no adaptation needed. +bool IsRuntimeWireUtf16Le(); + +// Bytes per wide character on the wire between this driver and its loaded +// manager. Equals sizeof(SQLWCHAR) by default; equals 2 once the UTF-16LE +// wire format has been latched. Use this in *every* arithmetic expression +// that converts between byte counts and character counts on a buffer that +// crosses the driver/manager boundary — never use sizeof(SQLWCHAR) directly +// for that purpose. +size_t WireWcharSize(); + +// Copies up to `count` wide characters from `src` (a std::wstring whose +// element width is wchar_t — 4 bytes on Linux/macOS, 2 bytes on Windows) +// into `dest` using the wire format. On the UTF-16LE-on-wire path each +// wchar_t is narrowed to 16 bits — fine for ASCII / BMP content (which +// covers ODBC metadata strings); supplementary-plane characters would be +// truncated, but the same is true today for any code that does +// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement +// for that pattern. +void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count); std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 92c4bf0cc1..710d122fbd 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -65,6 +65,8 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -280,7 +282,8 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString) { + if (SQL_SUCCEEDED(rc) && outConnectionString && + outConnectionStringBufferLen > 0) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -291,7 +294,13 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); + size_t buf_chars = static_cast(outConnectionStringBufferLen); + size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); + auto* term = static_cast(static_cast(outConnectionString)) + + (to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); + out_conn_str_len = static_cast(to_copy); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -393,10 +402,9 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, return utf16_out_conn_str.GetCalculatedReturnCode(); } std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * sizeof(SQLWCHAR)); - std::memcpy((SQLWCHAR*)outConnectionString, - ToSqlWChar(utf16_out_conn_str->data()), - utf16_out_conn_str->size() * sizeof(SQLWCHAR)); + outConnectionStringBufferLen * WireWcharSize()); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + utf16_out_conn_str->size()); } return rc; @@ -643,15 +651,19 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_info_val->begin(), - utf16_info_val->end()); - sql_w_str.emplace_back(L'\0'); std::size_t bytes_available = static_cast(infoValueBufferLen); - std::size_t bytes_to_copy = - std::min(sql_w_str.size() * sizeof(SQLWCHAR), bytes_available); - - std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); + // +1 to leave room for terminator within bytes_available, then + // shrink-to-fit at the wire char boundary. + std::size_t chars_available = bytes_available / WireWcharSize(); + std::size_t chars_to_copy = + std::min(utf16_info_val->size(), chars_available); + WriteWideToWireBuffer(*utf16_info_val, infoValue, chars_to_copy); + if (chars_to_copy < chars_available) { + auto* term = static_cast(infoValue) + + (chars_to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); + } } } else { if (info_val_buffer_len > 0) { @@ -662,7 +674,7 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, } } if (infoValueStringLen) - *infoValueStringLen = info_val_buffer_len * sizeof(SQLWCHAR); + *infoValueStringLen = info_val_buffer_len * WireWcharSize(); return rc; } @@ -806,7 +818,7 @@ SQLRETURN SQL_API SQLSetConnectAttrW(SQLHDBC connectionHandle, ConnectionValueType::kSqlChr) { if (valueStringLen && valueStringLen > 0) { updated_attrib_status = - ConvertSQLPointerToSQLChar(value, valueStringLen / sizeof(SQLWCHAR)); + ConvertSQLPointerToSQLChar(value, valueStringLen / WireWcharSize()); } else { updated_attrib_status = ConvertSQLPointerToSQLChar(value, valueStringLen); } @@ -912,14 +924,10 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - *valueStringLen = - wcslen(updated_out_attr_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_attr_status->c_str(), - updated_out_attr_status->c_str() + *valueStringLen); - sql_w_str.emplace_back(L'\0'); + size_t char_count = wcslen(updated_out_attr_status->data()); + *valueStringLen = char_count * WireWcharSize(); std::memset(value, '\0', valueBufferLen); - std::memcpy(value, sql_w_str.data(), sql_w_str.size()); + WriteWideToWireBuffer(*updated_out_attr_status, value, char_count); } return rc; @@ -1151,13 +1159,10 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - out_desc_val_string_len = - wcslen(utf16_out_desc_val->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str(utf16_out_desc_val->begin(), - utf16_out_desc_val->end()); - sql_w_str.emplace_back(L'\0'); + size_t char_count = utf16_out_desc_val->size(); + out_desc_val_string_len = char_count * WireWcharSize(); std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); + WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, char_count); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1238,9 +1243,9 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - std::memset(name, '\0', nameBufferLen * sizeof(SQLWCHAR)); + std::memset(name, '\0', nameBufferLen * WireWcharSize()); std::memcpy(name, ToSqlWChar(utf16_name->data()), - name_string_len * sizeof(SQLWCHAR)); + name_string_len * WireWcharSize()); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1526,11 +1531,11 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_cur_name->begin(), - utf16_cur_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memcpy(cursorName, sql_w_str.data(), - (sql_w_str.size() + 1) * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_cur_name, cursorName, + utf16_cur_name->size()); + auto* term = static_cast(static_cast(cursorName)) + + (utf16_cur_name->size() * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2103,14 +2108,16 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const bytes_to_copy = - std::min(static_cast(characterAttributeBufferLen), - wstr.size() * sizeof(SQLWCHAR)); - - std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); - if (characterAttributeBufferLen >= sizeof(SQLWCHAR)) { - SQLWCHAR* wchar_buf = static_cast(characterAttribute); - wchar_buf[bytes_to_copy / sizeof(SQLWCHAR)] = 0; + size_t const buf_chars = + static_cast(characterAttributeBufferLen) / WireWcharSize(); + size_t const chars_to_copy = std::min(wstr.size(), buf_chars); + + WriteWideToWireBuffer(wstr, characterAttribute, chars_to_copy); + if (characterAttributeBufferLen >= + static_cast(WireWcharSize())) { + auto* term = static_cast(characterAttribute) + + (chars_to_copy * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } character_attribute_string_len = static_cast(wstr.size()); @@ -2123,7 +2130,7 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, *characterAttributeStringLen = character_attribute_string_len; #ifdef WIN32 *characterAttributeStringLen = - character_attribute_string_len * sizeof(SQLWCHAR); + character_attribute_string_len * WireWcharSize(); #endif // WIN32 } @@ -2285,12 +2292,9 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_col_name->begin(), - utf16_col_name->end()); - sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - std::memcpy(columnName, sql_w_str.data(), - column_name_string_len * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_col_name, columnName, + static_cast(column_name_string_len)); } if (columnNameLen) { @@ -2495,13 +2499,13 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - diag_info_str_len = - wcslen(updated_out_diag_info_status->data()) * sizeof(SQLWCHAR); - std::vector sql_w_str( - updated_out_diag_info_status->c_str(), - updated_out_diag_info_status->c_str() + diag_info_str_len); - sql_w_str.emplace_back(L'\0'); - std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); + size_t char_count = updated_out_diag_info_status->size(); + diag_info_str_len = char_count * WireWcharSize(); + WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, + char_count); + auto* term = static_cast(diagInfo) + + (char_count * WireWcharSize()); + std::memset(term, 0, WireWcharSize()); } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2587,8 +2591,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), - utf16_sql_state->size() * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_sql_state, sqlState, + utf16_sql_state->size()); } if (messageText && message_text_buffer_len > 0) { @@ -2598,8 +2602,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), - utf16_msg_txt->size() * sizeof(SQLWCHAR)); + WriteWideToWireBuffer(*utf16_msg_txt, messageText, + utf16_msg_txt->size()); } if (messageTextLen) *messageTextLen = message_text_buffer_len; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index c9f1bd4e4f..57bc814b9e 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -29,6 +29,7 @@ namespace google::cloud::odbc_bq_driver { using google::cloud::odbc_bq_driver_internal::BQDataType; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::CheckTargetType; using google::cloud::odbc_bq_driver_internal::ConnectionHandle; using google::cloud::odbc_bq_driver_internal::CreateDSRowFromTypeInfo; @@ -776,7 +777,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, // 3. If the data fits or is not a variable-length type, return it directly in // the caller’s buffer. SQLLEN target_buff_len = (target_c_type == SQL_C_WCHAR) - ? (target_value_buffer_len / sizeof(SQLWCHAR)) + ? (target_value_buffer_len / WireWcharSize()) : target_value_buffer_len; if (offset == 0) { if ((ds_val.size() > target_buff_len) && @@ -789,7 +790,7 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, size_t buffer_size = 0; if (target_c_type == SQL_C_WCHAR) { - buffer_size = (ds_val.size() + 1) * sizeof(SQLWCHAR); + buffer_size = (ds_val.size() + 1) * WireWcharSize(); } else { buffer_size = ds_val.size() + 1; } @@ -845,18 +846,18 @@ SQLRETURN SQLGetDataInternal(SQLHSTMT statement_handle, result_set.translated_data.row_offset = offset + target_value_buffer_len; } else if (target_c_type == SQL_C_WCHAR) { auto data_size = result_set.translated_data.data.size(); - auto max_buff_chars = target_value_buffer_len / sizeof(SQLWCHAR); - auto offset_chars = offset / sizeof(SQLWCHAR); + auto max_buff_chars = target_value_buffer_len / WireWcharSize(); + auto offset_chars = offset / WireWcharSize(); auto remain_chars = (data_size > offset_chars) ? (data_size - offset_chars) : 0; auto copy_chars = (remain_chars >= max_buff_chars) ? (max_buff_chars - 1) : remain_chars; std::memcpy(target_value, result_set.translated_data.data.data() + offset, - copy_chars * sizeof(SQLWCHAR)); + copy_chars * WireWcharSize()); reinterpret_cast(target_value)[copy_chars] = 0; result_set.translated_data.row_offset = - offset + (copy_chars * sizeof(SQLWCHAR)); + offset + (copy_chars * WireWcharSize()); } else { std::memcpy(target_value, result_set.translated_data.data.data() + offset, target_value_buffer_len - 1); From 67b509f29371d128bc682a223cd2d933fa6b936b Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 13 May 2026 11:24:59 +0530 Subject: [PATCH 11/32] revert the pipeline changes --- ci/cloudbuild/builds/bq-driver-release.sh | 6 +- .../integration-production-bq-driver-dm.sh | 19 +---- .../ubuntu-22.04-install.Dockerfile | 70 +++++++++++-------- 3 files changed, 45 insertions(+), 50 deletions(-) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index 9316235856..a18385d934 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -81,9 +81,9 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -# mapfile -t ctest_args < <(ctest::common_args) -# io::run env -C cmake-out ctest "${ctest_args[@]}" +cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +mapfile -t ctest_args < <(ctest::common_args) +io::run env -C cmake-out ctest "${ctest_args[@]}" io::log_h1 "Packaging and Uploading Driver" diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index abb968c31a..2524de72a4 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -82,19 +82,6 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -# mapfile -t ctest_args < <(ctest::common_args) -# io::run env -C cmake-out ctest "${ctest_args[@]}" - -SO_FILE_PATH="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" -GCS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc" - -if [[ -f "$SO_FILE_PATH" ]]; then - echo "Uploading $SO_FILE_PATH to $GCS_BUCKET" - gsutil cp "$SO_FILE_PATH" "$GCS_BUCKET" -else - echo "Error: $SO_FILE_PATH not found. Upload skipped." - echo "Listing contents of cmake-out directory for debugging:" - ls -R cmake-out - exit 1 -fi \ No newline at end of file +cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +mapfile -t ctest_args < <(ctest::common_args) +io::run env -C cmake-out ctest "${ctest_args[@]}" diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 6007e7512a..33e5f46c3f 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,20 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:18.04 +FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get --no-install-recommends install -y \ - software-properties-common gnupg2 && \ - add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ - apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ # Dependency for arrow bison \ + clang-12 \ + lld-12 \ cmake \ curl \ # Dependency for arrow @@ -34,8 +32,10 @@ RUN apt-get update && \ git \ gcc \ g++ \ - gcc-11 \ - g++-11 \ + # Required by Ubsan in Ubuntu 22.04 + libunwind-12-dev \ + libc++-12-dev \ + libc++abi-12-dev \ libcurl4-openssl-dev \ # Needed to use autoreconf libltdl-dev \ @@ -50,7 +50,9 @@ RUN apt-get update && \ # Needed to use autoreconf perl \ pkg-config \ - libffi-dev \ + python3 \ + python3-dev \ + python3-pip \ tar \ unzip \ zip \ @@ -58,14 +60,21 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https + apt-transport-https \ + clang-tidy-12 + +# Needed for the existing driver v3.1.2.1004+ +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 +# Set clang as default +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=gcc -ENV CXX=g++ -RUN ln -s /usr/bin/make /usr/bin/gmake +ENV CC=clang +ENV CXX=clang++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -74,22 +83,14 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -RUN echo "ninja version: " && ninja --version -RUN echo "g++ version: " && g++ --version -RUN echo "cmake version: " && cmake --version -RUN echo "Glibc version" && ldd --version - -WORKDIR /usr/src -RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ - tar -xzf Python-3.10.14.tgz && \ - cd Python-3.10.14 && \ - ./configure --with-ensurepip=install && \ - make -j$(nproc) \ - && make altinstall - # clang-tidy-cache needs python -RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ - ln -sf /usr/local/bin/python3.10 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python $(which python3) 10 + +COPY ./requirements.txt /var/tmp/ci/requirements.txt +WORKDIR /var/tmp/downloads +RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ + then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi +RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded @@ -127,7 +128,6 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. - COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini @@ -137,3 +137,11 @@ COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.goog COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini + +# glibc 2.17 or later +RUN echo 'Installing glibc...' +RUN apt-get install -y --no-install-recommends libc6 +RUN echo 'Verifying glibc version...' +RUN dpkg -l libc6 +RUN if [ $(ldd --version | grep GLIBC | awk '{print $5}') -lt 2.17 ] ; \ + then echo 'glibc version is < 2.17: exiting...' ; exit 1 ; fi From 732447d6fc814c820510ef48acf87d508e7936f1 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 13 May 2026 15:25:14 +0530 Subject: [PATCH 12/32] Revert "revert the pipeline changes" This reverts commit da09558652a5f3a3c0a6cdd575c133fef578b1cd. --- ci/cloudbuild/builds/bq-driver-release.sh | 6 +- .../integration-production-bq-driver-dm.sh | 19 ++++- .../ubuntu-22.04-install.Dockerfile | 70 ++++++++----------- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index a18385d934..9316235856 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -81,9 +81,9 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" +# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +# mapfile -t ctest_args < <(ctest::common_args) +# io::run env -C cmake-out ctest "${ctest_args[@]}" io::log_h1 "Packaging and Uploading Driver" diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index 2524de72a4..abb968c31a 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -82,6 +82,19 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -mapfile -t ctest_args < <(ctest::common_args) -io::run env -C cmake-out ctest "${ctest_args[@]}" +# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +# mapfile -t ctest_args < <(ctest::common_args) +# io::run env -C cmake-out ctest "${ctest_args[@]}" + +SO_FILE_PATH="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +GCS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc" + +if [[ -f "$SO_FILE_PATH" ]]; then + echo "Uploading $SO_FILE_PATH to $GCS_BUCKET" + gsutil cp "$SO_FILE_PATH" "$GCS_BUCKET" +else + echo "Error: $SO_FILE_PATH not found. Upload skipped." + echo "Listing contents of cmake-out directory for debugging:" + ls -R cmake-out + exit 1 +fi \ No newline at end of file diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 33e5f46c3f..6007e7512a 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:22.04 +FROM ubuntu:18.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ + apt-get --no-install-recommends install -y \ + software-properties-common gnupg2 && \ + add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ # Dependency for arrow bison \ - clang-12 \ - lld-12 \ cmake \ curl \ # Dependency for arrow @@ -32,10 +34,8 @@ RUN apt-get update && \ git \ gcc \ g++ \ - # Required by Ubsan in Ubuntu 22.04 - libunwind-12-dev \ - libc++-12-dev \ - libc++abi-12-dev \ + gcc-11 \ + g++-11 \ libcurl4-openssl-dev \ # Needed to use autoreconf libltdl-dev \ @@ -50,9 +50,7 @@ RUN apt-get update && \ # Needed to use autoreconf perl \ pkg-config \ - python3 \ - python3-dev \ - python3-pip \ + libffi-dev \ tar \ unzip \ zip \ @@ -60,21 +58,14 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https \ - clang-tidy-12 - -# Needed for the existing driver v3.1.2.1004+ -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 + apt-transport-https -# Set clang as default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -ENV CC=clang -ENV CXX=clang++ +ENV CC=gcc +ENV CXX=g++ +RUN ln -s /usr/bin/make /usr/bin/gmake # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -83,14 +74,22 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -# clang-tidy-cache needs python -RUN update-alternatives --install /usr/bin/python python $(which python3) 10 +RUN echo "ninja version: " && ninja --version +RUN echo "g++ version: " && g++ --version +RUN echo "cmake version: " && cmake --version +RUN echo "Glibc version" && ldd --version -COPY ./requirements.txt /var/tmp/ci/requirements.txt -WORKDIR /var/tmp/downloads -RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ - then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi -RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt +WORKDIR /usr/src +RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ + tar -xzf Python-3.10.14.tgz && \ + cd Python-3.10.14 && \ + ./configure --with-ensurepip=install && \ + make -j$(nproc) \ + && make altinstall + +# clang-tidy-cache needs python +RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ + ln -sf /usr/local/bin/python3.10 /usr/bin/python # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded @@ -128,6 +127,7 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. + COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini @@ -137,11 +137,3 @@ COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.goog COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini - -# glibc 2.17 or later -RUN echo 'Installing glibc...' -RUN apt-get install -y --no-install-recommends libc6 -RUN echo 'Verifying glibc version...' -RUN dpkg -l libc6 -RUN if [ $(ldd --version | grep GLIBC | awk '{print $5}') -lt 2.17 ] ; \ - then echo 'glibc version is < 2.17: exiting...' ; exit 1 ; fi From ef2d214bccf1125a6143c0159dd823a0766c97fa Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 13 May 2026 15:27:15 +0530 Subject: [PATCH 13/32] added debug prints --- google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc index 701ba5fade..41127add34 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc @@ -254,7 +254,7 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { req.configuration.query.parameter_mode = "NAMED"; } } - + LOG(INFO) << "insert positional_pattern after: sectwo: "; std::vector combined_properties = conn_handle.GetDsn().connection_properties; From eb5eb555eef8c05dbdbecef256c09750b0dae539 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Thu, 14 May 2026 11:15:23 +0530 Subject: [PATCH 14/32] hide the implementation --- google/cloud/odbc/bq_driver.cmake | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/google/cloud/odbc/bq_driver.cmake b/google/cloud/odbc/bq_driver.cmake index beebb2c5fc..6ac6d8f371 100644 --- a/google/cloud/odbc/bq_driver.cmake +++ b/google/cloud/odbc/bq_driver.cmake @@ -230,11 +230,21 @@ if (UNIX AND NOT APPLE) "-static-libstdc++" "-static-libgcc") endif () endif () -set_target_properties( - google_cloud_odbc_bq_driver - PROPERTIES EXPORT_NAME google-cloud-odbc::bq-driver - VERSION "${PROJECT_VERSION}" - SOVERSION "${PROJECT_VERSION_MAJOR}") +set_target_properties(google_cloud_odbc_bq_driver PROPERTIES + EXPORT_NAME google-cloud-odbc::bq-driver + VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}" + # This prevents your internal libstdc++ symbols from being visible to HANA + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) + +# Force the linker to hide everything except the ODBC entry points +if (UNIX AND NOT APPLE) + target_link_options(google_cloud_odbc_bq_driver PRIVATE + "-Wl,--exclude-libs,ALL" + ) +endif() add_library(google-cloud-odbc::bq-driver ALIAS google_cloud_odbc_bq_driver) From b66c4ad12526ae2285770710ac9583e081bc190e Mon Sep 17 00:00:00 2001 From: Neeraj Dwivedi Date: Fri, 15 May 2026 11:47:26 +0530 Subject: [PATCH 15/32] revert WriteWideToWireBuffer function --- .../bq_driver/internal/data_translation.cc | 55 +++++----- .../bq_driver/internal/odbc_type_utils.cc | 16 ++- .../odbc/bq_driver/internal/odbc_type_utils.h | 23 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 14 --- google/cloud/odbc/bq_driver/internal/utils.h | 10 -- google/cloud/odbc/bq_driver/odbc_api.cc | 102 +++++++++--------- 6 files changed, 97 insertions(+), 123 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index ce190a616f..b43f3b8121 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -1007,25 +1007,26 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { *res_len = k_timestamp_src_len * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, k_timestamp_src_len); - write_terminator(k_timestamp_src_len); + std::memcpy(dest, wstr_data.data(), + (k_timestamp_src_len) * WireWcharSize()); + dest[k_timestamp_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); - write_terminator(wchar_capacity - 1); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * WireWcharSize()); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1184,24 +1185,25 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_datetime_src_len) { if (res_len) { *res_len = k_datetime_src_len * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, k_datetime_src_len); - write_terminator(k_datetime_src_len); + std::memcpy(dest, wstr_data.data(), + (k_datetime_src_len) * WireWcharSize()); + dest[k_datetime_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, wchar_capacity); - write_terminator(wchar_capacity - 1); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * WireWcharSize()); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -2043,17 +2045,14 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, std::wstring const& utf16_value = utf16_str.GetValue(); size_t const required_size = utf16_value.length() * WireWcharSize(); - auto write_terminator = [&](size_t char_index) { - auto* p = static_cast(dest_data.buf) + - (char_index * WireWcharSize()); - std::memset(p, 0, WireWcharSize()); - }; + auto* buffer = reinterpret_cast(dest_data.buf); // Handle truncation if buffer is insufficient if (static_cast(dest_data.buflen) < required_size) { size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; - WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy); - write_terminator(num_chars_to_copy); + std::memcpy(buffer, utf16_value.data(), + num_chars_to_copy * WireWcharSize()); + buffer[num_chars_to_copy] = L'\0'; if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; @@ -2061,10 +2060,12 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - WriteWideToWireBuffer(utf16_value, dest_data.buf, utf16_value.size()); + for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { + buffer[i] = static_cast(utf16_str.GetValue()[i]); + } size_t buffer_chars = dest_data.buflen / WireWcharSize(); if (utf16_value.size() < buffer_chars) { - write_terminator(utf16_value.size()); + buffer[utf16_value.size()] = L'\0'; } // Set output length diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 13d235d349..7755e4b9dc 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -46,24 +46,22 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t const wire_sz = WireWcharSize(); + std::vector wstr_data(wstr.begin(), wstr.end()); + wstr_data.emplace_back(L'\0'); - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + (char_index * wire_sz); - std::memset(p, 0, wire_sz); - }; - + auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { *res_len = char_len * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, char_len); - write_terminator(char_len); + std::memcpy(dest, wstr_data.data(), (char_len) * wire_sz); + dest[char_len] = L'\0'; } else if (buffer_length > whole_digits_count) { if (res_len) { *res_len = buffer_length * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, buffer_length); - write_terminator(buffer_length - 1); + std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index c7515445d3..3423e1d354 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -181,18 +181,10 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); - - // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset - // `byte_off` in dest_buf. - auto write_terminator = [&](SQLLEN char_index) { - auto* p = static_cast(dest_buf) + (char_index * wire_sz); - std::memset(p, 0, wire_sz); - }; - + size_t const wire_sz = WireWcharSize(); if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - write_terminator(0); + reinterpret_cast(dest_buf)[0] = L'\0'; } if (res_len) { *res_len = 0; @@ -200,18 +192,21 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } + std::vector wstr_data(wstr.begin(), wstr.end()); + + auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { *res_len = src_len * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, src_len); - write_terminator(src_len); + std::memcpy(dest, wstr_data.data(), (src_len) * wire_sz); + dest[src_len] = L'\0'; } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { *res_len = buffer_length * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, buffer_length); - write_terminator(buffer_length - 1); + std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 85cf5b9045..004c893dd0 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -62,20 +62,6 @@ size_t WireWcharSize() { return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); } -void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { - if (count > src.size()) count = src.size(); - if (IsRuntimeWireUtf16Le()) { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); - } - } else { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); - } - } -} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 26c123f597..673e1fbb4b 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -245,16 +245,6 @@ bool IsRuntimeWireUtf16Le(); // for that purpose. size_t WireWcharSize(); -// Copies up to `count` wide characters from `src` (a std::wstring whose -// element width is wchar_t — 4 bytes on Linux/macOS, 2 bytes on Windows) -// into `dest` using the wire format. On the UTF-16LE-on-wire path each -// wchar_t is narrowed to 16 bits — fine for ASCII / BMP content (which -// covers ODBC metadata strings); supplementary-plane characters would be -// truncated, but the same is true today for any code that does -// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement -// for that pattern. -void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count); - std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); bool IsDiagIdentifierString(SQLSMALLINT DiagIdentifier); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 710d122fbd..4e6c68e124 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -66,7 +66,6 @@ using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::WireWcharSize; -using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -296,7 +295,7 @@ SQLRETURN SQL_API SQLDriverConnectW( } size_t buf_chars = static_cast(outConnectionStringBufferLen); size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); + outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); auto* term = static_cast(static_cast(outConnectionString)) + (to_copy * WireWcharSize()); std::memset(term, 0, WireWcharSize()); @@ -403,8 +402,9 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, } std::memset(outConnectionString, '\0', outConnectionStringBufferLen * WireWcharSize()); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, - utf16_out_conn_str->size()); + std::memcpy((SQLWCHAR*)outConnectionString, + ToSqlWChar(utf16_out_conn_str->data()), + utf16_out_conn_str->size() * WireWcharSize()); } return rc; @@ -651,19 +651,15 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_info_val->begin(), + utf16_info_val->end()); + sql_w_str.emplace_back(L'\0'); + std::size_t bytes_available = static_cast(infoValueBufferLen); - // +1 to leave room for terminator within bytes_available, then - // shrink-to-fit at the wire char boundary. - std::size_t chars_available = bytes_available / WireWcharSize(); - std::size_t chars_to_copy = - std::min(utf16_info_val->size(), chars_available); - WriteWideToWireBuffer(*utf16_info_val, infoValue, chars_to_copy); - if (chars_to_copy < chars_available) { - auto* term = static_cast(infoValue) + - (chars_to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); - } + std::size_t bytes_to_copy = + std::min(sql_w_str.size() * WireWcharSize(), bytes_available); + std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); } } else { if (info_val_buffer_len > 0) { @@ -924,10 +920,14 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - size_t char_count = wcslen(updated_out_attr_status->data()); - *valueStringLen = char_count * WireWcharSize(); + *valueStringLen = + wcslen(updated_out_attr_status->data()) * WireWcharSize(); + std::vector sql_w_str( + updated_out_attr_status->c_str(), + updated_out_attr_status->c_str() + *valueStringLen); + sql_w_str.emplace_back(L'\0'); std::memset(value, '\0', valueBufferLen); - WriteWideToWireBuffer(*updated_out_attr_status, value, char_count); + std::memcpy(value, sql_w_str.data(), sql_w_str.size()); } return rc; @@ -1159,10 +1159,13 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - size_t char_count = utf16_out_desc_val->size(); - out_desc_val_string_len = char_count * WireWcharSize(); + out_desc_val_string_len = + wcslen(utf16_out_desc_val->data()) * WireWcharSize(); + std::vector sql_w_str(utf16_out_desc_val->begin(), + utf16_out_desc_val->end()); + sql_w_str.emplace_back(L'\0'); std::memset(outDescValue, '\0', outDescValueBufferLen); - WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, char_count); + std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1531,11 +1534,11 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_cur_name, cursorName, - utf16_cur_name->size()); - auto* term = static_cast(static_cast(cursorName)) + - (utf16_cur_name->size() * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + std::vector sql_w_str(utf16_cur_name->begin(), + utf16_cur_name->end()); + sql_w_str.emplace_back(L'\0'); + std::memcpy(cursorName, sql_w_str.data(), + (sql_w_str.size() + 1) * WireWcharSize()); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2108,16 +2111,14 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const buf_chars = - static_cast(characterAttributeBufferLen) / WireWcharSize(); - size_t const chars_to_copy = std::min(wstr.size(), buf_chars); - - WriteWideToWireBuffer(wstr, characterAttribute, chars_to_copy); - if (characterAttributeBufferLen >= - static_cast(WireWcharSize())) { - auto* term = static_cast(characterAttribute) + - (chars_to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + size_t const bytes_to_copy = + std::min(static_cast(characterAttributeBufferLen), + wstr.size() * WireWcharSize()); + + std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); + if (characterAttributeBufferLen >= WireWcharSize()) { + SQLWCHAR* wchar_buf = static_cast(characterAttribute); + wchar_buf[bytes_to_copy / WireWcharSize()] = 0; } character_attribute_string_len = static_cast(wstr.size()); @@ -2292,9 +2293,12 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_col_name->begin(), + utf16_col_name->end()); + sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - WriteWideToWireBuffer(*utf16_col_name, columnName, - static_cast(column_name_string_len)); + std::memcpy(columnName, sql_w_str.data(), + column_name_string_len * WireWcharSize()); } if (columnNameLen) { @@ -2499,13 +2503,13 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - size_t char_count = updated_out_diag_info_status->size(); - diag_info_str_len = char_count * WireWcharSize(); - WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, - char_count); - auto* term = static_cast(diagInfo) + - (char_count * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); + diag_info_str_len = + wcslen(updated_out_diag_info_status->data()) * WireWcharSize(); + std::vector sql_w_str( + updated_out_diag_info_status->c_str(), + updated_out_diag_info_status->c_str() + diag_info_str_len); + sql_w_str.emplace_back(L'\0'); + std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2591,8 +2595,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_sql_state, sqlState, - utf16_sql_state->size()); + std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), + utf16_sql_state->size() * WireWcharSize()); } if (messageText && message_text_buffer_len > 0) { @@ -2602,8 +2606,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - WriteWideToWireBuffer(*utf16_msg_txt, messageText, - utf16_msg_txt->size()); + std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), + utf16_msg_txt->size() * WireWcharSize()); } if (messageTextLen) *messageTextLen = message_text_buffer_len; From fbc0d77d7ce77af1621f8c1a8eaf0776f0326bb9 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 15 May 2026 12:09:12 +0530 Subject: [PATCH 16/32] removed more code --- google/cloud/odbc/bq_driver/odbc_api.cc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 4e6c68e124..67dc067882 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -281,8 +281,7 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString && - outConnectionStringBufferLen > 0) { + if (SQL_SUCCEEDED(rc) && outConnectionString) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -293,13 +292,7 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - size_t buf_chars = static_cast(outConnectionStringBufferLen); - size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); - auto* term = static_cast(static_cast(outConnectionString)) + - (to_copy * WireWcharSize()); - std::memset(term, 0, WireWcharSize()); - out_conn_str_len = static_cast(to_copy); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; From 230914b57f0501d0634af36352c3de13bb7859b7 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 15 May 2026 12:09:29 +0530 Subject: [PATCH 17/32] Revert "hide the implementation" This reverts commit 12da68376f9722ddbbdfbca3c1922c6e1089ec13. --- google/cloud/odbc/bq_driver.cmake | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/google/cloud/odbc/bq_driver.cmake b/google/cloud/odbc/bq_driver.cmake index 6ac6d8f371..beebb2c5fc 100644 --- a/google/cloud/odbc/bq_driver.cmake +++ b/google/cloud/odbc/bq_driver.cmake @@ -230,21 +230,11 @@ if (UNIX AND NOT APPLE) "-static-libstdc++" "-static-libgcc") endif () endif () -set_target_properties(google_cloud_odbc_bq_driver PROPERTIES - EXPORT_NAME google-cloud-odbc::bq-driver - VERSION "${PROJECT_VERSION}" - SOVERSION "${PROJECT_VERSION_MAJOR}" - # This prevents your internal libstdc++ symbols from being visible to HANA - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON -) - -# Force the linker to hide everything except the ODBC entry points -if (UNIX AND NOT APPLE) - target_link_options(google_cloud_odbc_bq_driver PRIVATE - "-Wl,--exclude-libs,ALL" - ) -endif() +set_target_properties( + google_cloud_odbc_bq_driver + PROPERTIES EXPORT_NAME google-cloud-odbc::bq-driver + VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}") add_library(google-cloud-odbc::bq-driver ALIAS google_cloud_odbc_bq_driver) From 6a4cab1c16df9f66e99c70be6d4318cbae1a30bd Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 15 May 2026 14:52:46 +0530 Subject: [PATCH 18/32] Reapply "revert the pipeline changes" This reverts commit a5c1842ef4096ef21480d7eadb6d73ac0c1c6f3a. --- ci/cloudbuild/builds/bq-driver-release.sh | 6 +- .../integration-production-bq-driver-dm.sh | 19 +---- .../ubuntu-22.04-install.Dockerfile | 70 +++++++++++-------- 3 files changed, 45 insertions(+), 50 deletions(-) diff --git a/ci/cloudbuild/builds/bq-driver-release.sh b/ci/cloudbuild/builds/bq-driver-release.sh index 9316235856..a18385d934 100755 --- a/ci/cloudbuild/builds/bq-driver-release.sh +++ b/ci/cloudbuild/builds/bq-driver-release.sh @@ -81,9 +81,9 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -# mapfile -t ctest_args < <(ctest::common_args) -# io::run env -C cmake-out ctest "${ctest_args[@]}" +cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +mapfile -t ctest_args < <(ctest::common_args) +io::run env -C cmake-out ctest "${ctest_args[@]}" io::log_h1 "Packaging and Uploading Driver" diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index abb968c31a..2524de72a4 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -82,19 +82,6 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # Copy the roots.pem file to the .so directory to run test cases. -# cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" -# mapfile -t ctest_args < <(ctest::common_args) -# io::run env -C cmake-out ctest "${ctest_args[@]}" - -SO_FILE_PATH="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" -GCS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc" - -if [[ -f "$SO_FILE_PATH" ]]; then - echo "Uploading $SO_FILE_PATH to $GCS_BUCKET" - gsutil cp "$SO_FILE_PATH" "$GCS_BUCKET" -else - echo "Error: $SO_FILE_PATH not found. Upload skipped." - echo "Listing contents of cmake-out directory for debugging:" - ls -R cmake-out - exit 1 -fi \ No newline at end of file +cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" +mapfile -t ctest_args < <(ctest::common_args) +io::run env -C cmake-out ctest "${ctest_args[@]}" diff --git a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile index 6007e7512a..33e5f46c3f 100644 --- a/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile +++ b/ci/cloudbuild/dockerfiles/ubuntu-22.04-install.Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,20 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:18.04 +FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get --no-install-recommends install -y \ - software-properties-common gnupg2 && \ - add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ - apt-get update && \ apt-get --no-install-recommends install -y \ automake \ autotools-dev \ build-essential \ # Dependency for arrow bison \ + clang-12 \ + lld-12 \ cmake \ curl \ # Dependency for arrow @@ -34,8 +32,10 @@ RUN apt-get update && \ git \ gcc \ g++ \ - gcc-11 \ - g++-11 \ + # Required by Ubsan in Ubuntu 22.04 + libunwind-12-dev \ + libc++-12-dev \ + libc++abi-12-dev \ libcurl4-openssl-dev \ # Needed to use autoreconf libltdl-dev \ @@ -50,7 +50,9 @@ RUN apt-get update && \ # Needed to use autoreconf perl \ pkg-config \ - libffi-dev \ + python3 \ + python3-dev \ + python3-pip \ tar \ unzip \ zip \ @@ -58,14 +60,21 @@ RUN apt-get update && \ zlib1g-dev \ apt-utils \ ca-certificates \ - apt-transport-https + apt-transport-https \ + clang-tidy-12 + +# Needed for the existing driver v3.1.2.1004+ +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 +# Set clang as default +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 100 -ENV CC=gcc -ENV CXX=g++ -RUN ln -s /usr/bin/make /usr/bin/gmake +ENV CC=clang +ENV CXX=clang++ # Install modern CMake locally RUN mkdir -p /opt/cmake && \ @@ -74,22 +83,14 @@ RUN mkdir -p /opt/cmake && \ ENV PATH=/opt/cmake/bin:$PATH -RUN echo "ninja version: " && ninja --version -RUN echo "g++ version: " && g++ --version -RUN echo "cmake version: " && cmake --version -RUN echo "Glibc version" && ldd --version - -WORKDIR /usr/src -RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ - tar -xzf Python-3.10.14.tgz && \ - cd Python-3.10.14 && \ - ./configure --with-ensurepip=install && \ - make -j$(nproc) \ - && make altinstall - # clang-tidy-cache needs python -RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 && \ - ln -sf /usr/local/bin/python3.10 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python $(which python3) 10 + +COPY ./requirements.txt /var/tmp/ci/requirements.txt +WORKDIR /var/tmp/downloads +RUN if [ $(ls /var/tmp/ci/requirements.txt | grep -c requirements.txt) -eq 0 ] ; \ + then echo 'Unable to find requirements.txt for python...' ; exit 1 ; fi +RUN pip3 install --require-hashes --no-deps -r /var/tmp/ci/requirements.txt # Install all the direct (and indirect) dependencies for cpp-bigquery-odbc. # Use a different directory for each build, and remove the downloaded @@ -127,7 +128,6 @@ ENV CLOUD_SDK_LOCATION=/usr/local/google-cloud-sdk ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH} ## BEGIN Installs pre-requisites for the ODBC Driver. - COPY ./etc/vcpkg-version.txt /tmp/vcpkg-version.txt COPY ./etc/roots.pem /opt/odbc-driver/roots.pem COPY ./gha/builds/lib/odbc.ini /opt/odbc-driver/odbc.ini @@ -137,3 +137,11 @@ COPY ./gha/builds/lib/google.googlebigqueryodbc.ini /opt/odbc-driver/google.goog COPY ./gha/builds/release/odbc.ini /opt/odbc-driver/odbc_template.ini COPY ./gha/builds/release/odbcinst.ini /opt/odbc-driver/odbcinst_template.ini COPY ./gha/builds/release/googlebigqueryodbc.ini /opt/odbc-driver/googlebigqueryodbc.ini + +# glibc 2.17 or later +RUN echo 'Installing glibc...' +RUN apt-get install -y --no-install-recommends libc6 +RUN echo 'Verifying glibc version...' +RUN dpkg -l libc6 +RUN if [ $(ldd --version | grep GLIBC | awk '{print $5}') -lt 2.17 ] ; \ + then echo 'glibc version is < 2.17: exiting...' ; exit 1 ; fi From 62d90c69dbc00e3bf3612f939a795552e13968d1 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 15 May 2026 17:16:30 +0530 Subject: [PATCH 19/32] improve encoding --- .../bq_driver/internal/data_translation.cc | 46 +++------- .../bq_driver/internal/odbc_type_utils.cc | 11 +-- .../odbc/bq_driver/internal/odbc_type_utils.h | 19 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 16 ++++ google/cloud/odbc/bq_driver/internal/utils.h | 10 +++ google/cloud/odbc/bq_driver/odbc_api.cc | 89 +++++++------------ 6 files changed, 86 insertions(+), 105 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index b43f3b8121..874de093aa 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -1007,26 +1007,20 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { *res_len = k_timestamp_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_timestamp_src_len) * WireWcharSize()); - dest[k_timestamp_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, + static_cast(buffer_length)); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * WireWcharSize()); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, + static_cast(buffer_length)); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1185,25 +1179,19 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_datetime_src_len) { if (res_len) { *res_len = k_datetime_src_len * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (k_datetime_src_len) * WireWcharSize()); - dest[k_datetime_src_len] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, + static_cast(buffer_length)); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * WireWcharSize()); - dest[wchar_capacity - 1] = L'\0'; + WriteWideToWireBuffer(*wstr, dest_buf, + static_cast(buffer_length)); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -2045,28 +2033,18 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, std::wstring const& utf16_value = utf16_str.GetValue(); size_t const required_size = utf16_value.length() * WireWcharSize(); - auto* buffer = reinterpret_cast(dest_data.buf); - // Handle truncation if buffer is insufficient if (static_cast(dest_data.buflen) < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; - std::memcpy(buffer, utf16_value.data(), - num_chars_to_copy * WireWcharSize()); - buffer[num_chars_to_copy] = L'\0'; - + WriteWideToWireBuffer(utf16_value, dest_data.buf, + static_cast(dest_data.buflen)); if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; } LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { - buffer[i] = static_cast(utf16_str.GetValue()[i]); - } - size_t buffer_chars = dest_data.buflen / WireWcharSize(); - if (utf16_value.size() < buffer_chars) { - buffer[utf16_value.size()] = L'\0'; - } + WriteWideToWireBuffer(utf16_value, dest_data.buf, + static_cast(dest_data.buflen)); // Set output length if (dest_data.result_len) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 7755e4b9dc..14f6f379b8 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -46,22 +46,19 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t const wire_sz = WireWcharSize(); - std::vector wstr_data(wstr.begin(), wstr.end()); - wstr_data.emplace_back(L'\0'); - auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { *res_len = char_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (char_len) * wire_sz); - dest[char_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length) * wire_sz); } else if (buffer_length > whole_digits_count) { if (res_len) { *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length) * wire_sz); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 3423e1d354..169fa74fde 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -181,10 +181,14 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); + size_t const wire_sz = WireWcharSize(); + + // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset + // `byte_off` in dest_buf. if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - reinterpret_cast(dest_buf)[0] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length) * wire_sz); } if (res_len) { *res_len = 0; @@ -192,21 +196,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - std::vector wstr_data(wstr.begin(), wstr.end()); - - auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { *res_len = src_len * wire_sz; } - std::memcpy(dest, wstr_data.data(), (src_len) * wire_sz); - dest[src_len] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length) * wire_sz); } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { *res_len = buffer_length * wire_sz; } - std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); - dest[buffer_length - 1] = L'\0'; + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length) * wire_sz); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 004c893dd0..2cf095f239 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -62,6 +62,22 @@ size_t WireWcharSize() { return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); } +void WriteWideToWireBuffer(std::wstring const& src, void* dest, + size_t dest_bytes) { + size_t const wire_char_size = WireWcharSize(); + if (!dest || dest_bytes < wire_char_size) return; + size_t const buf_chars = dest_bytes / wire_char_size; + size_t const count = std::min(src.size(), buf_chars - 1); + if (IsRuntimeWireUtf16Le()) { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) d[i] = static_cast(src[i]); + d[count] = 0; + } else { + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) d[i] = static_cast(src[i]); + d[count] = 0; + } +} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 673e1fbb4b..b92e070117 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -245,6 +245,16 @@ bool IsRuntimeWireUtf16Le(); // for that purpose. size_t WireWcharSize(); +// Copies up to `count` wide characters from `src` (a std::wstring whose +// element width is wchar_t ΓÇö 4 bytes on Linux/macOS, 2 bytes on Windows) +// into `dest` using the wire format. On the UTF-16LE-on-wire path each +// wchar_t is narrowed to 16 bits ΓÇö fine for ASCII / BMP content (which +// covers ODBC metadata strings); supplementary-plane characters would be +// truncated, but the same is true today for any code that does +// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement +// for that pattern. +void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t dest_bytes); + std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); bool IsDiagIdentifierString(SQLSMALLINT DiagIdentifier); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 67dc067882..1f93b5f23f 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -66,6 +66,7 @@ using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -292,7 +293,11 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); + size_t buf_chars = static_cast(outConnectionStringBufferLen); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + buf_chars * WireWcharSize()); + size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); + out_conn_str_len = static_cast(to_copy); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -395,9 +400,9 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, } std::memset(outConnectionString, '\0', outConnectionStringBufferLen * WireWcharSize()); - std::memcpy((SQLWCHAR*)outConnectionString, - ToSqlWChar(utf16_out_conn_str->data()), - utf16_out_conn_str->size() * WireWcharSize()); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + static_cast(outConnectionStringBufferLen) * + WireWcharSize()); } return rc; @@ -644,15 +649,9 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_info_val->begin(), - utf16_info_val->end()); - sql_w_str.emplace_back(L'\0'); - std::size_t bytes_available = static_cast(infoValueBufferLen); - std::size_t bytes_to_copy = - std::min(sql_w_str.size() * WireWcharSize(), bytes_available); - std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); + WriteWideToWireBuffer(*utf16_info_val, infoValue, bytes_available); } } else { if (info_val_buffer_len > 0) { @@ -913,14 +912,11 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - *valueStringLen = - wcslen(updated_out_attr_status->data()) * WireWcharSize(); - std::vector sql_w_str( - updated_out_attr_status->c_str(), - updated_out_attr_status->c_str() + *valueStringLen); - sql_w_str.emplace_back(L'\0'); + size_t char_count = wcslen(updated_out_attr_status->data()); + *valueStringLen = char_count * WireWcharSize(); std::memset(value, '\0', valueBufferLen); - std::memcpy(value, sql_w_str.data(), sql_w_str.size()); + WriteWideToWireBuffer(*updated_out_attr_status, value, + static_cast(valueBufferLen)); } return rc; @@ -1152,13 +1148,11 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - out_desc_val_string_len = - wcslen(utf16_out_desc_val->data()) * WireWcharSize(); - std::vector sql_w_str(utf16_out_desc_val->begin(), - utf16_out_desc_val->end()); - sql_w_str.emplace_back(L'\0'); + size_t char_count = utf16_out_desc_val->size(); + out_desc_val_string_len = char_count * WireWcharSize(); std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); + WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, + static_cast(outDescValueBufferLen)); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1527,11 +1521,9 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_cur_name->begin(), - utf16_cur_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memcpy(cursorName, sql_w_str.data(), - (sql_w_str.size() + 1) * WireWcharSize()); + WriteWideToWireBuffer(*utf16_cur_name, cursorName, + static_cast(cursorNameBufferLen) * + WireWcharSize()); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2104,15 +2096,8 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t const bytes_to_copy = - std::min(static_cast(characterAttributeBufferLen), - wstr.size() * WireWcharSize()); - - std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); - if (characterAttributeBufferLen >= WireWcharSize()) { - SQLWCHAR* wchar_buf = static_cast(characterAttribute); - wchar_buf[bytes_to_copy / WireWcharSize()] = 0; - } + WriteWideToWireBuffer(wstr, characterAttribute, + static_cast(characterAttributeBufferLen)); character_attribute_string_len = static_cast(wstr.size()); } else { @@ -2286,12 +2271,9 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_col_name->begin(), - utf16_col_name->end()); - sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - std::memcpy(columnName, sql_w_str.data(), - column_name_string_len * WireWcharSize()); + WriteWideToWireBuffer(*utf16_col_name, columnName, + static_cast(columnNameBufferLen)); } if (columnNameLen) { @@ -2496,14 +2478,11 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - diag_info_str_len = - wcslen(updated_out_diag_info_status->data()) * WireWcharSize(); - std::vector sql_w_str( - updated_out_diag_info_status->c_str(), - updated_out_diag_info_status->c_str() + diag_info_str_len); - sql_w_str.emplace_back(L'\0'); - std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); - + size_t char_count = updated_out_diag_info_status->size(); + diag_info_str_len = + static_cast(char_count * WireWcharSize()); + WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, + static_cast(diagInfoBufferLen)); } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); } @@ -2588,8 +2567,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), - utf16_sql_state->size() * WireWcharSize()); + WriteWideToWireBuffer(*utf16_sql_state, sqlState, + (utf16_sql_state->size() + 1) * WireWcharSize()); } if (messageText && message_text_buffer_len > 0) { @@ -2599,8 +2578,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), - utf16_msg_txt->size() * WireWcharSize()); + WriteWideToWireBuffer(*utf16_msg_txt, messageText, + static_cast(messageTextBufferLen)); } if (messageTextLen) *messageTextLen = message_text_buffer_len; From e1b416b74b6dcb2e24353988face64caa8e5b5fb Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 19 May 2026 11:32:45 +0530 Subject: [PATCH 20/32] Revert "improve encoding" This reverts commit bccbe20dbed96bc06f20d24d1191cd673bfc88ef. --- .../bq_driver/internal/data_translation.cc | 46 +++++++--- .../bq_driver/internal/odbc_type_utils.cc | 11 ++- .../odbc/bq_driver/internal/odbc_type_utils.h | 19 ++-- google/cloud/odbc/bq_driver/internal/utils.cc | 16 ---- google/cloud/odbc/bq_driver/internal/utils.h | 10 --- google/cloud/odbc/bq_driver/odbc_api.cc | 89 ++++++++++++------- 6 files changed, 105 insertions(+), 86 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index 874de093aa..b43f3b8121 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -1007,20 +1007,26 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { *res_len = k_timestamp_src_len * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, - static_cast(buffer_length)); + std::memcpy(dest, wstr_data.data(), + (k_timestamp_src_len) * WireWcharSize()); + dest[k_timestamp_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, - static_cast(buffer_length)); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * WireWcharSize()); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1179,19 +1185,25 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } + std::vector wstr_data(wstr->begin(), wstr->end()); + wstr_data.emplace_back(L'\0'); + + auto* dest = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / WireWcharSize(); if (wchar_capacity > k_datetime_src_len) { if (res_len) { *res_len = k_datetime_src_len * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, - static_cast(buffer_length)); + std::memcpy(dest, wstr_data.data(), + (k_datetime_src_len) * WireWcharSize()); + dest[k_datetime_src_len] = L'\0'; } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { *res_len = wchar_capacity * WireWcharSize(); } - WriteWideToWireBuffer(*wstr, dest_buf, - static_cast(buffer_length)); + std::memcpy(dest, wstr_data.data(), + (wchar_capacity) * WireWcharSize()); + dest[wchar_capacity - 1] = L'\0'; LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -2033,18 +2045,28 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, std::wstring const& utf16_value = utf16_str.GetValue(); size_t const required_size = utf16_value.length() * WireWcharSize(); + auto* buffer = reinterpret_cast(dest_data.buf); + // Handle truncation if buffer is insufficient if (static_cast(dest_data.buflen) < required_size) { - WriteWideToWireBuffer(utf16_value, dest_data.buf, - static_cast(dest_data.buflen)); + size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; + std::memcpy(buffer, utf16_value.data(), + num_chars_to_copy * WireWcharSize()); + buffer[num_chars_to_copy] = L'\0'; + if (dest_data.result_len) { *dest_data.result_len = dest_data.buflen; } LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - WriteWideToWireBuffer(utf16_value, dest_data.buf, - static_cast(dest_data.buflen)); + for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { + buffer[i] = static_cast(utf16_str.GetValue()[i]); + } + size_t buffer_chars = dest_data.buflen / WireWcharSize(); + if (utf16_value.size() < buffer_chars) { + buffer[utf16_value.size()] = L'\0'; + } // Set output length if (dest_data.result_len) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 14f6f379b8..7755e4b9dc 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -46,19 +46,22 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t const wire_sz = WireWcharSize(); + std::vector wstr_data(wstr.begin(), wstr.end()); + wstr_data.emplace_back(L'\0'); + auto* dest = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { *res_len = char_len * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, - static_cast(buffer_length) * wire_sz); + std::memcpy(dest, wstr_data.data(), (char_len) * wire_sz); + dest[char_len] = L'\0'; } else if (buffer_length > whole_digits_count) { if (res_len) { *res_len = buffer_length * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, - static_cast(buffer_length) * wire_sz); + std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 169fa74fde..3423e1d354 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -181,14 +181,10 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); - - // Writes a wire-format NUL terminator (1, 2, or 4 bytes) at byte offset - // `byte_off` in dest_buf. + size_t const wire_sz = WireWcharSize(); if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - WriteWideToWireBuffer(wstr, dest_buf, - static_cast(buffer_length) * wire_sz); + reinterpret_cast(dest_buf)[0] = L'\0'; } if (res_len) { *res_len = 0; @@ -196,18 +192,21 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } + std::vector wstr_data(wstr.begin(), wstr.end()); + + auto* dest = reinterpret_cast(dest_buf); if (buffer_length > src_len) { if (res_len) { *res_len = src_len * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, - static_cast(buffer_length) * wire_sz); + std::memcpy(dest, wstr_data.data(), (src_len) * wire_sz); + dest[src_len] = L'\0'; } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { *res_len = buffer_length * wire_sz; } - WriteWideToWireBuffer(wstr, dest_buf, - static_cast(buffer_length) * wire_sz); + std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); + dest[buffer_length - 1] = L'\0'; status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 2cf095f239..004c893dd0 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -62,22 +62,6 @@ size_t WireWcharSize() { return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); } -void WriteWideToWireBuffer(std::wstring const& src, void* dest, - size_t dest_bytes) { - size_t const wire_char_size = WireWcharSize(); - if (!dest || dest_bytes < wire_char_size) return; - size_t const buf_chars = dest_bytes / wire_char_size; - size_t const count = std::min(src.size(), buf_chars - 1); - if (IsRuntimeWireUtf16Le()) { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) d[i] = static_cast(src[i]); - d[count] = 0; - } else { - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) d[i] = static_cast(src[i]); - d[count] = 0; - } -} #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index b92e070117..673e1fbb4b 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -245,16 +245,6 @@ bool IsRuntimeWireUtf16Le(); // for that purpose. size_t WireWcharSize(); -// Copies up to `count` wide characters from `src` (a std::wstring whose -// element width is wchar_t ΓÇö 4 bytes on Linux/macOS, 2 bytes on Windows) -// into `dest` using the wire format. On the UTF-16LE-on-wire path each -// wchar_t is narrowed to 16 bits ΓÇö fine for ASCII / BMP content (which -// covers ODBC metadata strings); supplementary-plane characters would be -// truncated, but the same is true today for any code that does -// `std::vector(wstr.begin(), wstr.end())`. Drop-in replacement -// for that pattern. -void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t dest_bytes); - std::wstring SQLWcharToWstring(const SQLWCHAR* in_str); bool IsDiagIdentifierString(SQLSMALLINT DiagIdentifier); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 1f93b5f23f..67dc067882 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -66,7 +66,6 @@ using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::WireWcharSize; -using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -293,11 +292,7 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - size_t buf_chars = static_cast(outConnectionStringBufferLen); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, - buf_chars * WireWcharSize()); - size_t to_copy = std::min(utf16_out_conn_str->size(), buf_chars - 1); - out_conn_str_len = static_cast(to_copy); + outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -400,9 +395,9 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, } std::memset(outConnectionString, '\0', outConnectionStringBufferLen * WireWcharSize()); - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, - static_cast(outConnectionStringBufferLen) * - WireWcharSize()); + std::memcpy((SQLWCHAR*)outConnectionString, + ToSqlWChar(utf16_out_conn_str->data()), + utf16_out_conn_str->size() * WireWcharSize()); } return rc; @@ -649,9 +644,15 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_info_val->begin(), + utf16_info_val->end()); + sql_w_str.emplace_back(L'\0'); + std::size_t bytes_available = static_cast(infoValueBufferLen); - WriteWideToWireBuffer(*utf16_info_val, infoValue, bytes_available); + std::size_t bytes_to_copy = + std::min(sql_w_str.size() * WireWcharSize(), bytes_available); + std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); } } else { if (info_val_buffer_len > 0) { @@ -912,11 +913,14 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - size_t char_count = wcslen(updated_out_attr_status->data()); - *valueStringLen = char_count * WireWcharSize(); + *valueStringLen = + wcslen(updated_out_attr_status->data()) * WireWcharSize(); + std::vector sql_w_str( + updated_out_attr_status->c_str(), + updated_out_attr_status->c_str() + *valueStringLen); + sql_w_str.emplace_back(L'\0'); std::memset(value, '\0', valueBufferLen); - WriteWideToWireBuffer(*updated_out_attr_status, value, - static_cast(valueBufferLen)); + std::memcpy(value, sql_w_str.data(), sql_w_str.size()); } return rc; @@ -1148,11 +1152,13 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - size_t char_count = utf16_out_desc_val->size(); - out_desc_val_string_len = char_count * WireWcharSize(); + out_desc_val_string_len = + wcslen(utf16_out_desc_val->data()) * WireWcharSize(); + std::vector sql_w_str(utf16_out_desc_val->begin(), + utf16_out_desc_val->end()); + sql_w_str.emplace_back(L'\0'); std::memset(outDescValue, '\0', outDescValueBufferLen); - WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, - static_cast(outDescValueBufferLen)); + std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1521,9 +1527,11 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_cur_name, cursorName, - static_cast(cursorNameBufferLen) * - WireWcharSize()); + std::vector sql_w_str(utf16_cur_name->begin(), + utf16_cur_name->end()); + sql_w_str.emplace_back(L'\0'); + std::memcpy(cursorName, sql_w_str.data(), + (sql_w_str.size() + 1) * WireWcharSize()); } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2096,8 +2104,15 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - WriteWideToWireBuffer(wstr, characterAttribute, - static_cast(characterAttributeBufferLen)); + size_t const bytes_to_copy = + std::min(static_cast(characterAttributeBufferLen), + wstr.size() * WireWcharSize()); + + std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); + if (characterAttributeBufferLen >= WireWcharSize()) { + SQLWCHAR* wchar_buf = static_cast(characterAttribute); + wchar_buf[bytes_to_copy / WireWcharSize()] = 0; + } character_attribute_string_len = static_cast(wstr.size()); } else { @@ -2271,9 +2286,12 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } + std::vector sql_w_str(utf16_col_name->begin(), + utf16_col_name->end()); + sql_w_str.emplace_back(L'\0'); std::memset(columnName, '\0', columnNameBufferLen); - WriteWideToWireBuffer(*utf16_col_name, columnName, - static_cast(columnNameBufferLen)); + std::memcpy(columnName, sql_w_str.data(), + column_name_string_len * WireWcharSize()); } if (columnNameLen) { @@ -2478,11 +2496,14 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - size_t char_count = updated_out_diag_info_status->size(); - diag_info_str_len = - static_cast(char_count * WireWcharSize()); - WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, - static_cast(diagInfoBufferLen)); + diag_info_str_len = + wcslen(updated_out_diag_info_status->data()) * WireWcharSize(); + std::vector sql_w_str( + updated_out_diag_info_status->c_str(), + updated_out_diag_info_status->c_str() + diag_info_str_len); + sql_w_str.emplace_back(L'\0'); + std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); + } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); } @@ -2567,8 +2588,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_sql_state, sqlState, - (utf16_sql_state->size() + 1) * WireWcharSize()); + std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), + utf16_sql_state->size() * WireWcharSize()); } if (messageText && message_text_buffer_len > 0) { @@ -2578,8 +2599,8 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } std::memset(messageText, '\0', messageTextBufferLen); - WriteWideToWireBuffer(*utf16_msg_txt, messageText, - static_cast(messageTextBufferLen)); + std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), + utf16_msg_txt->size() * WireWcharSize()); } if (messageTextLen) *messageTextLen = message_text_buffer_len; From 16114cae6f172cd9ac871942aec568fac56b24f0 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 19 May 2026 11:33:03 +0530 Subject: [PATCH 21/32] Revert "added debug prints" This reverts commit edbd250c02f8c581c0c23d010bd14fc16885f408. --- google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc index 41127add34..701ba5fade 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.cc @@ -254,7 +254,7 @@ StatusRecord StatementHandle::PrepareQuery(std::string const& query) { req.configuration.query.parameter_mode = "NAMED"; } } - LOG(INFO) << "insert positional_pattern after: sectwo: "; + std::vector combined_properties = conn_handle.GetDsn().connection_properties; From e7374f4805ce32793dad54fb7b644e8c23df118a Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Mon, 18 May 2026 12:27:57 +0530 Subject: [PATCH 22/32] fix windfows --- .../bq_driver/internal/data_translation.cc | 152 +++++++++------- .../bq_driver/internal/odbc_type_utils.cc | 22 +-- .../odbc/bq_driver/internal/odbc_type_utils.h | 81 +++++++-- google/cloud/odbc/bq_driver/internal/utils.cc | 5 +- google/cloud/odbc/bq_driver/odbc_api.cc | 168 +++++++++++------- 5 files changed, 275 insertions(+), 153 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index b43f3b8121..033784a4e2 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -1007,26 +1007,27 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(wstr.GetValue(), &wire_char_count); + size_t const wire_sz = WireWcharSize(); + auto* dest8 = reinterpret_cast(dest_buf); + SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = k_timestamp_src_len * WireWcharSize(); + *res_len = static_cast(wire_char_count * wire_sz); } - std::memcpy(dest, wstr_data.data(), - (k_timestamp_src_len) * WireWcharSize()); - dest[k_timestamp_src_len] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { - *res_len = wchar_capacity * WireWcharSize(); + *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * WireWcharSize()); - dest[wchar_capacity - 1] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), + static_cast(wchar_capacity - 1) * wire_sz); + std::memset(dest8 + static_cast(wchar_capacity - 1) * wire_sz, + 0, wire_sz); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1185,25 +1186,26 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - std::vector wstr_data(wstr->begin(), wstr->end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = reinterpret_cast(dest_buf); - SQLLEN wchar_capacity = buffer_length / WireWcharSize(); + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(wstr.GetValue(), &wire_char_count); + size_t const wire_sz = WireWcharSize(); + auto* dest8 = reinterpret_cast(dest_buf); + SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = k_datetime_src_len * WireWcharSize(); + *res_len = static_cast(wire_char_count * wire_sz); } - std::memcpy(dest, wstr_data.data(), - (k_datetime_src_len) * WireWcharSize()); - dest[k_datetime_src_len] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { - *res_len = wchar_capacity * WireWcharSize(); + *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), - (wchar_capacity) * WireWcharSize()); - dest[wchar_capacity - 1] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), + static_cast(wchar_capacity - 1) * wire_sz); + std::memset(dest8 + static_cast(wchar_capacity - 1) * wire_sz, + 0, wire_sz); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -2028,53 +2030,65 @@ StatusRecord ConvertBytesToChar(DSValue const& conn_val, // This func converts a vector of SQLCHAR bytes to a UTF-16 wchar_t string, // ensuring proper truncation handling. StatusRecord ConvertBytesToWChar(DSValue const& conn_val, - DataBuffer& dest_data) { - StatusRecord status_record = StatusRecord::Ok(); - - // Convert input bytes to a UTF-8 string - std::string utf8_str(conn_val.begin(), conn_val.end()); - - // Convert UTF-8 to UTF-16 - StatusRecordOr utf16_str = Utf8ToUtf16(utf8_str); - if (!utf16_str.Ok()) { - LOG(ERROR) << "ConvertBytesToWChar:: UTF-8 to UTF-16 conversion failed: "; - return StatusRecord{SQLStates::k_01004(), - "UTF-8 to UTF-16 conversion failed."}; - } - - std::wstring const& utf16_value = utf16_str.GetValue(); - size_t const required_size = utf16_value.length() * WireWcharSize(); - - auto* buffer = reinterpret_cast(dest_data.buf); - - // Handle truncation if buffer is insufficient - if (static_cast(dest_data.buflen) < required_size) { - size_t num_chars_to_copy = (dest_data.buflen / WireWcharSize()) - 1; - std::memcpy(buffer, utf16_value.data(), - num_chars_to_copy * WireWcharSize()); - buffer[num_chars_to_copy] = L'\0'; + DataBuffer& dest_data) { +StatusRecord status_record = StatusRecord::Ok(); + +// Convert input bytes to a UTF-8 string +std::string utf8_str(conn_val.begin(), conn_val.end()); + +// Convert UTF-8 to UTF-16 +StatusRecordOr utf16_str = Utf8ToUtf16(utf8_str); +if (!utf16_str.Ok()) { +LOG(ERROR) << "ConvertBytesToWChar:: UTF-8 to UTF-16 conversion failed: "; +return StatusRecord{SQLStates::k_01004(), +"UTF-8 to UTF-16 conversion failed."}; +} - if (dest_data.result_len) { - *dest_data.result_len = dest_data.buflen; - } - LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; - return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; - } - for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) { - buffer[i] = static_cast(utf16_str.GetValue()[i]); - } - size_t buffer_chars = dest_data.buflen / WireWcharSize(); - if (utf16_value.size() < buffer_chars) { - buffer[utf16_value.size()] = L'\0'; - } +std::wstring const& utf16_value = utf16_str.GetValue(); + +// Use WstrToWireBytes to produce correctly-encoded output. When +// IsRuntimeWireUtf16Le() the 4-byte wchar_t values are narrowed to +// uint16_t so memcpy does not embed zero high bytes as spurious null +// terminators. +size_t wire_char_count = 0; +std::vector wire_bytes = WstrToWireBytes(utf16_value, &wire_char_count); +size_t const wire_sz = WireWcharSize(); +size_t const required_size = wire_char_count * wire_sz; + +auto* dest8 = reinterpret_cast(dest_data.buf); + +// Handle truncation if buffer cannot hold the data (even without null). +// An exact fit (buflen == required_size) is treated as success; the caller's +// buffer is presumed to be zeroed or the null terminator is not required. +if (static_cast(dest_data.buflen) < required_size) { +size_t num_chars_to_copy = dest_data.buflen / wire_sz; +if (num_chars_to_copy > 0) { +num_chars_to_copy--; // leave one slot for the null terminator +std::memcpy(dest8, wire_bytes.data(), num_chars_to_copy * wire_sz); +std::memset(dest8 + num_chars_to_copy * wire_sz, 0, wire_sz); +} +if (dest_data.result_len) { +*dest_data.result_len = required_size; +} +LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; +return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; +} +// Copy data; append null terminator only when there is room for it. +if (static_cast(dest_data.buflen) >= required_size + wire_sz) { +std::memcpy(dest8, wire_bytes.data(), (wire_char_count + 1) * wire_sz); +} else { +// Exact fit: buffer holds the data but has no null terminator slot. +std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); +} - // Set output length - if (dest_data.result_len) { - *dest_data.result_len = utf16_value.size() * WireWcharSize(); - } - return status_record; +// Set output length +if (dest_data.result_len) { +*dest_data.result_len = wire_char_count * wire_sz; +} +return status_record; } + StatusRecord ConvertFromBytesDSValue(DSValue const& src_dsval, DataBuffer& dest_data) { if (!dest_data.buf) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 7755e4b9dc..d89abc7705 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,23 +45,25 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); + size_t wire_char_count = 0; + std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); size_t const wire_sz = WireWcharSize(); - std::vector wstr_data(wstr.begin(), wstr.end()); - wstr_data.emplace_back(L'\0'); - - auto* dest = static_cast(dest_buf); + auto* dest8 = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = char_len * wire_sz; + *res_len = + static_cast(wire_char_count) * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (char_len) * wire_sz); - dest[char_len] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * wire_sz; + *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); - dest[buffer_length - 1] = L'\0'; + std::memcpy(dest8, wire_bytes.data(), + static_cast(buffer_length - 1) * wire_sz); + std::memset(dest8 + static_cast(buffer_length - 1) * wire_sz, 0, + wire_sz); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 3423e1d354..7900fdc919 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -177,14 +177,64 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, return SQL_SUCCESS; } +// Converts a std::wstring to a flat byte array ready to memcpy into a SQLWCHAR +// output buffer using the correct wire encoding. +// +// When IsRuntimeWireUtf16Le() is true (iODBC-built driver loaded by unixODBC), +// each 4-byte wchar_t code point is narrowed to one or two uint16_t code units +// (UTF-16LE; surrogate pairs are generated for code points above U+FFFF). +// Otherwise the raw SQLWCHAR bytes are emitted unchanged. +// +// The returned vector always ends with one wire-format null code unit. +// *out_char_count (optional) receives the number of wire code units written, +// excluding the null terminator. +inline std::vector WstrToWireBytes(std::wstring const& wstr, + size_t* out_char_count = nullptr) { +std::vector bytes; +#if !defined(_WIN32) +if (IsRuntimeWireUtf16Le()) { +std::vector utf16; +utf16.reserve(wstr.size() + 1); +for (wchar_t wc : wstr) { +uint32_t cp = static_cast(wc); +if (cp <= 0xFFFFu) { +utf16.push_back(static_cast(cp)); +} else { +cp -= 0x10000u; +utf16.push_back(static_cast(0xD800u | (cp >> 10))); +utf16.push_back(static_cast(0xDC00u | (cp & 0x3FFu))); +} +} +if (out_char_count) *out_char_count = utf16.size(); +utf16.push_back(0); // null terminator +bytes.resize(utf16.size() * 2); +std::memcpy(bytes.data(), utf16.data(), bytes.size()); +return bytes; +} +#endif +if (out_char_count) *out_char_count = wstr.size(); +bytes.resize((wstr.size() + 1) * sizeof(SQLWCHAR), 0); +// Cast each wchar_t to SQLWCHAR individually. On Linux/unixODBC, +// sizeof(wchar_t)==4 but sizeof(SQLWCHAR)==2, so memcpy of raw wstring +// bytes would pick up only the low byte of each character and treat the +// intervening zero high bytes as null terminators. +auto* sqlwchar_dest = reinterpret_cast(bytes.data()); +for (size_t i = 0; i < wstr.size(); ++i) { +sqlwchar_dest[i] = static_cast(wstr[i]); +} +// Null terminator is already zero from the resize above. +return bytes; +} + inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t const wire_sz = WireWcharSize(); + size_t const wire_sz = WireWcharSize(); if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - reinterpret_cast(dest_buf)[0] = L'\0'; + // Write a wire-format null terminator (2 bytes for UTF-16LE, 4 for UCS-4) + std::memset(dest_buf, 0, wire_sz); } if (res_len) { *res_len = 0; @@ -192,21 +242,28 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - std::vector wstr_data(wstr.begin(), wstr.end()); + // Build a flat byte array in the correct wire encoding. When + // IsRuntimeWireUtf16Le() each 4-byte wchar_t is narrowed to uint16_t so + // that memcpy produces valid UTF-16LE instead of embedding the zero high + // bytes of each UCS-4 code unit as spurious null terminators. + size_t wire_char_count = 0; + std::vector wire_data = WstrToWireBytes(wstr, &wire_char_count); + SQLINTEGER u_src_len = static_cast(wire_char_count); + auto* dest = reinterpret_cast(dest_buf); - auto* dest = reinterpret_cast(dest_buf); - if (buffer_length > src_len) { + if (buffer_length > u_src_len) { if (res_len) { - *res_len = src_len * wire_sz; + *res_len = u_src_len * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (src_len) * wire_sz); - dest[src_len] = L'\0'; - } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { + // Copy data + null terminator (wire_data has u_src_len+1 code units) + std::memcpy(dest, wire_data.data(), (u_src_len + 1) * wire_sz); + } else if (supp_max_len <= buffer_length && buffer_length <= u_src_len) { if (res_len) { - *res_len = buffer_length * wire_sz; + *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest, wstr_data.data(), (buffer_length) * wire_sz); - dest[buffer_length - 1] = L'\0'; + // Copy buffer_length-1 code units, then write null terminator + std::memcpy(dest, wire_data.data(), (buffer_length - 1) * wire_sz); + std::memset(dest + (buffer_length - 1) * wire_sz, 0, wire_sz); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 004c893dd0..f102e33f63 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -800,7 +800,10 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( return StatusRecord{SQLStates::k_HY000(), "Error while converting string to wstring"}; } - utf16Str.push_back(L'\0'); + // MultiByteToWideChar was called with an explicit input length, so + // utf16Length is the character count without a null terminator and + // wstring::size() already reflects the actual character count, + // matching the Linux iconv path behaviour. return utf16Str; #else iconv_t cd = iconv_open(kFromCode.c_str(), "UTF-8"); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 67dc067882..569905a6c6 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -66,6 +66,7 @@ using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WstrToWireBytes; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -393,11 +394,17 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * WireWcharSize()); - std::memcpy((SQLWCHAR*)outConnectionString, - ToSqlWChar(utf16_out_conn_str->data()), - utf16_out_conn_str->size() * WireWcharSize()); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_out_conn_str, &wire_char_count); + std::memset(outConnectionString, '\0', + outConnectionStringBufferLen * WireWcharSize()); + size_t bytes_to_copy = std::min( + wire_bytes.size(), + static_cast(outConnectionStringBufferLen * WireWcharSize())); + std::memcpy(outConnectionString, wire_bytes.data(), bytes_to_copy); + } } return rc; @@ -644,15 +651,14 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_info_val->begin(), - utf16_info_val->end()); - sql_w_str.emplace_back(L'\0'); - + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_info_val, &wire_char_count); std::size_t bytes_available = static_cast(infoValueBufferLen); std::size_t bytes_to_copy = - std::min(sql_w_str.size() * WireWcharSize(), bytes_available); - std::memcpy(infoValue, sql_w_str.data(), bytes_to_copy); + std::min(wire_bytes.size(), bytes_available); + std::memcpy(infoValue, wire_bytes.data(), bytes_to_copy); } } else { if (info_val_buffer_len > 0) { @@ -901,9 +907,10 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, // Handle Unicode conversion of input parameters. // Call to internal common function for SQLGetConnectAttr and // SQLGetConnectAttrW in odbc_connection.h. + SQLINTEGER internal_str_len = 0; rc = ::google::cloud::odbc_bq_driver::SQLGetConnectAttrInternal( - connectionHandle, attribute, updated_attrib_val, valueBufferLen, - valueStringLen); + connectionHandle, attribute, updated_attrib_val, + static_cast(kBufferLength), &internal_str_len); // Handle unicode conversion for attribute string values for output // parameters. if (SQL_SUCCEEDED(rc) && conn_attr.GetAttributeValueType(attribute) == @@ -913,14 +920,19 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - *valueStringLen = - wcslen(updated_out_attr_status->data()) * WireWcharSize(); - std::vector sql_w_str( - updated_out_attr_status->c_str(), - updated_out_attr_status->c_str() + *valueStringLen); - sql_w_str.emplace_back(L'\0'); - std::memset(value, '\0', valueBufferLen); - std::memcpy(value, sql_w_str.data(), sql_w_str.size()); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*updated_out_attr_status, &wire_char_count); + if (valueStringLen) { + *valueStringLen = + static_cast(wire_char_count * WireWcharSize()); + } + std::memset(value, '\0', valueBufferLen); + std::memcpy(value, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(valueBufferLen))); + } } return rc; @@ -1152,13 +1164,17 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - out_desc_val_string_len = - wcslen(utf16_out_desc_val->data()) * WireWcharSize(); - std::vector sql_w_str(utf16_out_desc_val->begin(), - utf16_out_desc_val->end()); - sql_w_str.emplace_back(L'\0'); - std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, sql_w_str.data(), out_desc_val_string_len); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_out_desc_val, &wire_char_count); + out_desc_val_string_len = + static_cast(wire_char_count * WireWcharSize()); + std::memset(outDescValue, '\0', outDescValueBufferLen); + std::memcpy(outDescValue, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(outDescValueBufferLen))); + } } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1239,9 +1255,14 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - std::memset(name, '\0', nameBufferLen * WireWcharSize()); - std::memcpy(name, ToSqlWChar(utf16_name->data()), - name_string_len * WireWcharSize()); + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_name, &wire_char_count); + size_t const max_bytes = + static_cast(nameBufferLen) * WireWcharSize(); + std::memset(name, '\0', max_bytes); + std::memcpy(name, wire_bytes.data(), + std::min(wire_bytes.size(), max_bytes)); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1527,11 +1548,12 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_cur_name->begin(), - utf16_cur_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memcpy(cursorName, sql_w_str.data(), - (sql_w_str.size() + 1) * WireWcharSize()); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_cur_name, &wire_char_count); + std::memcpy(cursorName, wire_bytes.data(), wire_bytes.size()); + } } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2104,16 +2126,20 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; + size_t wire_char_count = 0; + std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); + // Exclude null terminator from bytes_to_copy so we can write it explicitly size_t const bytes_to_copy = std::min(static_cast(characterAttributeBufferLen), - wstr.size() * WireWcharSize()); - - std::memcpy(characterAttribute, wstr.data(), bytes_to_copy); - if (characterAttributeBufferLen >= WireWcharSize()) { - SQLWCHAR* wchar_buf = static_cast(characterAttribute); - wchar_buf[bytes_to_copy / WireWcharSize()] = 0; + wire_char_count * WireWcharSize()); + std::memcpy(characterAttribute, wire_bytes.data(), bytes_to_copy); + if (static_cast(characterAttributeBufferLen) >= WireWcharSize()) { + // Write null terminator right after the copied data + std::memset(reinterpret_cast(characterAttribute) + + bytes_to_copy, 0, WireWcharSize()); } - character_attribute_string_len = static_cast(wstr.size()); + character_attribute_string_len = + static_cast(wire_char_count); } else { std::memcpy(characterAttribute, (SQLPOINTER)updated_character_attrib_val, @@ -2286,12 +2312,17 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - std::vector sql_w_str(utf16_col_name->begin(), - utf16_col_name->end()); - sql_w_str.emplace_back(L'\0'); - std::memset(columnName, '\0', columnNameBufferLen); - std::memcpy(columnName, sql_w_str.data(), - column_name_string_len * WireWcharSize()); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_col_name, &wire_char_count); + // columnNameBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const max_bytes = + static_cast(columnNameBufferLen) * WireWcharSize(); + std::memset(columnName, '\0', max_bytes); + std::memcpy(columnName, wire_bytes.data(), + std::min(wire_bytes.size(), max_bytes)); + } } if (columnNameLen) { @@ -2485,7 +2516,7 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, // in odbc_diagnostics.h. rc = google::cloud::odbc_bq_driver::SQLGetDiagFieldInternal( handleType, handle, recNumber, diagIdentifier, updated_diag_info, - diagInfoBufferLen, &diag_info_str_len); + static_cast(kBufferLength), &diag_info_str_len); // Handle Unicode conversion of output parameters. if (SQL_SUCCEEDED(rc) && diag_info_str_len > 0) { @@ -2496,13 +2527,16 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*updated_out_diag_info_status, &wire_char_count); diag_info_str_len = - wcslen(updated_out_diag_info_status->data()) * WireWcharSize(); - std::vector sql_w_str( - updated_out_diag_info_status->c_str(), - updated_out_diag_info_status->c_str() + diag_info_str_len); - sql_w_str.emplace_back(L'\0'); - std::memcpy(diagInfo, sql_w_str.data(), sql_w_str.size()); + static_cast(wire_char_count * WireWcharSize()); + std::memcpy(diagInfo, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(diagInfoBufferLen))); + } } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2588,8 +2622,12 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - std::memcpy(sqlState, ToSqlWChar(utf16_sql_state->data()), - utf16_sql_state->size() * WireWcharSize()); + { + size_t wire_char_count_state = 0; + std::vector wire_bytes_state = + WstrToWireBytes(*utf16_sql_state, &wire_char_count_state); + std::memcpy(sqlState, wire_bytes_state.data(), wire_bytes_state.size()); + } } if (messageText && message_text_buffer_len > 0) { @@ -2598,9 +2636,17 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_msg_txt) { return utf16_msg_txt.GetCalculatedReturnCode(); } - std::memset(messageText, '\0', messageTextBufferLen); - std::memcpy(messageText, ToSqlWChar(utf16_msg_txt->data()), - utf16_msg_txt->size() * WireWcharSize()); + { + size_t wire_char_count_msg = 0; + std::vector wire_bytes_msg = + WstrToWireBytes(*utf16_msg_txt, &wire_char_count_msg); + // messageTextBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const max_bytes = + static_cast(messageTextBufferLen) * WireWcharSize(); + std::memset(messageText, '\0', max_bytes); + std::memcpy(messageText, wire_bytes_msg.data(), + std::min(wire_bytes_msg.size(), max_bytes)); + } } if (messageTextLen) *messageTextLen = message_text_buffer_len; From d4437acdc42b4f718cc3795a19be7ba8601e8c28 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 19 May 2026 12:45:33 +0530 Subject: [PATCH 23/32] checkers fix --- .../bq_driver/internal/data_translation.cc | 112 ++++----- .../bq_driver/internal/odbc_type_utils.cc | 24 +- .../odbc/bq_driver/internal/odbc_type_utils.h | 70 +++--- google/cloud/odbc/bq_driver/internal/utils.cc | 3 +- google/cloud/odbc/bq_driver/odbc_api.cc | 219 +++++++++--------- .../cloud/odbc/bq_driver/odbc_sql_results.cc | 2 +- 6 files changed, 216 insertions(+), 214 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index 033784a4e2..f62e65899f 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -2030,64 +2030,64 @@ StatusRecord ConvertBytesToChar(DSValue const& conn_val, // This func converts a vector of SQLCHAR bytes to a UTF-16 wchar_t string, // ensuring proper truncation handling. StatusRecord ConvertBytesToWChar(DSValue const& conn_val, - DataBuffer& dest_data) { -StatusRecord status_record = StatusRecord::Ok(); - -// Convert input bytes to a UTF-8 string -std::string utf8_str(conn_val.begin(), conn_val.end()); - -// Convert UTF-8 to UTF-16 -StatusRecordOr utf16_str = Utf8ToUtf16(utf8_str); -if (!utf16_str.Ok()) { -LOG(ERROR) << "ConvertBytesToWChar:: UTF-8 to UTF-16 conversion failed: "; -return StatusRecord{SQLStates::k_01004(), -"UTF-8 to UTF-16 conversion failed."}; -} + DataBuffer& dest_data) { + StatusRecord status_record = StatusRecord::Ok(); -std::wstring const& utf16_value = utf16_str.GetValue(); - -// Use WstrToWireBytes to produce correctly-encoded output. When -// IsRuntimeWireUtf16Le() the 4-byte wchar_t values are narrowed to -// uint16_t so memcpy does not embed zero high bytes as spurious null -// terminators. -size_t wire_char_count = 0; -std::vector wire_bytes = WstrToWireBytes(utf16_value, &wire_char_count); -size_t const wire_sz = WireWcharSize(); -size_t const required_size = wire_char_count * wire_sz; - -auto* dest8 = reinterpret_cast(dest_data.buf); - -// Handle truncation if buffer cannot hold the data (even without null). -// An exact fit (buflen == required_size) is treated as success; the caller's -// buffer is presumed to be zeroed or the null terminator is not required. -if (static_cast(dest_data.buflen) < required_size) { -size_t num_chars_to_copy = dest_data.buflen / wire_sz; -if (num_chars_to_copy > 0) { -num_chars_to_copy--; // leave one slot for the null terminator -std::memcpy(dest8, wire_bytes.data(), num_chars_to_copy * wire_sz); -std::memset(dest8 + num_chars_to_copy * wire_sz, 0, wire_sz); -} -if (dest_data.result_len) { -*dest_data.result_len = required_size; -} -LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; -return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; -} -// Copy data; append null terminator only when there is room for it. -if (static_cast(dest_data.buflen) >= required_size + wire_sz) { -std::memcpy(dest8, wire_bytes.data(), (wire_char_count + 1) * wire_sz); -} else { -// Exact fit: buffer holds the data but has no null terminator slot. -std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); -} + // Convert input bytes to a UTF-8 string + std::string utf8_str(conn_val.begin(), conn_val.end()); -// Set output length -if (dest_data.result_len) { -*dest_data.result_len = wire_char_count * wire_sz; -} -return status_record; -} + // Convert UTF-8 to UTF-16 + StatusRecordOr utf16_str = Utf8ToUtf16(utf8_str); + if (!utf16_str.Ok()) { + LOG(ERROR) << "ConvertBytesToWChar:: UTF-8 to UTF-16 conversion failed: "; + return StatusRecord{SQLStates::k_01004(), + "UTF-8 to UTF-16 conversion failed."}; + } + + std::wstring const& utf16_value = utf16_str.GetValue(); + + // Use WstrToWireBytes to produce correctly-encoded output. When + // IsRuntimeWireUtf16Le() the 4-byte wchar_t values are narrowed to + // uint16_t so memcpy does not embed zero high bytes as spurious null + // terminators. + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(utf16_value, &wire_char_count); + size_t const wire_sz = WireWcharSize(); + size_t const required_size = wire_char_count * wire_sz; + + auto* dest8 = reinterpret_cast(dest_data.buf); + + // Handle truncation if buffer cannot hold the data (even without null). + // An exact fit (buflen == required_size) is treated as success; the caller's + // buffer is presumed to be zeroed or the null terminator is not required. + if (static_cast(dest_data.buflen) < required_size) { + size_t num_chars_to_copy = dest_data.buflen / wire_sz; + if (num_chars_to_copy > 0) { + num_chars_to_copy--; // leave one slot for the null terminator + std::memcpy(dest8, wire_bytes.data(), num_chars_to_copy * wire_sz); + std::memset(dest8 + num_chars_to_copy * wire_sz, 0, wire_sz); + } + if (dest_data.result_len) { + *dest_data.result_len = required_size; + } + LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; + return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; + } + // Copy data; append null terminator only when there is room for it. + if (static_cast(dest_data.buflen) >= required_size + wire_sz) { + std::memcpy(dest8, wire_bytes.data(), (wire_char_count + 1) * wire_sz); + } else { + // Exact fit: buffer holds the data but has no null terminator slot. + std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + } + // Set output length + if (dest_data.result_len) { + *dest_data.result_len = wire_char_count * wire_sz; + } + return status_record; +} StatusRecord ConvertFromBytesDSValue(DSValue const& src_dsval, DataBuffer& dest_data) { @@ -2224,7 +2224,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval, auto status = ConvertRangeToTimestampFormat(src_str); if (!status.ok()) { return status; - } + } } switch (dest_data.type) { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index d89abc7705..8e93533fb7 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,25 +45,25 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t wire_char_count = 0; - std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); + size_t wire_char_count = 0; + std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); size_t const wire_sz = WireWcharSize(); - auto* dest8 = static_cast(dest_buf); + auto* dest8 = static_cast(dest_buf); if (buffer_length > char_len) { if (res_len) { - *res_len = - static_cast(wire_char_count) * static_cast(wire_sz); + *res_len = + static_cast(wire_char_count) * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); - std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); + std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); } else if (buffer_length > whole_digits_count) { if (res_len) { - *res_len = buffer_length * static_cast(wire_sz); + *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), - static_cast(buffer_length - 1) * wire_sz); - std::memset(dest8 + static_cast(buffer_length - 1) * wire_sz, 0, - wire_sz); + std::memcpy(dest8, wire_bytes.data(), + static_cast(buffer_length - 1) * wire_sz); + std::memset(dest8 + static_cast(buffer_length - 1) * wire_sz, 0, + wire_sz); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 7900fdc919..da0319685d 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -15,11 +15,11 @@ #ifndef CPP_BIGQUERY_ODBC_GOOGLE_CLOUD_ODBC_BQ_DRIVER_INTERNAL_ODBC_TYPE_UTILS_H #define CPP_BIGQUERY_ODBC_GOOGLE_CLOUD_ODBC_BQ_DRIVER_INTERNAL_ODBC_TYPE_UTILS_H +#include "google/cloud/odbc/bq_driver/internal/utils.h" #include "google/cloud/odbc/internal/diagnostic_records.h" #include "google/cloud/odbc/internal/sql_state_constants.h" -#include "google/cloud/odbc/bq_driver/internal/utils.h" -#include #include +#include #include #include #include @@ -189,41 +189,41 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, // *out_char_count (optional) receives the number of wire code units written, // excluding the null terminator. inline std::vector WstrToWireBytes(std::wstring const& wstr, - size_t* out_char_count = nullptr) { -std::vector bytes; + size_t* out_char_count = nullptr) { + std::vector bytes; #if !defined(_WIN32) -if (IsRuntimeWireUtf16Le()) { -std::vector utf16; -utf16.reserve(wstr.size() + 1); -for (wchar_t wc : wstr) { -uint32_t cp = static_cast(wc); -if (cp <= 0xFFFFu) { -utf16.push_back(static_cast(cp)); -} else { -cp -= 0x10000u; -utf16.push_back(static_cast(0xD800u | (cp >> 10))); -utf16.push_back(static_cast(0xDC00u | (cp & 0x3FFu))); -} -} -if (out_char_count) *out_char_count = utf16.size(); -utf16.push_back(0); // null terminator -bytes.resize(utf16.size() * 2); -std::memcpy(bytes.data(), utf16.data(), bytes.size()); -return bytes; -} + if (IsRuntimeWireUtf16Le()) { + std::vector utf16; + utf16.reserve(wstr.size() + 1); + for (wchar_t wc : wstr) { + uint32_t cp = static_cast(wc); + if (cp <= 0xFFFFu) { + utf16.push_back(static_cast(cp)); + } else { + cp -= 0x10000u; + utf16.push_back(static_cast(0xD800u | (cp >> 10))); + utf16.push_back(static_cast(0xDC00u | (cp & 0x3FFu))); + } + } + if (out_char_count) *out_char_count = utf16.size(); + utf16.push_back(0); // null terminator + bytes.resize(utf16.size() * 2); + std::memcpy(bytes.data(), utf16.data(), bytes.size()); + return bytes; + } #endif -if (out_char_count) *out_char_count = wstr.size(); -bytes.resize((wstr.size() + 1) * sizeof(SQLWCHAR), 0); -// Cast each wchar_t to SQLWCHAR individually. On Linux/unixODBC, -// sizeof(wchar_t)==4 but sizeof(SQLWCHAR)==2, so memcpy of raw wstring -// bytes would pick up only the low byte of each character and treat the -// intervening zero high bytes as null terminators. -auto* sqlwchar_dest = reinterpret_cast(bytes.data()); -for (size_t i = 0; i < wstr.size(); ++i) { -sqlwchar_dest[i] = static_cast(wstr[i]); -} -// Null terminator is already zero from the resize above. -return bytes; + if (out_char_count) *out_char_count = wstr.size(); + bytes.resize((wstr.size() + 1) * sizeof(SQLWCHAR), 0); + // Cast each wchar_t to SQLWCHAR individually. On Linux/unixODBC, + // sizeof(wchar_t)==4 but sizeof(SQLWCHAR)==2, so memcpy of raw wstring + // bytes would pick up only the low byte of each character and treat the + // intervening zero high bytes as null terminators. + auto* sqlwchar_dest = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < wstr.size(); ++i) { + sqlwchar_dest[i] = static_cast(wstr[i]); + } + // Null terminator is already zero from the resize above. + return bytes; } inline odbc_internal::StatusRecord WStrToOutputBufferResponse( diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index f102e33f63..99cfccc889 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -862,8 +862,7 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( // later call (including the ambiguous ones from SQLTables(catalog="%")) // honors the latch via IsRuntimeWireUtf16Le(). if (sizeof(SQLWCHAR) == 4) { - bool use_utf16le = - g_utf16le_wire_latched.load(std::memory_order_relaxed); + bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); auto const* bytes = reinterpret_cast(in_str); if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 569905a6c6..6ac51c3896 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -64,9 +64,9 @@ using google::cloud::odbc_bq_driver_internal::IsInfoTypeString; using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; -using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::WireWcharSize; -using google::cloud::odbc_bq_driver_internal::WstrToWireBytes; +using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; +using google::cloud::odbc_bq_driver_internal::WstrToWireBytes; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -394,17 +394,17 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_out_conn_str, &wire_char_count); - std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * WireWcharSize()); - size_t bytes_to_copy = std::min( - wire_bytes.size(), - static_cast(outConnectionStringBufferLen * WireWcharSize())); - std::memcpy(outConnectionString, wire_bytes.data(), bytes_to_copy); - } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_out_conn_str, &wire_char_count); + std::memset(outConnectionString, '\0', + outConnectionStringBufferLen * WireWcharSize()); + size_t bytes_to_copy = std::min( + wire_bytes.size(), + static_cast(outConnectionStringBufferLen * WireWcharSize())); + std::memcpy(outConnectionString, wire_bytes.data(), bytes_to_copy); + } } return rc; @@ -651,14 +651,14 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_info_val, &wire_char_count); + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_info_val, &wire_char_count); std::size_t bytes_available = static_cast(infoValueBufferLen); std::size_t bytes_to_copy = - std::min(wire_bytes.size(), bytes_available); - std::memcpy(infoValue, wire_bytes.data(), bytes_to_copy); + std::min(wire_bytes.size(), bytes_available); + std::memcpy(infoValue, wire_bytes.data(), bytes_to_copy); } } else { if (info_val_buffer_len > 0) { @@ -907,10 +907,10 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, // Handle Unicode conversion of input parameters. // Call to internal common function for SQLGetConnectAttr and // SQLGetConnectAttrW in odbc_connection.h. - SQLINTEGER internal_str_len = 0; + SQLINTEGER internal_str_len = 0; rc = ::google::cloud::odbc_bq_driver::SQLGetConnectAttrInternal( - connectionHandle, attribute, updated_attrib_val, - static_cast(kBufferLength), &internal_str_len); + connectionHandle, attribute, updated_attrib_val, + static_cast(kBufferLength), &internal_str_len); // Handle unicode conversion for attribute string values for output // parameters. if (SQL_SUCCEEDED(rc) && conn_attr.GetAttributeValueType(attribute) == @@ -920,19 +920,19 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, if (!updated_out_attr_status) { return updated_out_attr_status.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*updated_out_attr_status, &wire_char_count); - if (valueStringLen) { - *valueStringLen = - static_cast(wire_char_count * WireWcharSize()); - } - std::memset(value, '\0', valueBufferLen); - std::memcpy(value, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(valueBufferLen))); - } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*updated_out_attr_status, &wire_char_count); + if (valueStringLen) { + *valueStringLen = + static_cast(wire_char_count * WireWcharSize()); + } + std::memset(value, '\0', valueBufferLen); + std::memcpy(value, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(valueBufferLen))); + } } return rc; @@ -1164,17 +1164,18 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, if (!utf16_out_desc_val) { return utf16_out_desc_val.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_out_desc_val, &wire_char_count); - out_desc_val_string_len = - static_cast(wire_char_count * WireWcharSize()); - std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy(outDescValue, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(outDescValueBufferLen))); - } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_out_desc_val, &wire_char_count); + out_desc_val_string_len = + static_cast(wire_char_count * WireWcharSize()); + std::memset(outDescValue, '\0', outDescValueBufferLen); + std::memcpy( + outDescValue, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(outDescValueBufferLen))); + } } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, out_desc_val_string_len); @@ -1255,14 +1256,14 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_name, &wire_char_count); - size_t const max_bytes = - static_cast(nameBufferLen) * WireWcharSize(); - std::memset(name, '\0', max_bytes); - std::memcpy(name, wire_bytes.data(), - std::min(wire_bytes.size(), max_bytes)); + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_name, &wire_char_count); + size_t const max_bytes = + static_cast(nameBufferLen) * WireWcharSize(); + std::memset(name, '\0', max_bytes); + std::memcpy(name, wire_bytes.data(), + std::min(wire_bytes.size(), max_bytes)); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1548,12 +1549,12 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, if (!utf16_cur_name) { return utf16_cur_name.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_cur_name, &wire_char_count); - std::memcpy(cursorName, wire_bytes.data(), wire_bytes.size()); - } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_cur_name, &wire_char_count); + std::memcpy(cursorName, wire_bytes.data(), wire_bytes.size()); + } } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2126,20 +2127,22 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t wire_char_count = 0; - std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); - // Exclude null terminator from bytes_to_copy so we can write it explicitly + size_t wire_char_count = 0; + std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); + // Exclude null terminator from bytes_to_copy so we can write it + // explicitly size_t const bytes_to_copy = std::min(static_cast(characterAttributeBufferLen), - wire_char_count * WireWcharSize()); - std::memcpy(characterAttribute, wire_bytes.data(), bytes_to_copy); - if (static_cast(characterAttributeBufferLen) >= WireWcharSize()) { - // Write null terminator right after the copied data - std::memset(reinterpret_cast(characterAttribute) + - bytes_to_copy, 0, WireWcharSize()); + wire_char_count * WireWcharSize()); + std::memcpy(characterAttribute, wire_bytes.data(), bytes_to_copy); + if (static_cast(characterAttributeBufferLen) >= WireWcharSize()) { + // Write null terminator right after the copied data + std::memset( + reinterpret_cast(characterAttribute) + bytes_to_copy, 0, + WireWcharSize()); } - character_attribute_string_len = - static_cast(wire_char_count); + character_attribute_string_len = + static_cast(wire_char_count); } else { std::memcpy(characterAttribute, (SQLPOINTER)updated_character_attrib_val, @@ -2312,17 +2315,17 @@ SQLRETURN SQL_API SQLDescribeColW( if (!utf16_col_name) { return utf16_col_name.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_col_name, &wire_char_count); - // columnNameBufferLen is in SQLWCHAR characters per ODBC spec. - size_t const max_bytes = - static_cast(columnNameBufferLen) * WireWcharSize(); - std::memset(columnName, '\0', max_bytes); - std::memcpy(columnName, wire_bytes.data(), - std::min(wire_bytes.size(), max_bytes)); - } + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*utf16_col_name, &wire_char_count); + // columnNameBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const max_bytes = + static_cast(columnNameBufferLen) * WireWcharSize(); + std::memset(columnName, '\0', max_bytes); + std::memcpy(columnName, wire_bytes.data(), + std::min(wire_bytes.size(), max_bytes)); + } } if (columnNameLen) { @@ -2516,7 +2519,7 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, // in odbc_diagnostics.h. rc = google::cloud::odbc_bq_driver::SQLGetDiagFieldInternal( handleType, handle, recNumber, diagIdentifier, updated_diag_info, - static_cast(kBufferLength), &diag_info_str_len); + static_cast(kBufferLength), &diag_info_str_len); // Handle Unicode conversion of output parameters. if (SQL_SUCCEEDED(rc) && diag_info_str_len > 0) { @@ -2527,16 +2530,16 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, if (!updated_out_diag_info_status) { return updated_out_diag_info_status.GetCalculatedReturnCode(); } - { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*updated_out_diag_info_status, &wire_char_count); + { + size_t wire_char_count = 0; + std::vector wire_bytes = + WstrToWireBytes(*updated_out_diag_info_status, &wire_char_count); diag_info_str_len = - static_cast(wire_char_count * WireWcharSize()); - std::memcpy(diagInfo, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(diagInfoBufferLen))); - } + static_cast(wire_char_count * WireWcharSize()); + std::memcpy(diagInfo, wire_bytes.data(), + std::min(wire_bytes.size(), + static_cast(diagInfoBufferLen))); + } } else { std::memcpy(diagInfo, updated_diag_info, diagInfoBufferLen); @@ -2622,12 +2625,12 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_sql_state) { return utf16_sql_state.GetCalculatedReturnCode(); } - { - size_t wire_char_count_state = 0; - std::vector wire_bytes_state = - WstrToWireBytes(*utf16_sql_state, &wire_char_count_state); - std::memcpy(sqlState, wire_bytes_state.data(), wire_bytes_state.size()); - } + { + size_t wire_char_count_state = 0; + std::vector wire_bytes_state = + WstrToWireBytes(*utf16_sql_state, &wire_char_count_state); + std::memcpy(sqlState, wire_bytes_state.data(), wire_bytes_state.size()); + } } if (messageText && message_text_buffer_len > 0) { @@ -2636,17 +2639,17 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, if (!utf16_msg_txt) { return utf16_msg_txt.GetCalculatedReturnCode(); } - { - size_t wire_char_count_msg = 0; - std::vector wire_bytes_msg = - WstrToWireBytes(*utf16_msg_txt, &wire_char_count_msg); - // messageTextBufferLen is in SQLWCHAR characters per ODBC spec. - size_t const max_bytes = - static_cast(messageTextBufferLen) * WireWcharSize(); - std::memset(messageText, '\0', max_bytes); - std::memcpy(messageText, wire_bytes_msg.data(), - std::min(wire_bytes_msg.size(), max_bytes)); - } + { + size_t wire_char_count_msg = 0; + std::vector wire_bytes_msg = + WstrToWireBytes(*utf16_msg_txt, &wire_char_count_msg); + // messageTextBufferLen is in SQLWCHAR characters per ODBC spec. + size_t const max_bytes = + static_cast(messageTextBufferLen) * WireWcharSize(); + std::memset(messageText, '\0', max_bytes); + std::memcpy(messageText, wire_bytes_msg.data(), + std::min(wire_bytes_msg.size(), max_bytes)); + } } if (messageTextLen) *messageTextLen = message_text_buffer_len; diff --git a/google/cloud/odbc/bq_driver/odbc_sql_results.cc b/google/cloud/odbc/bq_driver/odbc_sql_results.cc index 57bc814b9e..dc800a116c 100644 --- a/google/cloud/odbc/bq_driver/odbc_sql_results.cc +++ b/google/cloud/odbc/bq_driver/odbc_sql_results.cc @@ -29,7 +29,6 @@ namespace google::cloud::odbc_bq_driver { using google::cloud::odbc_bq_driver_internal::BQDataType; -using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::CheckTargetType; using google::cloud::odbc_bq_driver_internal::ConnectionHandle; using google::cloud::odbc_bq_driver_internal::CreateDSRowFromTypeInfo; @@ -51,6 +50,7 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using google::cloud::odbc_bq_driver_internal::StmtStates; using google::cloud::odbc_bq_driver_internal::StringValueToOutputBufferResponse; using google::cloud::odbc_bq_driver_internal::ToSqlPointer; +using google::cloud::odbc_bq_driver_internal::WireWcharSize; using google::cloud::odbc_bq_driver_internal::WriteRowset; using google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; From b46032b3d111b92cdd3cd4881cef579a923b6d88 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 19 May 2026 21:24:32 +0530 Subject: [PATCH 24/32] re2 changes --- .../bq_driver/internal/odbc_sql_tables.cc | 73 +++++-------------- google/cloud/odbc/bq_driver/internal/utils.cc | 8 +- 2 files changed, 22 insertions(+), 59 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 28d64f59e9..f4a6553155 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -82,60 +82,25 @@ StatusRecord ValidateInputParameters( StatusRecordOr> GetFilteredProjectIds( ODBCBQClient& bq_client, std::string const& projects_filter, SQLULEN metadata_id) { - LOG(INFO) << "GetFilteredProjectIds:: Start (filter='" << projects_filter - << "', filter.size()=" << projects_filter.size() - << ", metadata_id=" << metadata_id << ")"; - // Wrap the entire body in try/catch so any thrown exception (regex_error, - // bad_alloc, anything from the BigQuery client) becomes a logged error - // instead of an unhandled exception that abort()s the driver process. - try { - LOG(INFO) << "GetFilteredProjectIds:: declaring project_ids vector"; - std::vector project_ids; - LOG(INFO) << "GetFilteredProjectIds:: about to call BuildRegex"; - std::regex filter_regex = BuildRegex(projects_filter, metadata_id); - LOG(INFO) << "GetFilteredProjectIds:: BuildRegex returned ok"; - // For now, we use default options. - // We can set timeout here as needed later. - LOG(INFO) << "GetFilteredProjectIds:: constructing Options"; - Options options; - LOG(INFO) << "GetFilteredProjectIds:: Options constructed"; - LOG(INFO) << "GetFilteredProjectIds:: calling ListAllProjects (filter='" - << projects_filter << "', metadata_id=" << metadata_id << ")"; - StatusRecordOr> projects = - bq_client.ListAllProjects(options); - LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned, ok=" - << static_cast(static_cast(projects)); - if (!projects) { - LOG(ERROR) << "GetFilteredProjectIds::ListAllProjects:: " - << projects.GetStatusRecord().message; - return projects.GetStatusRecord(); - } - LOG(INFO) << "GetFilteredProjectIds:: ListAllProjects returned " - << projects->size() << " projects; applying filter"; - for (auto const& project : *projects) { - if ((!metadata_id && projects_filter == "%") || - std::regex_match(project.id, filter_regex)) { - project_ids.push_back(project.id); - } + std::vector project_ids; + auto filter_regex = BuildRegex(projects_filter, metadata_id); + // For now, we use default options. + // We can set timeout here as needed later. + Options options; + StatusRecordOr> projects = + bq_client.ListAllProjects(options); + if (!projects) { + LOG(ERROR) << "GetFilteredProjectIds::ListAllProjects:: " + << projects.GetStatusRecord().message; + return projects.GetStatusRecord(); + } + for (auto const& project : *projects) { + if ((!metadata_id && projects_filter == "%") || + re2::RE2::FullMatch(project.id, *filter_regex)) { + project_ids.push_back(project.id); } - LOG(INFO) << "GetFilteredProjectIds:: kept " << project_ids.size() - << " projects after filter"; - return project_ids; - } catch (std::regex_error const& e) { - LOG(ERROR) << "GetFilteredProjectIds:: std::regex_error caught: code=" - << static_cast(e.code()) << " what='" << e.what() << "'"; - return StatusRecord{SQLStates::k_HY000(), - std::string("regex_error: ") + e.what()}; - } catch (std::exception const& e) { - LOG(ERROR) << "GetFilteredProjectIds:: std::exception caught: what='" - << e.what() << "'"; - return StatusRecord{SQLStates::k_HY000(), - std::string("exception: ") + e.what()}; - } catch (...) { - LOG(ERROR) << "GetFilteredProjectIds:: unknown exception caught"; - return StatusRecord{SQLStates::k_HY000(), - "Unknown exception in GetFilteredProjectIds"}; } + return project_ids; } StatusRecordOr> GetFilteredDatasetIds( @@ -143,6 +108,7 @@ StatusRecordOr> GetFilteredDatasetIds( std::string const& datasets_filter, SQLULEN metadata_id) { std::vector dataset_ids; auto filter_regex = BuildRegex(datasets_filter, metadata_id); + auto filter_regex = BuildRegex(datasets_filter, metadata_id); // For now, we use default options. // We can set timeout here as needed later. Options options; @@ -157,8 +123,7 @@ StatusRecordOr> GetFilteredDatasetIds( } for (auto const& dataset : *datasets) { if ((!metadata_id && datasets_filter == "%") || - re2::RE2::FullMatch(dataset.dataset_reference.dataset_id, - *filter_regex)) { + re2::RE2::FullMatch(dataset.dataset_reference.dataset_id, *filter_regex)) { dataset_ids.push_back(dataset.dataset_reference.dataset_id); } } diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 99cfccc889..0ad0169013 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -1020,11 +1020,9 @@ bool IsInfoTypeString(SQLUSMALLINT InfoType) { // '\X' -> literal X (the backslash is the escape; consumed in output) // any other char -> emitted as-is // -// Implemented as a manual single-pass loop rather than chained -// std::regex_replace calls: on some host libstdc++/libc++ builds the -// regex DFA initialization could throw and unwind across the ODBC ABI -// boundary, aborting the driver. The manual loop has no -// dependency. +// Implemented as a manual single-pass loop producing an RE2-compatible +// pattern. RE2 is used instead of std::regex to avoid DFA initialization +// crashes in libstdc++/libc++ on certain hosts (e.g. SAP HANA). std::string CastOdbcRegexToCppRegex(std::string const& str) { std::string result; // Worst case: every character becomes ".*" (2 chars). From 366b320a20284913022c72fa98b95b55abcf8c70 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Tue, 19 May 2026 22:24:33 +0530 Subject: [PATCH 25/32] update cmke --- google/cloud/odbc/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/google/cloud/odbc/CMakeLists.txt b/google/cloud/odbc/CMakeLists.txt index 57882661fd..ec5d80d45a 100644 --- a/google/cloud/odbc/CMakeLists.txt +++ b/google/cloud/odbc/CMakeLists.txt @@ -159,6 +159,15 @@ else () find_package(google_cloud_cpp_serviceusage REQUIRED) endif () +find_package(re2 CONFIG QUIET) +if(NOT TARGET re2::re2) + if(TARGET re2) + add_library(re2::re2 ALIAS re2) + else() + find_package(re2 REQUIRED) + endif() +endif() + # Restore the original BUILD_SHARED_LIBS value set(BUILD_SHARED_LIBS ${ORIGINAL_BUILD_SHARED_LIBS}) From 4418088aced1d72fc4d2c7c95147bf26efc90e50 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 20 May 2026 19:27:01 +0530 Subject: [PATCH 26/32] fixed clang --- google/cloud/odbc/CMakeLists.txt | 10 +++++----- .../odbc/bq_driver/internal/odbc_sql_tables.cc | 3 ++- .../odbc/bq_driver/internal/odbc_type_utils.h | 16 ++++++++-------- .../odbc_driver_tests/statement_test.cc | 1 + 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/google/cloud/odbc/CMakeLists.txt b/google/cloud/odbc/CMakeLists.txt index ec5d80d45a..ec04725342 100644 --- a/google/cloud/odbc/CMakeLists.txt +++ b/google/cloud/odbc/CMakeLists.txt @@ -160,13 +160,13 @@ else () endif () find_package(re2 CONFIG QUIET) -if(NOT TARGET re2::re2) - if(TARGET re2) +if (NOT TARGET re2::re2) + if (TARGET re2) add_library(re2::re2 ALIAS re2) - else() + else () find_package(re2 REQUIRED) - endif() -endif() + endif () +endif () # Restore the original BUILD_SHARED_LIBS value set(BUILD_SHARED_LIBS ${ORIGINAL_BUILD_SHARED_LIBS}) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index f4a6553155..532a638af2 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -123,7 +123,8 @@ StatusRecordOr> GetFilteredDatasetIds( } for (auto const& dataset : *datasets) { if ((!metadata_id && datasets_filter == "%") || - re2::RE2::FullMatch(dataset.dataset_reference.dataset_id, *filter_regex)) { + re2::RE2::FullMatch(dataset.dataset_reference.dataset_id, + *filter_regex)) { dataset_ids.push_back(dataset.dataset_reference.dataset_id); } } diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index da0319685d..81ad520c18 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -196,13 +196,13 @@ inline std::vector WstrToWireBytes(std::wstring const& wstr, std::vector utf16; utf16.reserve(wstr.size() + 1); for (wchar_t wc : wstr) { - uint32_t cp = static_cast(wc); - if (cp <= 0xFFFFu) { + auto cp = static_cast(wc); + if (cp <= 0xFFFFU) { utf16.push_back(static_cast(cp)); } else { - cp -= 0x10000u; - utf16.push_back(static_cast(0xD800u | (cp >> 10))); - utf16.push_back(static_cast(0xDC00u | (cp & 0x3FFu))); + cp -= 0x10000U; + utf16.push_back(static_cast(0xD800U | (cp >> 10))); + utf16.push_back(static_cast(0xDC00U | (cp & 0x3FFU))); } } if (out_char_count) *out_char_count = utf16.size(); @@ -227,8 +227,8 @@ inline std::vector WstrToWireBytes(std::wstring const& wstr, } inline odbc_internal::StatusRecord WStrToOutputBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, - SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + SQLINTEGER /*src_len*/, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t const wire_sz = WireWcharSize(); if (wstr.empty()) { @@ -248,7 +248,7 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( // bytes of each UCS-4 code unit as spurious null terminators. size_t wire_char_count = 0; std::vector wire_data = WstrToWireBytes(wstr, &wire_char_count); - SQLINTEGER u_src_len = static_cast(wire_char_count); + auto u_src_len = static_cast(wire_char_count); auto* dest = reinterpret_cast(dest_buf); if (buffer_length > u_src_len) { diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc index eadd203c8a..6f02daee0d 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc @@ -616,6 +616,7 @@ TEST(StatementTest, SQLExecDirect_htapi_bytes_type) { SQLRETURN status; auto conn = std::make_shared(); EXPECT_EQ( + Connect(kDefaultConnectionString + ";AllowHtapiForLargeResults=1;HTAPI_ActivationThreshold=0", conn), From bb26b6d9b40d91a9723448b1223d17f2b164d9e6 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Thu, 21 May 2026 11:08:41 +0530 Subject: [PATCH 27/32] udpate comment --- google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc | 3 ++- .../cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc | 2 +- google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc | 3 ++- google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc | 2 +- google/cloud/odbc/bq_driver/internal/odbc_type_utils.h | 5 +++-- google/cloud/odbc/bq_driver/internal/utils.cc | 2 +- .../odbc/integration_tests/odbc_driver_tests/catalog_test.cc | 1 - 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc index fc445675b5..18a4c39254 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc @@ -356,7 +356,8 @@ StatusRecordOr ProcessTableResults( for (TableFieldSchema const& table_field_schema : bq_table.schema.fields) { // bq_table_column could contain a search pattern character so do a regex // match. - auto column_pattern = BuildRegex(bq_table_column, metadata_id); + std::unique_ptr column_pattern = + BuildRegex(bq_table_column, metadata_id); if (re2::RE2::FullMatch(table_field_schema.name, *column_pattern)) { auto ds_row_status = CreateResultSetDSRow( conn_handle, bq_table.table_reference.project_id, diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc index 70974bf7f2..09b572964f 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc @@ -299,7 +299,7 @@ void ProcessTableResultsHelper(std::string const& column, expected_sql_int_row.ord_pos = (column == "%" || column.empty()) ? 2 : 1; expected_sql_int_row.is_nullable = "NO"; - auto column_pattern = BuildRegex(column, metadata_id); + std::unique_ptr column_pattern = BuildRegex(column, metadata_id); if (!metadata_id && (column.empty() || column == "%")) { ASSERT_EQ(result_set.rows.size(), 2); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 532a638af2..1ced1ab2d7 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -83,7 +83,8 @@ StatusRecordOr> GetFilteredProjectIds( ODBCBQClient& bq_client, std::string const& projects_filter, SQLULEN metadata_id) { std::vector project_ids; - auto filter_regex = BuildRegex(projects_filter, metadata_id); + std::unique_ptr filter_regex = + BuildRegex(projects_filter, metadata_id); // For now, we use default options. // We can set timeout here as needed later. Options options; diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index 8e93533fb7..d037a23a56 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -42,7 +42,7 @@ SQLRETURN AddressToPointer(SQLPOINTER ptr, SQLPOINTER out_buf, } odbc_internal::StatusRecord WStrIntervalBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t wire_char_count = 0; diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 81ad520c18..5ab2289866 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -196,7 +196,8 @@ inline std::vector WstrToWireBytes(std::wstring const& wstr, std::vector utf16; utf16.reserve(wstr.size() + 1); for (wchar_t wc : wstr) { - auto cp = static_cast(wc); + auto cp = + static_cast(static_cast>(wc)); if (cp <= 0xFFFFU) { utf16.push_back(static_cast(cp)); } else { @@ -281,7 +282,7 @@ SQLRETURN AddressToPointer(SQLPOINTER ptr, SQLPOINTER out_buf, SQLSMALLINT* str_len_ptr); odbc_internal::StatusRecord WStrIntervalBufferResponse( - std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, + std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len); } // namespace google::cloud::odbc_bq_driver_internal diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 0ad0169013..7ae3c791da 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -59,7 +59,7 @@ bool IsRuntimeWireUtf16Le() { } size_t WireWcharSize() { - return IsRuntimeWireUtf16Le() ? 2u : sizeof(SQLWCHAR); + return IsRuntimeWireUtf16Le() ? 2U : sizeof(SQLWCHAR); } #ifdef _WIN32 diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc index 3a32ffe9e3..3052712907 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/catalog_test.cc @@ -1804,7 +1804,6 @@ TEST(CatalogTest, SQLTables_Filter_DefaultDataset_SchemaNull) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } - #ifdef BQ_DRIVER_INTEGRATION_TESTS // This test case currently crashes with the existing ODBC Driver for BigQuery // v3.1.6.1026. The crash occurs in SQLColumns when schema_name is NULL, From cf9d96820d3faf20f2ec89b03ccc150947610d79 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Wed, 3 Jun 2026 16:55:17 +0530 Subject: [PATCH 28/32] test_improvement --- .../bq_driver/internal/data_translation.cc | 70 +++----- .../bq_driver/internal/odbc_type_utils.cc | 18 +- .../odbc/bq_driver/internal/odbc_type_utils.h | 101 +++++------ google/cloud/odbc/bq_driver/odbc_api.cc | 157 ++++++++---------- 4 files changed, 140 insertions(+), 206 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index f62e65899f..a2a9074328 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -1007,27 +1007,23 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue( "DSValueToWchar Conversion Failed"}; break; } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(wstr.GetValue(), &wire_char_count); size_t const wire_sz = WireWcharSize(); - auto* dest8 = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_timestamp_src_len) { if (res_len) { - *res_len = static_cast(wire_char_count * wire_sz); + *res_len = static_cast(wstr.GetValue().size() * wire_sz); } - std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); - std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + wstr.GetValue().size()); + WriteWireNul(dest_buf, wstr.GetValue().size()); } else if (20 <= wchar_capacity && wchar_capacity <= k_timestamp_src_len) { if (res_len) { *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), - static_cast(wchar_capacity - 1) * wire_sz); - std::memset(dest8 + static_cast(wchar_capacity - 1) * wire_sz, - 0, wire_sz); + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + static_cast(wchar_capacity - 1)); + WriteWireNul(dest_buf, static_cast(wchar_capacity - 1)); LOG(WARNING) << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -1186,26 +1182,22 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval, "DSValueToWchar Conversion Failed"}; break; } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(wstr.GetValue(), &wire_char_count); size_t const wire_sz = WireWcharSize(); - auto* dest8 = reinterpret_cast(dest_buf); SQLLEN wchar_capacity = buffer_length / static_cast(wire_sz); if (wchar_capacity > k_datetime_src_len) { if (res_len) { - *res_len = static_cast(wire_char_count * wire_sz); + *res_len = static_cast(wstr.GetValue().size() * wire_sz); } - std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); - std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + wstr.GetValue().size()); + WriteWireNul(dest_buf, wstr.GetValue().size()); } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) { if (res_len) { *res_len = wchar_capacity * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), - static_cast(wchar_capacity - 1) * wire_sz); - std::memset(dest8 + static_cast(wchar_capacity - 1) * wire_sz, - 0, wire_sz); + WriteWideToWireBuffer(wstr.GetValue(), dest_buf, + static_cast(wchar_capacity - 1)); + WriteWireNul(dest_buf, static_cast(wchar_capacity - 1)); LOG(WARNING) << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR."; status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"}; @@ -2046,27 +2038,20 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, std::wstring const& utf16_value = utf16_str.GetValue(); - // Use WstrToWireBytes to produce correctly-encoded output. When - // IsRuntimeWireUtf16Le() the 4-byte wchar_t values are narrowed to - // uint16_t so memcpy does not embed zero high bytes as spurious null - // terminators. - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(utf16_value, &wire_char_count); + // Narrow wchar_t -> wire encoding directly into the caller's buffer. + // No intermediate vector; WriteWideToWireBuffer is a memcpy when the wire + // SQLWCHAR width matches sizeof(wchar_t) and a per-element narrowing loop + // only on the iODBC-built / unixODBC-loaded path. size_t const wire_sz = WireWcharSize(); - size_t const required_size = wire_char_count * wire_sz; + size_t const src_chars = utf16_value.size(); + size_t const required_size = src_chars * wire_sz; - auto* dest8 = reinterpret_cast(dest_data.buf); - - // Handle truncation if buffer cannot hold the data (even without null). - // An exact fit (buflen == required_size) is treated as success; the caller's - // buffer is presumed to be zeroed or the null terminator is not required. if (static_cast(dest_data.buflen) < required_size) { size_t num_chars_to_copy = dest_data.buflen / wire_sz; if (num_chars_to_copy > 0) { num_chars_to_copy--; // leave one slot for the null terminator - std::memcpy(dest8, wire_bytes.data(), num_chars_to_copy * wire_sz); - std::memset(dest8 + num_chars_to_copy * wire_sz, 0, wire_sz); + WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy); + WriteWireNul(dest_data.buf, num_chars_to_copy); } if (dest_data.result_len) { *dest_data.result_len = required_size; @@ -2074,17 +2059,12 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val, LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated."; return StatusRecord{SQLStates::k_01004(), "String data, right truncated"}; } - // Copy data; append null terminator only when there is room for it. + WriteWideToWireBuffer(utf16_value, dest_data.buf, src_chars); if (static_cast(dest_data.buflen) >= required_size + wire_sz) { - std::memcpy(dest8, wire_bytes.data(), (wire_char_count + 1) * wire_sz); - } else { - // Exact fit: buffer holds the data but has no null terminator slot. - std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); + WriteWireNul(dest_data.buf, src_chars); } - - // Set output length if (dest_data.result_len) { - *dest_data.result_len = wire_char_count * wire_sz; + *dest_data.result_len = required_size; } return status_record; } diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc index d037a23a56..365c722aa4 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc @@ -45,25 +45,21 @@ odbc_internal::StatusRecord WStrIntervalBufferResponse( std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); - size_t wire_char_count = 0; - std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); size_t const wire_sz = WireWcharSize(); - auto* dest8 = static_cast(dest_buf); + if (buffer_length > char_len) { if (res_len) { - *res_len = - static_cast(wire_char_count) * static_cast(wire_sz); + *res_len = static_cast(char_len) * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), wire_char_count * wire_sz); - std::memset(dest8 + wire_char_count * wire_sz, 0, wire_sz); + WriteWideToWireBuffer(wstr, dest_buf, static_cast(char_len)); + WriteWireNul(dest_buf, static_cast(char_len)); } else if (buffer_length > whole_digits_count) { if (res_len) { *res_len = buffer_length * static_cast(wire_sz); } - std::memcpy(dest8, wire_bytes.data(), - static_cast(buffer_length - 1) * wire_sz); - std::memset(dest8 + static_cast(buffer_length - 1) * wire_sz, 0, - wire_sz); + WriteWideToWireBuffer(wstr, dest_buf, + static_cast(buffer_length - 1)); + WriteWireNul(dest_buf, static_cast(buffer_length - 1)); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 5ab2289866..9af7ed1848 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -177,65 +177,52 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, return SQL_SUCCESS; } -// Converts a std::wstring to a flat byte array ready to memcpy into a SQLWCHAR -// output buffer using the correct wire encoding. +// Writes `count` wide characters from `src` directly into `dest` using the +// current wire encoding. `dest` must point to caller-owned storage of at +// least `count * WireWcharSize()` bytes. The caller writes its own NUL +// terminator (`WriteWireNul` below) if it wants one. // -// When IsRuntimeWireUtf16Le() is true (iODBC-built driver loaded by unixODBC), -// each 4-byte wchar_t code point is narrowed to one or two uint16_t code units -// (UTF-16LE; surrogate pairs are generated for code points above U+FFFF). -// Otherwise the raw SQLWCHAR bytes are emitted unchanged. -// -// The returned vector always ends with one wire-format null code unit. -// *out_char_count (optional) receives the number of wire code units written, -// excluding the null terminator. -inline std::vector WstrToWireBytes(std::wstring const& wstr, - size_t* out_char_count = nullptr) { - std::vector bytes; +// When the wire SQLWCHAR width matches `sizeof(wchar_t)` (Windows and the +// iODBC build — the common case), this is a single memcpy of the wstring's +// raw bytes, equivalent to the main-branch `ToSqlWChar` + memcpy pattern. +// Only when the runtime wire format is UTF-16LE but the driver was compiled +// with 4-byte SQLWCHAR (iODBC-built driver loaded by unixODBC, the SAP HANA +// SDA case) do we fall to a per-element narrowing loop, because the source +// and destination element widths actually differ. +inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, + size_t count) { + if (count > src.size()) count = src.size(); #if !defined(_WIN32) if (IsRuntimeWireUtf16Le()) { - std::vector utf16; - utf16.reserve(wstr.size() + 1); - for (wchar_t wc : wstr) { - auto cp = - static_cast(static_cast>(wc)); - if (cp <= 0xFFFFU) { - utf16.push_back(static_cast(cp)); - } else { - cp -= 0x10000U; - utf16.push_back(static_cast(0xD800U | (cp >> 10))); - utf16.push_back(static_cast(0xDC00U | (cp & 0x3FFU))); - } + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); } - if (out_char_count) *out_char_count = utf16.size(); - utf16.push_back(0); // null terminator - bytes.resize(utf16.size() * 2); - std::memcpy(bytes.data(), utf16.data(), bytes.size()); - return bytes; + return; } #endif - if (out_char_count) *out_char_count = wstr.size(); - bytes.resize((wstr.size() + 1) * sizeof(SQLWCHAR), 0); - // Cast each wchar_t to SQLWCHAR individually. On Linux/unixODBC, - // sizeof(wchar_t)==4 but sizeof(SQLWCHAR)==2, so memcpy of raw wstring - // bytes would pick up only the low byte of each character and treat the - // intervening zero high bytes as null terminators. - auto* sqlwchar_dest = reinterpret_cast(bytes.data()); - for (size_t i = 0; i < wstr.size(); ++i) { - sqlwchar_dest[i] = static_cast(wstr[i]); - } - // Null terminator is already zero from the resize above. - return bytes; + // Wire SQLWCHAR width matches wchar_t — single memcpy. + std::memcpy(dest, src.data(), count * sizeof(SQLWCHAR)); +} + +// Writes a single wire-format NUL terminator (one code unit, 2 or 4 bytes) +// at byte offset `char_index * WireWcharSize()` from `dest`. +inline void WriteWireNul(void* dest, size_t char_index) { + size_t const wire_sz = WireWcharSize(); + std::memset(static_cast(dest) + (char_index * wire_sz), 0, + wire_sz); } + inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, - SQLINTEGER /*src_len*/, SQLINTEGER supp_max_len, SQLLEN* res_len) { + SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { auto status_record = odbc_internal::StatusRecord::Ok(); size_t const wire_sz = WireWcharSize(); + if (wstr.empty()) { if (dest_buf && buffer_length > 0) { - // Write a wire-format null terminator (2 bytes for UTF-16LE, 4 for UCS-4) - std::memset(dest_buf, 0, wire_sz); + WriteWireNul(dest_buf, 0); } if (res_len) { *res_len = 0; @@ -243,28 +230,18 @@ inline odbc_internal::StatusRecord WStrToOutputBufferResponse( return status_record; } - // Build a flat byte array in the correct wire encoding. When - // IsRuntimeWireUtf16Le() each 4-byte wchar_t is narrowed to uint16_t so - // that memcpy produces valid UTF-16LE instead of embedding the zero high - // bytes of each UCS-4 code unit as spurious null terminators. - size_t wire_char_count = 0; - std::vector wire_data = WstrToWireBytes(wstr, &wire_char_count); - auto u_src_len = static_cast(wire_char_count); - auto* dest = reinterpret_cast(dest_buf); - - if (buffer_length > u_src_len) { + if (buffer_length > src_len) { if (res_len) { - *res_len = u_src_len * static_cast(wire_sz); + *res_len = src_len * static_cast(wire_sz); } - // Copy data + null terminator (wire_data has u_src_len+1 code units) - std::memcpy(dest, wire_data.data(), (u_src_len + 1) * wire_sz); - } else if (supp_max_len <= buffer_length && buffer_length <= u_src_len) { + WriteWideToWireBuffer(wstr, dest_buf, src_len); + WriteWireNul(dest_buf, src_len); + } else if (supp_max_len <= buffer_length && buffer_length <= src_len) { if (res_len) { *res_len = buffer_length * static_cast(wire_sz); } - // Copy buffer_length-1 code units, then write null terminator - std::memcpy(dest, wire_data.data(), (buffer_length - 1) * wire_sz); - std::memset(dest + (buffer_length - 1) * wire_sz, 0, wire_sz); + WriteWideToWireBuffer(wstr, dest_buf, buffer_length - 1); + WriteWireNul(dest_buf, buffer_length - 1); status_record = odbc_internal::StatusRecord{ google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"}; } else { diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 6ac51c3896..935a8d3cdd 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -65,8 +65,9 @@ using google::cloud::odbc_bq_driver_internal::StatementHandle; using ::google::cloud::odbc_bq_driver_internal::TraceOptions; using google::cloud::odbc_bq_driver_internal::Utf8ToUtf16; using google::cloud::odbc_bq_driver_internal::WireWcharSize; +using google::cloud::odbc_bq_driver_internal::WriteWideToWireBuffer; +using google::cloud::odbc_bq_driver_internal::WriteWireNul; using google::cloud::odbc_bq_driver_internal::WStrToOutputBufferResponse; -using google::cloud::odbc_bq_driver_internal::WstrToWireBytes; using ::google::cloud::odbc_internal::SQLStates; using google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; @@ -395,15 +396,14 @@ SQLRETURN SQL_API SQLBrowseConnectW(SQLHDBC connectionHandle, return utf16_out_conn_str.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_out_conn_str, &wire_char_count); - std::memset(outConnectionString, '\0', - outConnectionStringBufferLen * WireWcharSize()); - size_t bytes_to_copy = std::min( - wire_bytes.size(), - static_cast(outConnectionStringBufferLen * WireWcharSize())); - std::memcpy(outConnectionString, wire_bytes.data(), bytes_to_copy); + size_t const dest_chars = + static_cast(outConnectionStringBufferLen); + size_t const to_copy = + std::min(utf16_out_conn_str->size(), dest_chars); + // memset zeros the entire dest, which leaves the trailing wire NUL in + // place after we write `to_copy` chars. + std::memset(outConnectionString, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, to_copy); } } @@ -651,14 +651,15 @@ SQLRETURN SQL_API SQLGetInfoW(SQLHDBC connectionHandle, SQLUSMALLINT infoType, return utf16_info_val.GetCalculatedReturnCode(); } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_info_val, &wire_char_count); - std::size_t bytes_available = - static_cast(infoValueBufferLen); - std::size_t bytes_to_copy = - std::min(wire_bytes.size(), bytes_available); - std::memcpy(infoValue, wire_bytes.data(), bytes_to_copy); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(infoValueBufferLen) / wire_sz; + size_t const to_copy = + std::min(utf16_info_val->size(), dest_chars); + WriteWideToWireBuffer(*utf16_info_val, infoValue, to_copy); + if (to_copy < dest_chars) { + WriteWireNul(infoValue, to_copy); + } } } else { if (info_val_buffer_len > 0) { @@ -921,17 +922,17 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, return updated_out_attr_status.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*updated_out_attr_status, &wire_char_count); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(valueBufferLen) / wire_sz; + size_t const to_copy = + std::min(updated_out_attr_status->size(), dest_chars); if (valueStringLen) { *valueStringLen = - static_cast(wire_char_count * WireWcharSize()); + static_cast(updated_out_attr_status->size() * wire_sz); } std::memset(value, '\0', valueBufferLen); - std::memcpy(value, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(valueBufferLen))); + WriteWideToWireBuffer(*updated_out_attr_status, value, to_copy); } } @@ -1165,16 +1166,15 @@ SQLRETURN SQL_API SQLGetDescFieldW(SQLHDESC descriptorHandle, return utf16_out_desc_val.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_out_desc_val, &wire_char_count); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(outDescValueBufferLen) / wire_sz; + size_t const to_copy = + std::min(utf16_out_desc_val->size(), dest_chars); out_desc_val_string_len = - static_cast(wire_char_count * WireWcharSize()); + static_cast(utf16_out_desc_val->size() * wire_sz); std::memset(outDescValue, '\0', outDescValueBufferLen); - std::memcpy( - outDescValue, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(outDescValueBufferLen))); + WriteWideToWireBuffer(*utf16_out_desc_val, outDescValue, to_copy); } } else { std::memcpy(outDescValue, (SQLPOINTER)out_desc_val, @@ -1256,14 +1256,10 @@ SQLRETURN SQL_API SQLGetDescRecW( if (!utf16_name) { return utf16_name.GetCalculatedReturnCode(); } - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_name, &wire_char_count); - size_t const max_bytes = - static_cast(nameBufferLen) * WireWcharSize(); - std::memset(name, '\0', max_bytes); - std::memcpy(name, wire_bytes.data(), - std::min(wire_bytes.size(), max_bytes)); + size_t const dest_chars = static_cast(nameBufferLen); + size_t const to_copy = std::min(utf16_name->size(), dest_chars); + std::memset(name, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_name, name, to_copy); } if (nameStringLen) *nameStringLen = name_string_len; @@ -1550,10 +1546,9 @@ SQLRETURN SQL_API SQLGetCursorNameW(SQLHSTMT statementHandle, return utf16_cur_name.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_cur_name, &wire_char_count); - std::memcpy(cursorName, wire_bytes.data(), wire_bytes.size()); + WriteWideToWireBuffer(*utf16_cur_name, cursorName, + utf16_cur_name->size()); + WriteWireNul(cursorName, utf16_cur_name->size()); } } if (cursorNameStringLen) *cursorNameStringLen = cursor_name_len; @@ -2127,22 +2122,15 @@ SQLRETURN SQL_API SQLColAttributeW(SQLHSTMT statementHandle, return updated_out_character_attr_status.GetCalculatedReturnCode(); } std::wstring const& wstr = *updated_out_character_attr_status; - size_t wire_char_count = 0; - std::vector wire_bytes = WstrToWireBytes(wstr, &wire_char_count); - // Exclude null terminator from bytes_to_copy so we can write it - // explicitly - size_t const bytes_to_copy = - std::min(static_cast(characterAttributeBufferLen), - wire_char_count * WireWcharSize()); - std::memcpy(characterAttribute, wire_bytes.data(), bytes_to_copy); - if (static_cast(characterAttributeBufferLen) >= WireWcharSize()) { - // Write null terminator right after the copied data - std::memset( - reinterpret_cast(characterAttribute) + bytes_to_copy, 0, - WireWcharSize()); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(characterAttributeBufferLen) / wire_sz; + size_t const to_copy = std::min(wstr.size(), dest_chars); + WriteWideToWireBuffer(wstr, characterAttribute, to_copy); + if (static_cast(characterAttributeBufferLen) >= wire_sz) { + WriteWireNul(characterAttribute, to_copy); } - character_attribute_string_len = - static_cast(wire_char_count); + character_attribute_string_len = static_cast(wstr.size()); } else { std::memcpy(characterAttribute, (SQLPOINTER)updated_character_attrib_val, @@ -2316,15 +2304,12 @@ SQLRETURN SQL_API SQLDescribeColW( return utf16_col_name.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*utf16_col_name, &wire_char_count); // columnNameBufferLen is in SQLWCHAR characters per ODBC spec. - size_t const max_bytes = - static_cast(columnNameBufferLen) * WireWcharSize(); - std::memset(columnName, '\0', max_bytes); - std::memcpy(columnName, wire_bytes.data(), - std::min(wire_bytes.size(), max_bytes)); + size_t const dest_chars = static_cast(columnNameBufferLen); + size_t const to_copy = + std::min(utf16_col_name->size(), dest_chars); + std::memset(columnName, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_col_name, columnName, to_copy); } } @@ -2531,14 +2516,14 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, return updated_out_diag_info_status.GetCalculatedReturnCode(); } { - size_t wire_char_count = 0; - std::vector wire_bytes = - WstrToWireBytes(*updated_out_diag_info_status, &wire_char_count); - diag_info_str_len = - static_cast(wire_char_count * WireWcharSize()); - std::memcpy(diagInfo, wire_bytes.data(), - std::min(wire_bytes.size(), - static_cast(diagInfoBufferLen))); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(diagInfoBufferLen) / wire_sz; + size_t const to_copy = + std::min(updated_out_diag_info_status->size(), dest_chars); + diag_info_str_len = static_cast( + updated_out_diag_info_status->size() * wire_sz); + WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, to_copy); } } else { @@ -2626,10 +2611,9 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_sql_state.GetCalculatedReturnCode(); } { - size_t wire_char_count_state = 0; - std::vector wire_bytes_state = - WstrToWireBytes(*utf16_sql_state, &wire_char_count_state); - std::memcpy(sqlState, wire_bytes_state.data(), wire_bytes_state.size()); + WriteWideToWireBuffer(*utf16_sql_state, sqlState, + utf16_sql_state->size()); + WriteWireNul(sqlState, utf16_sql_state->size()); } } @@ -2640,15 +2624,12 @@ SQLRETURN SQL_API SQLGetDiagRecW(SQLSMALLINT handleType, SQLHANDLE handle, return utf16_msg_txt.GetCalculatedReturnCode(); } { - size_t wire_char_count_msg = 0; - std::vector wire_bytes_msg = - WstrToWireBytes(*utf16_msg_txt, &wire_char_count_msg); // messageTextBufferLen is in SQLWCHAR characters per ODBC spec. - size_t const max_bytes = - static_cast(messageTextBufferLen) * WireWcharSize(); - std::memset(messageText, '\0', max_bytes); - std::memcpy(messageText, wire_bytes_msg.data(), - std::min(wire_bytes_msg.size(), max_bytes)); + size_t const dest_chars = static_cast(messageTextBufferLen); + size_t const to_copy = + std::min(utf16_msg_txt->size(), dest_chars); + std::memset(messageText, '\0', dest_chars * WireWcharSize()); + WriteWideToWireBuffer(*utf16_msg_txt, messageText, to_copy); } } if (messageTextLen) *messageTextLen = message_text_buffer_len; From ee7c8b2027ec7de2e2dbefb93943d48661a74bdf Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Thu, 4 Jun 2026 11:41:49 +0530 Subject: [PATCH 29/32] removed all references sqlwchar --- .../odbc/bq_driver/internal/odbc_type_utils.h | 14 +++++++----- google/cloud/odbc/bq_driver/odbc_api.cc | 22 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 9af7ed1848..afea6c7f1e 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -184,11 +184,7 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, // // When the wire SQLWCHAR width matches `sizeof(wchar_t)` (Windows and the // iODBC build — the common case), this is a single memcpy of the wstring's -// raw bytes, equivalent to the main-branch `ToSqlWChar` + memcpy pattern. -// Only when the runtime wire format is UTF-16LE but the driver was compiled -// with 4-byte SQLWCHAR (iODBC-built driver loaded by unixODBC, the SAP HANA -// SDA case) do we fall to a per-element narrowing loop, because the source -// and destination element widths actually differ. +// raw bytes inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { if (count > src.size()) count = src.size(); @@ -200,6 +196,14 @@ inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, } return; } + if constexpr (sizeof(SQLWCHAR) != sizeof(wchar_t)) { + // e.g. unixODBC: SQLWCHAR is 2 bytes, wchar_t is 4 bytes + auto* d = static_cast(dest); + for (size_t i = 0; i < count; ++i) { + d[i] = static_cast(src[i]); + } + return; + } #endif // Wire SQLWCHAR width matches wchar_t — single memcpy. std::memcpy(dest, src.data(), count * sizeof(SQLWCHAR)); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 935a8d3cdd..81b1f5607a 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -77,7 +77,6 @@ using ::google::cloud::odbc_bq_driver::HandleLockError; using google::cloud::odbc_bq_driver::ToCharStr; using google::cloud::odbc_bq_driver::ToSqlChar; -using google::cloud::odbc_bq_driver::ToSqlWChar; constexpr int kBufferLength = 4096; @@ -283,7 +282,7 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString) { + if (SQL_SUCCEEDED(rc) && outConnectionString ) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -294,7 +293,8 @@ SQLRETURN SQL_API SQLDriverConnectW( if (!utf16_out_conn_str) { return utf16_out_conn_str.GetCalculatedReturnCode(); } - outConnectionString = ToSqlWChar(utf16_out_conn_str->data()); + + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, out_conn_str_len); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -551,7 +551,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_server_name.GetCalculatedReturnCode(); } serverNameLen = utf16_server_name->length(); - std::memcpy(serverName, ToSqlWChar(utf16_server_name->data()), serverNameLen); + WriteWideToWireBuffer(*utf16_server_name, serverName, serverNameLen); if (w_user_name_len > 0) { StatusRecordOr utf16_user_name = Utf8ToUtf16(*utf8_user_name); @@ -559,7 +559,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_user_name.GetCalculatedReturnCode(); } userNameLen = utf16_user_name->length(); - std::memcpy(userName, ToSqlWChar(utf16_user_name->data()), userNameLen); + WriteWideToWireBuffer(*utf16_user_name, userName, userNameLen); } if (w_auth_str_len > 0) { @@ -568,7 +568,7 @@ SQLRETURN SQL_API SQLConnectW(SQLHDBC connectionHandle, SQLWCHAR* serverName, return utf16_auth_str.GetCalculatedReturnCode(); } authStringLen = utf16_auth_str->length(); - std::memcpy(authString, ToSqlWChar(utf16_auth_str->data()), authStringLen); + WriteWideToWireBuffer(*utf16_auth_str, authString, authStringLen); } return rc; @@ -2220,9 +2220,13 @@ SQLRETURN SQL_API SQLColAttributesW(SQLHSTMT statementHandle, if (!utf16_character_attribute) { return utf16_character_attribute.GetCalculatedReturnCode(); } - std::memcpy(characterAttribute, - (SQLPOINTER)ToSqlWChar(utf16_character_attribute->data()), - character_attribute_buffer_len); + size_t const wire_sz = WireWcharSize(); + size_t const dest_chars = + static_cast(character_attribute_buffer_len) / wire_sz; + size_t const to_copy = + std::min(utf16_character_attribute->size(), dest_chars); + WriteWideToWireBuffer(*utf16_character_attribute, characterAttribute, + to_copy); } if (characterAttributeStringLen) *characterAttributeStringLen = character_attribute_buffer_len; From 777a4a65cbaf6fbe6c738ac715b5e70ffed66365 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Thu, 28 May 2026 08:47:52 +0530 Subject: [PATCH 30/32] test --- .../builds/lib/google.googlebigqueryodbc.ini | 9 ++ .../odbc/bq_driver/internal/trace_utils.cc | 2 + .../odbc/bq_driver/internal/trace_utils.h | 3 + google/cloud/odbc/bq_driver/internal/utils.cc | 93 ++++++++++--------- google/cloud/odbc/bq_driver/internal/utils.h | 19 ++-- 5 files changed, 72 insertions(+), 54 deletions(-) diff --git a/ci/gha/builds/lib/google.googlebigqueryodbc.ini b/ci/gha/builds/lib/google.googlebigqueryodbc.ini index cd13ac1435..e49210b3c6 100644 --- a/ci/gha/builds/lib/google.googlebigqueryodbc.ini +++ b/ci/gha/builds/lib/google.googlebigqueryodbc.ini @@ -1,3 +1,12 @@ [Driver] LogLevel=0 LogPath= +# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS +# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4). +# +# Accepted values: +# UTF-16LE - 2-byte UTF-16LE per code unit (unixODBC-loaded driver) +# UTF-32LE - 4-byte UTF-32LE per code unit (native iODBC) +# (empty) - use sizeof(SQLWCHAR) as-is (default) +# +WcharEncoding= diff --git a/google/cloud/odbc/bq_driver/internal/trace_utils.cc b/google/cloud/odbc/bq_driver/internal/trace_utils.cc index 9d9ee8aded..c748c10f21 100644 --- a/google/cloud/odbc/bq_driver/internal/trace_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/trace_utils.cc @@ -295,6 +295,8 @@ TraceOptions::CreateTraceOptionsFile( log_file_size = std::strtol(s.second.c_str(), nullptr, 10); } else if (s.first == kMaxThreadsParam) { max_threads = std::stoull(s.second); + } else if (s.first == kWcharEncoding) { + SetWcharEncodingFromConfig(s.second); } } diff --git a/google/cloud/odbc/bq_driver/internal/trace_utils.h b/google/cloud/odbc/bq_driver/internal/trace_utils.h index 6124b12495..4de27108d8 100644 --- a/google/cloud/odbc/bq_driver/internal/trace_utils.h +++ b/google/cloud/odbc/bq_driver/internal/trace_utils.h @@ -41,6 +41,9 @@ inline std::string const kLogPath = "LogPath"; inline std::string const kLogFileCount = "LogFileCount"; inline std::string const kLogFileSize = "LogFileSize"; inline std::string const kMaxThreadsParam = "MaxThreads"; +// Key controlling the wire encoding of SQLWCHAR buffers on Linux/macOS. +// Accepted values: "UTF-16LE", "UCS-4LE", or empty (auto-detect). +inline std::string const kWcharEncoding = "WcharEncoding"; inline std::uint32_t const kDefaultMaxThreads = 8; inline std::string const kDefaultMaxFiles = "50"; inline std::string const kDefaultMaxSize = "2000"; diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 7ae3c791da..0c9cea4297 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -42,19 +42,23 @@ using ::google::cloud::odbc_internal::StatusRecord; using ::google::cloud::odbc_internal::StatusRecordOr; namespace { -// Process-lifetime latch for "loaded ODBC manager speaks UTF-16LE on the wire". -// Set inside BqConvertSQLWCHARToString once we observe the UTF-16LE byte -// pattern in an input buffer. The wire format never changes within a process, -// so once latched it stays latched. -std::atomic g_utf16le_wire_latched{false}; +// Wire-encoding override read from the [Driver] WcharEncoding key in +// google.googlebigqueryodbc.ini (Linux/macOS) or the equivalent registry key. +// Only meaningful when sizeof(SQLWCHAR) == 4 (iODBC build on Linux/macOS). +// kDefault ΓÇô use sizeof(SQLWCHAR) as the wire size (no adaptation) +// kUtf16Le ΓÇô 2-byte UTF-16LE wire format (unixODBC loaded driver) +// kUtf32Le ΓÇô 4-byte UTF-32LE wire format (native iODBC) +enum class WcharEncodingOverride { kDefault, kUtf16Le, kUtf32Le }; +std::atomic g_wchar_encoding_override{ + WcharEncodingOverride::kDefault}; } // namespace bool IsRuntimeWireUtf16Le() { #if defined(_WIN32) return false; #else - if (sizeof(SQLWCHAR) != 4) return false; - return g_utf16le_wire_latched.load(std::memory_order_relaxed); + return g_wchar_encoding_override.load(std::memory_order_relaxed) == + WcharEncodingOverride::kUtf16Le; #endif } @@ -62,6 +66,26 @@ size_t WireWcharSize() { return IsRuntimeWireUtf16Le() ? 2U : sizeof(SQLWCHAR); } +void SetWcharEncodingFromConfig(std::string const& value) { +#if !defined(_WIN32) + if (value == "UTF-16LE") { + g_wchar_encoding_override.store(WcharEncodingOverride::kUtf16Le, + std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: UTF-16LE wire format (2 bytes/char)"; + } else if (value == "UTF-32LE") { + g_wchar_encoding_override.store(WcharEncodingOverride::kUtf32Le, + std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: UTF-32LE wire format (4 bytes/char)"; + } else if (value.empty()) { + g_wchar_encoding_override.store(WcharEncodingOverride::kDefault, + std::memory_order_relaxed); + LOG(INFO) << "WcharEncoding: default (sizeof(SQLWCHAR) bytes/char)"; + } else { + LOG(WARNING) << "WcharEncoding: unrecognised value '" << value << "'"; + } +#endif +} + #ifdef _WIN32 using google::cloud::odbc_bigquery_client_interface::OauthMechanism; static std::string const kOAuthMechanism = "OAuthMechanism"; @@ -850,46 +874,25 @@ odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( } #if !defined(_WIN32) - // The driver may be compiled against iODBC headers, where SQLWCHAR is - // wchar_t (4 bytes UCS-4LE on Linux/macOS), but loaded by unixODBC, which - // by default delivers 2-byte UTF-16LE wide chars. SAP HANA on Linux uses - // unixODBC, so without `IconvEncoding=UCS-4LE` in odbcinst.ini we receive - // UTF-16LE in a buffer typed as 4-byte SQLWCHAR. - // - // Detection: a UCS-4LE ASCII char looks like `XX 00 00 00`, UTF-16LE - // looks like `XX 00 YY 00`. Single-char and empty strings are ambiguous - // so we *latch* the result the first time we get a clear signal — every - // later call (including the ambiguous ones from SQLTables(catalog="%")) - // honors the latch via IsRuntimeWireUtf16Le(). - if (sizeof(SQLWCHAR) == 4) { - bool use_utf16le = g_utf16le_wire_latched.load(std::memory_order_relaxed); - - auto const* bytes = reinterpret_cast(in_str); - if (!use_utf16le && bytes[0] >= 0x20 && bytes[0] < 0x80 && bytes[1] == 0 && - bytes[2] >= 0x20 && bytes[2] < 0x80 && bytes[3] == 0) { - use_utf16le = true; - g_utf16le_wire_latched.store(true, std::memory_order_relaxed); - LOG(INFO) << "BqConvertSQLWCHARToString:: latched UTF-16LE wire format " - "for this process (unixODBC under iODBC-built driver)"; + // When WcharEncoding=UTF-16LE is set in google.googlebigqueryodbc.ini, + // the ODBC manager delivers 2-byte UTF-16LE code units packed into the + // buffer even though sizeof(SQLWCHAR)==4. Read them as uint16_t. + if (IsRuntimeWireUtf16Le()) { + auto const* utf16 = reinterpret_cast(in_str); + if (utf16[0] == 0) { + return std::string(); } - - if (use_utf16le) { - auto const* utf16 = reinterpret_cast(in_str); - if (utf16[0] == 0) { - return std::string(); - } - SQLINTEGER count = in_str_len; - if (count == SQL_NTS || count == NULL) { - count = 0; - while (utf16[count] != 0) ++count; - } - std::wstring wstr; - wstr.reserve(count); - for (SQLINTEGER i = 0; i < count; ++i) { - wstr.push_back(static_cast(utf16[i])); - } - return Utf16ToUtf8(wstr); + SQLINTEGER count = in_str_len; + if (count == SQL_NTS || count == NULL) { + count = 0; + while (utf16[count] != 0) ++count; + } + std::wstring wstr; + wstr.reserve(count); + for (SQLINTEGER i = 0; i < count; ++i) { + wstr.push_back(static_cast(utf16[i])); } + return Utf16ToUtf8(wstr); } #endif diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 673e1fbb4b..4ca84edab8 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -226,17 +226,18 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( odbc_internal::StatusRecordOr BqConvertSQLWCHARToString( SQLWCHAR* in_str, SQLINTEGER in_str_len); -// True once the driver has detected that the loaded ODBC manager delivers -// wide-char buffers as 2-byte UTF-16LE even though the driver was compiled -// with sizeof(SQLWCHAR) == 4 (iODBC headers on Linux/macOS). This is the -// SAP HANA SDA case: HANA's bundled unixODBC speaks UTF-16LE, but the -// release-pipeline binary was built against iODBC. -// -// Latched (process-lifetime, atomic) inside BqConvertSQLWCHARToString the -// first time the wire-format heuristic fires. Always false on Windows and -// for builds where sizeof(SQLWCHAR) is already 2 — no adaptation needed. +// Returns true when WcharEncoding=UTF-16LE is set in +// google.googlebigqueryodbc.ini. Always false on Windows. bool IsRuntimeWireUtf16Le(); +// Apply the WcharEncoding value read from google.googlebigqueryodbc.ini (or +// the Windows registry equivalent). Accepted values: +// "UTF-16LE" ΓÇô 2-byte wire format (unixODBC loaded under iODBC build) +// "UTF-32LE" ΓÇô 4-byte wire format (native iODBC / wchar_t) +// "" ΓÇô default: use sizeof(SQLWCHAR) as-is +// No-op on Windows. +void SetWcharEncodingFromConfig(std::string const& value); + // Bytes per wide character on the wire between this driver and its loaded // manager. Equals sizeof(SQLWCHAR) by default; equals 2 once the UTF-16LE // wire format has been latched. Use this in *every* arithmetic expression From 8e21434fa374f3f7c0982e3d6911fb076c7078f3 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 5 Jun 2026 12:27:34 +0530 Subject: [PATCH 31/32] checkers fix --- .../cloud/odbc/bq_driver/internal/odbc_type_utils.h | 4 +--- google/cloud/odbc/bq_driver/internal/utils.cc | 8 ++++---- google/cloud/odbc/bq_driver/internal/utils.h | 6 +++--- google/cloud/odbc/bq_driver/odbc_api.cc | 12 ++++++------ 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index afea6c7f1e..559797cf14 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -213,11 +213,9 @@ inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, // at byte offset `char_index * WireWcharSize()` from `dest`. inline void WriteWireNul(void* dest, size_t char_index) { size_t const wire_sz = WireWcharSize(); - std::memset(static_cast(dest) + (char_index * wire_sz), 0, - wire_sz); + std::memset(static_cast(dest) + (char_index * wire_sz), 0, wire_sz); } - inline odbc_internal::StatusRecord WStrToOutputBufferResponse( std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length, SQLINTEGER src_len, SQLINTEGER supp_max_len, SQLLEN* res_len) { diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 0c9cea4297..328282fc3b 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -43,11 +43,11 @@ using ::google::cloud::odbc_internal::StatusRecordOr; namespace { // Wire-encoding override read from the [Driver] WcharEncoding key in -// google.googlebigqueryodbc.ini (Linux/macOS) or the equivalent registry key. +// google.googlebigqueryodbc.ini // Only meaningful when sizeof(SQLWCHAR) == 4 (iODBC build on Linux/macOS). -// kDefault ΓÇô use sizeof(SQLWCHAR) as the wire size (no adaptation) -// kUtf16Le ΓÇô 2-byte UTF-16LE wire format (unixODBC loaded driver) -// kUtf32Le ΓÇô 4-byte UTF-32LE wire format (native iODBC) +// kDefault use sizeof(SQLWCHAR) as the wire size (no adaptation) +// kUtf16Le 2-byte UTF-16LE wire format (unixODBC loaded driver) +// kUtf32Le 4-byte UTF-32LE wire format (native iODBC) enum class WcharEncodingOverride { kDefault, kUtf16Le, kUtf32Le }; std::atomic g_wchar_encoding_override{ WcharEncodingOverride::kDefault}; diff --git a/google/cloud/odbc/bq_driver/internal/utils.h b/google/cloud/odbc/bq_driver/internal/utils.h index 4ca84edab8..e4eab8398d 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.h +++ b/google/cloud/odbc/bq_driver/internal/utils.h @@ -232,9 +232,9 @@ bool IsRuntimeWireUtf16Le(); // Apply the WcharEncoding value read from google.googlebigqueryodbc.ini (or // the Windows registry equivalent). Accepted values: -// "UTF-16LE" ΓÇô 2-byte wire format (unixODBC loaded under iODBC build) -// "UTF-32LE" ΓÇô 4-byte wire format (native iODBC / wchar_t) -// "" ΓÇô default: use sizeof(SQLWCHAR) as-is +// "UTF-16LE" 2-byte wire format (unixODBC loaded under iODBC build) +// "UTF-32LE" 4-byte wire format (native iODBC / wchar_t) +// "" default: use sizeof(SQLWCHAR) as-is // No-op on Windows. void SetWcharEncodingFromConfig(std::string const& value); diff --git a/google/cloud/odbc/bq_driver/odbc_api.cc b/google/cloud/odbc/bq_driver/odbc_api.cc index 81b1f5607a..5a8ccebfe7 100644 --- a/google/cloud/odbc/bq_driver/odbc_api.cc +++ b/google/cloud/odbc/bq_driver/odbc_api.cc @@ -282,7 +282,7 @@ SQLRETURN SQL_API SQLDriverConnectW( &out_conn_str_len, driverCompletion); // Handle Unicode conversion of output parameters. - if (SQL_SUCCEEDED(rc) && outConnectionString ) { + if (SQL_SUCCEEDED(rc) && outConnectionString) { StatusRecordOr utf16_out_conn_str; if (out_conn_str_len > 0) { utf16_out_conn_str = Utf8ToUtf16((char*)out_conn_str); @@ -294,7 +294,8 @@ SQLRETURN SQL_API SQLDriverConnectW( return utf16_out_conn_str.GetCalculatedReturnCode(); } - WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, out_conn_str_len); + WriteWideToWireBuffer(*utf16_out_conn_str, outConnectionString, + out_conn_str_len); } if (outConnectionStringLen) *outConnectionStringLen = out_conn_str_len; @@ -923,8 +924,7 @@ SQLRETURN SQL_API SQLGetConnectAttrW(SQLHDBC connectionHandle, } { size_t const wire_sz = WireWcharSize(); - size_t const dest_chars = - static_cast(valueBufferLen) / wire_sz; + size_t const dest_chars = static_cast(valueBufferLen) / wire_sz; size_t const to_copy = std::min(updated_out_attr_status->size(), dest_chars); if (valueStringLen) { @@ -2525,8 +2525,8 @@ SQLRETURN SQL_API SQLGetDiagFieldW(SQLSMALLINT handleType, SQLHANDLE handle, static_cast(diagInfoBufferLen) / wire_sz; size_t const to_copy = std::min(updated_out_diag_info_status->size(), dest_chars); - diag_info_str_len = static_cast( - updated_out_diag_info_status->size() * wire_sz); + diag_info_str_len = + static_cast(updated_out_diag_info_status->size() * wire_sz); WriteWideToWireBuffer(*updated_out_diag_info_status, diagInfo, to_copy); } From 4501c39363c32ed89dd543788c349c2c73d7c019 Mon Sep 17 00:00:00 2001 From: Shivam Dabas Date: Fri, 5 Jun 2026 13:25:26 +0530 Subject: [PATCH 32/32] checkers fix --- .../bq_driver/internal/data_translation.cc | 2 +- .../bq_driver/internal/odbc_sql_tables.cc | 1 - .../odbc/bq_driver/internal/odbc_type_utils.h | 17 +++----- google/cloud/odbc/bq_driver/internal/utils.cc | 43 ++----------------- 4 files changed, 10 insertions(+), 53 deletions(-) diff --git a/google/cloud/odbc/bq_driver/internal/data_translation.cc b/google/cloud/odbc/bq_driver/internal/data_translation.cc index a2a9074328..e0b0ef8568 100644 --- a/google/cloud/odbc/bq_driver/internal/data_translation.cc +++ b/google/cloud/odbc/bq_driver/internal/data_translation.cc @@ -2176,7 +2176,7 @@ void NormalizeDatetimeRange(std::string& src_str) { } } - namespace { +namespace { // Example: [2024-10-10, 2024-10-11) re2::RE2 const kDateRangeRegex(R"(\[\d{4}-\d{2}-\d{2}, \d{4}-\d{2}-\d{2}\))"); diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc index 1ced1ab2d7..adb278ae54 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc @@ -109,7 +109,6 @@ StatusRecordOr> GetFilteredDatasetIds( std::string const& datasets_filter, SQLULEN metadata_id) { std::vector dataset_ids; auto filter_regex = BuildRegex(datasets_filter, metadata_id); - auto filter_regex = BuildRegex(datasets_filter, metadata_id); // For now, we use default options. // We can set timeout here as needed later. Options options; diff --git a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h index 559797cf14..71f40e05a9 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h +++ b/google/cloud/odbc/bq_driver/internal/odbc_type_utils.h @@ -188,24 +188,19 @@ SQLRETURN IntValueToOutputBufferResponse(T val, SQLPOINTER buffer_ptr, inline void WriteWideToWireBuffer(std::wstring const& src, void* dest, size_t count) { if (count > src.size()) count = src.size(); + #if !defined(_WIN32) - if (IsRuntimeWireUtf16Le()) { + if (IsRuntimeWireUtf16Le() || sizeof(SQLWCHAR) != sizeof(wchar_t)) { auto* d = static_cast(dest); for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); - } - return; - } - if constexpr (sizeof(SQLWCHAR) != sizeof(wchar_t)) { - // e.g. unixODBC: SQLWCHAR is 2 bytes, wchar_t is 4 bytes - auto* d = static_cast(dest); - for (size_t i = 0; i < count; ++i) { - d[i] = static_cast(src[i]); + // Cast to unsigned 32-bit first to avoid signed→unsigned misuse warning, + // then narrow to uint16_t (valid for BMP code points / UTF-16 units). + d[i] = static_cast(static_cast(src[i])); } return; } #endif - // Wire SQLWCHAR width matches wchar_t — single memcpy. + std::memcpy(dest, src.data(), count * sizeof(SQLWCHAR)); } diff --git a/google/cloud/odbc/bq_driver/internal/utils.cc b/google/cloud/odbc/bq_driver/internal/utils.cc index 328282fc3b..e11b6ec5c5 100644 --- a/google/cloud/odbc/bq_driver/internal/utils.cc +++ b/google/cloud/odbc/bq_driver/internal/utils.cc @@ -859,10 +859,11 @@ odbc_internal::StatusRecordOr Utf8ToUtf16( iconv_close(cd); - // Resize the output string to the actual converted size + // Resize the output string to the actual converted size. No trailing NUL is + // appended: wstring::size() is the character count, matching the Windows + // MultiByteToWideChar path above. Callers own their own NUL termination. utf16str.resize((outbuf - reinterpret_cast(utf16str.data())) / sizeof(wchar_t)); - utf16str.push_back(L'\0'); return utf16str; #endif } @@ -1053,44 +1054,6 @@ std::string CastOdbcRegexToCppRegex(std::string const& str) { return result; } -// Translate an ODBC LIKE pattern into a C++ regex pattern. -// -// ODBC pattern semantics (per ODBC spec): -// '%' -> any sequence of characters (regex ".*") -// '_' -> any single character (regex ".") -// '\X' -> literal X (the backslash is the escape; consumed in output) -// any other char -> emitted as-is -// -// Implemented as a manual single-pass loop producing an RE2-compatible -// pattern. RE2 is used instead of std::regex to avoid DFA initialization -// crashes in libstdc++/libc++ on certain hosts (e.g. SAP HANA). -std::string CastOdbcRegexToCppRegex(std::string const& str) { - std::string result; - // Worst case: every character becomes ".*" (2 chars). - result.reserve(str.size() * 2); - for (size_t i = 0; i < str.size(); ++i) { - char c = str[i]; - if (c == '\\') { - if (i + 1 < str.size()) { - // Escape: emit the next char literally, consume both the - // backslash and the following char from the input. - result.push_back(str[i + 1]); - ++i; - } - // else: lone trailing backslash — drop it (matches the previous - // regex-based implementation, which stripped all stray backslashes - // at the end of processing). - } else if (c == '%') { - result.append(".*"); - } else if (c == '_') { - result.push_back('.'); - } else { - result.push_back(c); - } - } - return result; -} - std::unique_ptr BuildRegex(std::string filter_pattern, SQLULEN metadata_id) { if (metadata_id == SQL_TRUE) {