fix(bq_driver): Fix encoding for SAP HANA

ci/gha/builds/lib/google.googlebigqueryodbc.ini

@@ -1,3 +1,12 @@
 [Driver]
 LogLevel=0
 LogPath=
+# WcharEncoding sets the wire encoding of SQLWCHAR buffers on Linux/macOS
+# when the driver is built against iODBC headers (sizeof(SQLWCHAR) == 4).
+#
+# Accepted values:
+#   UTF-16LE  - 2-byte UTF-16LE per code unit (unixODBC-loaded driver)
+#   UTF-32LE  - 4-byte UTF-32LE per code unit (native iODBC)
+#   (empty)   - use sizeof(SQLWCHAR) as-is (default)
+#
+WcharEncoding=

google/cloud/odbc/CMakeLists.txt

@@ -159,6 +159,15 @@ else ()
     find_package(google_cloud_cpp_serviceusage REQUIRED)
 endif ()
 
+find_package(re2 CONFIG QUIET)
+if (NOT TARGET re2::re2)
+    if (TARGET re2)
+        add_library(re2::re2 ALIAS re2)
+    else ()
+        find_package(re2 REQUIRED)
+    endif ()
+endif ()
+
 # Restore the original BUILD_SHARED_LIBS value
 set(BUILD_SHARED_LIBS ${ORIGINAL_BUILD_SHARED_LIBS})
 

google/cloud/odbc/bq_driver/internal/data_translation.cc

@@ -104,7 +104,7 @@ odbc_internal::StatusRecord ConvertFromNumericDSValue(DSValue const& src_dsval,
                                      "DSValueToWchar Conversion Failed"};
         break;
       }
-      SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize();
       auto src_len = static_cast<SQLINTEGER>(wstr->length());
       SQLINTEGER required_chars = src_len + 1;
       WStrToOutputBufferResponse(wstr.GetValue(), dest_data.buf, wchar_capacity,
@@ -326,7 +326,7 @@ odbc_internal::StatusRecord ConvertFromStringDSValue(DSValue const& src_dsval,
     }
 
     auto src_len = static_cast<SQLINTEGER>(wide_str.length());
-    SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR);
+    SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize();
     SQLINTEGER required_chars = src_len + 1;
     return WStrToOutputBufferResponse(wide_str, dest_data.buf, wchar_capacity,
                                       src_len, required_chars,
@@ -907,7 +907,7 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval,
                                      "DSValueToWchar Conversion Failed"};
         break;
       }
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       SQLLEN required_chars = static_cast<SQLLEN>(wstr->length()) + 1;
       return WStrToOutputBufferResponse(
           wstr.GetValue(), dest_buf, wchar_capacity, k_time_src_len,
@@ -1007,30 +1007,23 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue(
                                      "DSValueToWchar Conversion Failed"};
         break;
       }
-      std::wstring wstr_val = wstr.GetValue();
-      if (!wstr_val.empty() && wstr_val.back() == L'\0') {
-        wstr_val.pop_back();
-      }
-      std::vector<SQLWCHAR> wstr_data(wstr_val.begin(), wstr_val.end());
-      wstr_data.emplace_back(L'\0');
-
-      auto* dest = reinterpret_cast<SQLWCHAR*>(dest_buf);
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      size_t const wire_sz = WireWcharSize();
+      SQLLEN wchar_capacity = buffer_length / static_cast<SQLLEN>(wire_sz);
       if (wchar_capacity > k_timestamp_src_len) {
         if (res_len) {
-          *res_len = k_timestamp_src_len * sizeof(SQLWCHAR);
+          *res_len = static_cast<SQLLEN>(wstr.GetValue().size() * wire_sz);
         }
-        std::memcpy(dest, wstr_data.data(),
-                    (k_timestamp_src_len) * sizeof(SQLWCHAR));
-        dest[k_timestamp_src_len] = L'\0';
+        WriteWideToWireBuffer(wstr.GetValue(), dest_buf,
+                              wstr.GetValue().size());
+        WriteWireNul(dest_buf, wstr.GetValue().size());
       } else if (20 <= wchar_capacity &&
                  wchar_capacity <= k_timestamp_src_len) {
         if (res_len) {
-          *res_len = wchar_capacity * sizeof(SQLWCHAR);
+          *res_len = wchar_capacity * static_cast<SQLLEN>(wire_sz);
         }
-        std::memcpy(dest, wstr_data.data(),
-                    (wchar_capacity) * sizeof(SQLWCHAR));
-        dest[wchar_capacity - 1] = L'\0';
+        WriteWideToWireBuffer(wstr.GetValue(), dest_buf,
+                              static_cast<size_t>(wchar_capacity - 1));
+        WriteWireNul(dest_buf, static_cast<size_t>(wchar_capacity - 1));
         LOG(WARNING)
             << "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR.";
         status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"};
@@ -1189,29 +1182,22 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval,
                                      "DSValueToWchar Conversion Failed"};
         break;
       }
-      std::wstring wstr_val = wstr.GetValue();
-      if (!wstr_val.empty() && wstr_val.back() == L'\0') {
-        wstr_val.pop_back();
-      }
-      std::vector<SQLWCHAR> wstr_data(wstr_val.begin(), wstr_val.end());
-      wstr_data.emplace_back(L'\0');
-
-      auto* dest = reinterpret_cast<SQLWCHAR*>(dest_buf);
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      size_t const wire_sz = WireWcharSize();
+      SQLLEN wchar_capacity = buffer_length / static_cast<SQLLEN>(wire_sz);
       if (wchar_capacity > k_datetime_src_len) {
         if (res_len) {
-          *res_len = k_datetime_src_len * sizeof(SQLWCHAR);
+          *res_len = static_cast<SQLLEN>(wstr.GetValue().size() * wire_sz);
         }
-        std::memcpy(dest, wstr_data.data(),
-                    (k_datetime_src_len) * sizeof(SQLWCHAR));
-        dest[k_datetime_src_len] = L'\0';
+        WriteWideToWireBuffer(wstr.GetValue(), dest_buf,
+                              wstr.GetValue().size());
+        WriteWireNul(dest_buf, wstr.GetValue().size());
       } else if (20 <= wchar_capacity && wchar_capacity <= k_datetime_src_len) {
         if (res_len) {
-          *res_len = wchar_capacity * sizeof(SQLWCHAR);
+          *res_len = wchar_capacity * static_cast<SQLLEN>(wire_sz);
         }
-        std::memcpy(dest, wstr_data.data(),
-                    (wchar_capacity) * sizeof(SQLWCHAR));
-        dest[wchar_capacity - 1] = L'\0';
+        WriteWideToWireBuffer(wstr.GetValue(), dest_buf,
+                              static_cast<size_t>(wchar_capacity - 1));
+        WriteWireNul(dest_buf, static_cast<size_t>(wchar_capacity - 1));
         LOG(WARNING)
             << "ConvertFromDatetimeDSValue:: Data truncated for SQL_C_WCHAR.";
         status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"};
@@ -1398,7 +1384,7 @@ odbc_internal::StatusRecord ConvertFromDateDSValue(DSValue const& src_dsval,
         return StatusRecord{SQLStates::k_HY000(),
                             "DSValueToWchar Conversion Failed"};
       }
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       auto src_len = static_cast<SQLINTEGER>(wstr->length());
       SQLINTEGER required_chars = src_len + 1;
       return WStrToOutputBufferResponse(
@@ -1434,7 +1420,7 @@ StatusRecord ConvertStringToJsonOutputBuffer(std::string const& src_str,
         return StatusRecord{SQLStates::k_HY000(),
                             "Conversion to UTF-16 failed"};
       }
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       auto src_len = static_cast<SQLINTEGER>(wide_string->length());
       SQLINTEGER required_chars = src_len + 1;
       return WStrToOutputBufferResponse(wide_string.GetValue(), dest_buf,
@@ -1498,15 +1484,11 @@ StatusRecord ConvertFromArrayDSValue(DSValue const& src_dsval,
       if (!wide_string.Ok()) {
         return StatusRecord{SQLStates::k_HY000(), "Conversion Failed"};
       }
-      std::wstring wide_val = wide_string.GetValue();
-      if (!wide_val.empty() && wide_val.back() == L'\0') {
-        wide_val.pop_back();
-      }
-      SQLLEN wchar_capacity = dest_data.buflen / sizeof(SQLWCHAR);
-      auto src_len = static_cast<SQLINTEGER>(wide_val.length());
+      SQLLEN wchar_capacity = dest_data.buflen / WireWcharSize();
+      auto src_len = static_cast<SQLINTEGER>(wide_string->length());
       SQLINTEGER required_chars = src_len + 1;
       return WStrToOutputBufferResponse(
-          wide_val, dest_data.buf, wchar_capacity, src_len, required_chars,
+          *wide_string, dest_data.buf, wchar_capacity, src_len, required_chars,
           reinterpret_cast<SQLLEN*>(dest_data.result_len));
     }
     case SQL_C_BINARY: {
@@ -1621,7 +1603,7 @@ odbc_internal::StatusRecord ConvertFromIntervalDSValue(DSValue const& src_dsval,
             StatusRecord{SQLStates::k_HY000(), wstr.GetStatusRecord().message};
         break;
       }
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       auto interval_char_length =
           static_cast<SQLINTEGER>(wstr.GetValue().length());
       return WStrIntervalBufferResponse(
@@ -1907,7 +1889,7 @@ StatusRecord ConvertFromGeographyDSValue(DSValue const& src_dsval,
       }
       std::memset(dest_data.buf, 0, buffer_length);
       std::wstring const& wide_str = wstr.GetValue();
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       SQLLEN src_len = static_cast<SQLLEN>(wide_str.length());
       SQLLEN required_chars = src_len + 1;
       status_record = WStrToOutputBufferResponse(
@@ -2054,38 +2036,35 @@ StatusRecord ConvertBytesToWChar(DSValue const& conn_val,
                         "UTF-8 to UTF-16 conversion failed."};
   }
 
-  std::wstring utf16_value = utf16_str.GetValue();
-  if (!utf16_value.empty() && utf16_value.back() == L'\0') {
-    utf16_value.pop_back();
-  }
-  size_t const required_size = utf16_value.length() * sizeof(SQLWCHAR);
-
-  auto* buffer = reinterpret_cast<SQLWCHAR*>(dest_data.buf);
+  std::wstring const& utf16_value = utf16_str.GetValue();
 
-  // Handle truncation if buffer is insufficient
-  if (dest_data.buflen < required_size) {
-    size_t num_chars_to_copy = (dest_data.buflen / sizeof(SQLWCHAR)) - 1;
-    std::memcpy(buffer, utf16_value.data(),
-                num_chars_to_copy * sizeof(SQLWCHAR));
-    buffer[num_chars_to_copy] = L'\0';
+  // Narrow wchar_t -> wire encoding directly into the caller's buffer.
+  // No intermediate vector; WriteWideToWireBuffer is a memcpy when the wire
+  // SQLWCHAR width matches sizeof(wchar_t) and a per-element narrowing loop
+  // only on the iODBC-built / unixODBC-loaded path.
+  size_t const wire_sz = WireWcharSize();
+  size_t const src_chars = utf16_value.size();
+  size_t const required_size = src_chars * wire_sz;
 
+  if (static_cast<size_t>(dest_data.buflen) < required_size) {
+    size_t num_chars_to_copy = dest_data.buflen / wire_sz;
+    if (num_chars_to_copy > 0) {
+      num_chars_to_copy--;  // leave one slot for the null terminator
+      WriteWideToWireBuffer(utf16_value, dest_data.buf, num_chars_to_copy);
+      WriteWireNul(dest_data.buf, num_chars_to_copy);
+    }
     if (dest_data.result_len) {
-      *dest_data.result_len = dest_data.buflen;
+      *dest_data.result_len = required_size;
     }
     LOG(WARNING) << "ConvertBytesToWChar:: String data, right truncated.";
     return StatusRecord{SQLStates::k_01004(), "String data, right truncated"};
   }
-  for (size_t i = 0; i < utf16_str.GetValue().size(); ++i) {
-    buffer[i] = static_cast<SQLWCHAR>(utf16_str.GetValue()[i]);
-  }
-  size_t buffer_chars = dest_data.buflen / sizeof(SQLWCHAR);
-  if (utf16_str.GetValue().size() < buffer_chars) {
-    buffer[utf16_str.GetValue().size()] = L'\0';
+  WriteWideToWireBuffer(utf16_value, dest_data.buf, src_chars);
+  if (static_cast<size_t>(dest_data.buflen) >= required_size + wire_sz) {
+    WriteWireNul(dest_data.buf, src_chars);
   }
-
-  // Set output length
   if (dest_data.result_len) {
-    *dest_data.result_len = utf16_str.GetValue().size() * sizeof(SQLWCHAR);
+    *dest_data.result_len = required_size;
   }
   return status_record;
 }
@@ -2253,7 +2232,7 @@ StatusRecord ConvertFromRangeDSValue(DSValue const& src_dsval,
         return StatusRecord{SQLStates::k_HY000(),
                             "Conversion to SQL_C_WCHAR failed."};
       }
-      SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
+      SQLLEN wchar_capacity = buffer_length / WireWcharSize();
       SQLLEN src_len = static_cast<SQLLEN>(wstr->length());
       SQLLEN required_chars = src_len + 1;
       return WStrToOutputBufferResponse(

google/cloud/odbc/bq_driver/internal/data_translation_inv.cc

@@ -53,7 +53,7 @@ StatusRecordOr<std::string> ConvertFromCharBuffer(DataBuffer& src_data,
       auto* wchar_buf = static_cast<SQLWCHAR*>(src_buf);
       if ((result_len > 0) || (result_len == SQL_NTS)) {
         if (result_len > 0) {
-          result_len /= sizeof(SQLWCHAR);
+          result_len /= WireWcharSize();
         }
         auto utf8_res = BqConvertSQLWCHARToString(
             wchar_buf, static_cast<SQLINTEGER>(result_len));

google/cloud/odbc/bq_driver/internal/odbc_desc_attr.cc

@@ -14,6 +14,7 @@
 
 #include "google/cloud/odbc/bq_driver/internal/odbc_desc_attr.h"
 #include "google/cloud/odbc/bq_driver/internal/trace_utils.h"
+#include "google/cloud/odbc/bq_driver/internal/utils.h"
 #include "google/cloud/odbc/internal/sql_state_constants.h"
 #include "google/cloud/odbc/internal/status_record_or.h"
 #include <vector>
@@ -388,7 +389,7 @@ StatusRecord DescriptorRecord::SetOctetLength(SQLSMALLINT type,
     case SQL_WCHAR:
     case SQL_WVARCHAR:
     case SQL_WLONGVARCHAR:
-      octet_length = value * sizeof(SQLWCHAR);
+      octet_length = value * WireWcharSize();
       break;
     case SQL_DECIMAL:
     case SQL_NUMERIC:

google/cloud/odbc/bq_driver/internal/odbc_sql_columns.cc

@@ -356,7 +356,8 @@ StatusRecordOr<ResultSet> ProcessTableResults(
     for (TableFieldSchema const& table_field_schema : bq_table.schema.fields) {
       // bq_table_column could contain a search pattern character so do a regex
       // match.
-      auto column_pattern = BuildRegex(bq_table_column, metadata_id);
+      std::unique_ptr<re2::RE2> column_pattern =
+          BuildRegex(bq_table_column, metadata_id);
       if (re2::RE2::FullMatch(table_field_schema.name, *column_pattern)) {
         auto ds_row_status = CreateResultSetDSRow(
             conn_handle, bq_table.table_reference.project_id,

google/cloud/odbc/bq_driver/internal/odbc_sql_columns_test.cc

@@ -299,7 +299,7 @@ void ProcessTableResultsHelper(std::string const& column,
   expected_sql_int_row.ord_pos = (column == "%" || column.empty()) ? 2 : 1;
   expected_sql_int_row.is_nullable = "NO";
 
-  auto column_pattern = BuildRegex(column, metadata_id);
+  std::unique_ptr<re2::RE2> column_pattern = BuildRegex(column, metadata_id);
 
   if (!metadata_id && (column.empty() || column == "%")) {
     ASSERT_EQ(result_set.rows.size(), 2);

google/cloud/odbc/bq_driver/internal/odbc_sql_tables.cc

@@ -83,7 +83,8 @@ StatusRecordOr<std::vector<std::string>> GetFilteredProjectIds(
     ODBCBQClient& bq_client, std::string const& projects_filter,
     SQLULEN metadata_id) {
   std::vector<std::string> project_ids;
-  auto filter_regex = BuildRegex(projects_filter, metadata_id);
+  std::unique_ptr<re2::RE2> filter_regex =
+      BuildRegex(projects_filter, metadata_id);
   // For now, we use default options.
   // We can set timeout here as needed later.
   Options options;

google/cloud/odbc/bq_driver/internal/odbc_type_utils.cc

@@ -42,25 +42,24 @@ SQLRETURN AddressToPointer(SQLPOINTER ptr, SQLPOINTER out_buf,
 }
 
 odbc_internal::StatusRecord WStrIntervalBufferResponse(
-    std::wstring wstr, SQLPOINTER dest_buf, SQLLEN buffer_length,
+    std::wstring const& wstr, SQLPOINTER dest_buf, SQLLEN buffer_length,
     SQLINTEGER char_len, SQLINTEGER whole_digits_count, SQLLEN* res_len) {
   auto status_record = odbc_internal::StatusRecord::Ok();
-  std::vector<SQLWCHAR> wstr_data(wstr.begin(), wstr.end());
-  wstr_data.emplace_back(L'\0');
+  size_t const wire_sz = WireWcharSize();
 
-  auto* dest = static_cast<SQLWCHAR*>(dest_buf);
   if (buffer_length > char_len) {
     if (res_len) {
-      *res_len = char_len * sizeof(SQLWCHAR);
+      *res_len = static_cast<SQLLEN>(char_len) * static_cast<SQLLEN>(wire_sz);
     }
-    std::memcpy(dest, wstr_data.data(), (char_len) * sizeof(SQLWCHAR));
-    dest[char_len] = L'\0';
+    WriteWideToWireBuffer(wstr, dest_buf, static_cast<size_t>(char_len));
+    WriteWireNul(dest_buf, static_cast<size_t>(char_len));
   } else if (buffer_length > whole_digits_count) {
     if (res_len) {
-      *res_len = buffer_length * sizeof(SQLWCHAR);
+      *res_len = buffer_length * static_cast<SQLLEN>(wire_sz);
     }
-    std::memcpy(dest, wstr_data.data(), (buffer_length) * sizeof(SQLWCHAR));
-    dest[buffer_length - 1] = L'\0';
+    WriteWideToWireBuffer(wstr, dest_buf,
+                          static_cast<size_t>(buffer_length - 1));
+    WriteWireNul(dest_buf, static_cast<size_t>(buffer_length - 1));
     status_record = odbc_internal::StatusRecord{
         google::cloud::odbc_internal::SQLStates::k_01004(), "Data truncated"};
   } else {