diff --git a/CHANGELOG.md b/CHANGELOG.md index 8acc5d8f..79aaf486 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History ## Unreleased +- **Default `useArrowNativeDecimal` to `true`** so the Thrift backend requests DECIMAL columns as native Arrow `decimal128` on the wire (a compact fixed-width binary encoding) instead of UTF8 strings, matching the `databricks-sql-python` and `databricks-jdbc` drivers and reducing decimal-heavy result payloads. Scanning through `database/sql` is unchanged — DECIMAL is still returned as a lossless, scale-applied string (Go's `driver.Value` has no decimal type). **Breaking for `GetArrowBatches` consumers only:** DECIMAL columns now arrive as `arrow.Decimal128` arrays rather than string arrays; pass `WithArrowNativeDecimal(false)` (or DSN `useArrowNativeDecimal=false`) to restore the previous string wire format. The kernel backend already renders DECIMAL exactly and is unaffected. - Improve telemetry error reporting: driver failures are now categorized by cause instead of reported as a generic error (databricks/databricks-sql-go#414, #415, #417, #419, #424) ## v1.14.0 (2026-07-13) diff --git a/README.md b/README.md index 425c0c62..f877755f 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ session parameter on both backends. | DSN parameter | Connector option | Protocol | Default | Description | |---|---|---|---|---| -| `useArrowNativeDecimal` | `WithArrowNativeDecimal` | Thrift only (inert on kernel) | `false` | Thrift: return DECIMAL as native Arrow `decimal128` (lossless string when scanned via `database/sql`). The kernel path already renders DECIMAL as the exact string regardless. | +| `useArrowNativeDecimal` | `WithArrowNativeDecimal` | Thrift only (inert on kernel) | `true` | Thrift: return DECIMAL as native Arrow `decimal128` on the wire (lossless string when scanned via `database/sql`), matching the Python and JDBC drivers. Set to `false` for the legacy UTF8-string wire format. The kernel path already renders DECIMAL as the exact string regardless. | | | `WithKernelDecimalAsFloat(b)` | SEA only | `false` | Scan top-level DECIMAL as lossy `float64` instead of the exact string. | See [Cloud Fetch](#cloud-fetch), [TLS](#tls), and [Proxy](#proxy) for the remaining diff --git a/connector.go b/connector.go index 040c3271..3c602426 100644 --- a/connector.go +++ b/connector.go @@ -282,8 +282,13 @@ func withUserConfig(ucfg config.UserConfig) ConnOption { c.UserConfig = ucfg // The useArrowNativeDecimal DSN parameter is carried on UserConfig (all // ParseDSN can return) but is consumed from ArrowConfig. This is the one - // place that bridges the two. - c.ArrowConfig.UseArrowNativeDecimal = ucfg.UseArrowNativeDecimalDSN + // place that bridges the two. Only override the ArrowConfig default when + // the DSN actually specified the parameter; a nil carrier means the DSN + // was silent, so the native-on default is preserved (an explicit + // useArrowNativeDecimal=false still wins). + if ucfg.UseArrowNativeDecimalDSN != nil { + c.ArrowConfig.UseArrowNativeDecimal = *ucfg.UseArrowNativeDecimalDSN + } } } @@ -506,13 +511,20 @@ func WithEnableMetricViewMetadata(enable bool) ConnOption { } // WithArrowNativeDecimal controls whether DECIMAL columns are returned as native -// Arrow decimal128 values. Default is false, in which case the server returns -// DECIMAL columns as strings. +// Arrow decimal128 values. Default is true, matching the databricks-sql-python +// and databricks-jdbc drivers, which request DECIMAL as native Arrow decimal128 +// on the wire (a compact fixed-width binary encoding) rather than UTF8 strings. // // When enabled, DECIMAL columns retrieved via GetArrowBatches carry the native // arrow.Decimal128 type. When scanned through the standard database/sql Rows // interface, DECIMAL values are returned as lossless, scale-applied strings to -// avoid the precision loss that a float64 would introduce. +// avoid the precision loss that a float64 would introduce (Go's driver.Value +// contract has no decimal type, so a string is the faithful scalar there). +// +// Pass false to restore the legacy behavior in which the server serializes +// DECIMAL columns as UTF8 strings; GetArrowBatches then yields string-typed +// columns. The kernel backend always renders DECIMAL exactly and is unaffected +// by this option. // // See https://github.com/databricks/databricks-sql-go/issues/274. func WithArrowNativeDecimal(useNativeDecimal bool) ConnOption { diff --git a/connector_test.go b/connector_test.go index 9eff225b..83e2bf45 100644 --- a/connector_test.go +++ b/connector_test.go @@ -265,7 +265,7 @@ func TestNewConnector(t *testing.T) { assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) }) - t.Run("Connector test WithArrowNativeDecimal disabled by default", func(t *testing.T) { + t.Run("Connector test WithArrowNativeDecimal enabled by default", func(t *testing.T) { host := "databricks-host" accessToken := "token" httpPath := "http-path" @@ -276,6 +276,20 @@ func TestNewConnector(t *testing.T) { ) assert.Nil(t, err) + coni, ok := con.(*connector) + require.True(t, ok) + assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) + }) + + t.Run("Connector test WithArrowNativeDecimal explicitly disabled", func(t *testing.T) { + con, err := NewConnector( + WithServerHostname("databricks-host"), + WithAccessToken("token"), + WithHTTPPath("http-path"), + WithArrowNativeDecimal(false), + ) + assert.Nil(t, err) + coni, ok := con.(*connector) require.True(t, ok) assert.False(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) @@ -295,6 +309,30 @@ func TestNewConnector(t *testing.T) { assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) }) + t.Run("Connector test useArrowNativeDecimal=false DSN param overrides the native-on default", func(t *testing.T) { + // An explicit =false in the DSN must win over the new native-on default. + ucfg, err := config.ParseDSN("token:supersecret@databricks-host:443/sql/1.0/endpoints/abc?useArrowNativeDecimal=false") + require.NoError(t, err) + con, err := NewConnector(withUserConfig(ucfg)) + require.NoError(t, err) + + coni, ok := con.(*connector) + require.True(t, ok) + assert.False(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) + }) + + t.Run("Connector test DSN without useArrowNativeDecimal keeps the native-on default", func(t *testing.T) { + // A silent DSN must not reset the default back to false. + ucfg, err := config.ParseDSN("token:supersecret@databricks-host:443/sql/1.0/endpoints/abc") + require.NoError(t, err) + con, err := NewConnector(withUserConfig(ucfg)) + require.NoError(t, err) + + coni, ok := con.(*connector) + require.True(t, ok) + assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal) + }) + t.Run("Connector test WithTransport sets HTTPClient in CloudFetchConfig", func(t *testing.T) { host := "databricks-host" accessToken := "token" diff --git a/doc.go b/doc.go index 266deff0..3d04877c 100644 --- a/doc.go +++ b/doc.go @@ -39,7 +39,7 @@ Supported optional connection parameters can be specified in param=value and inc - userAgentEntry: Used to identify partners. Set as a string with format - useCloudFetch: Used to enable cloud fetch for the query execution. Default is true - maxDownloadThreads: Sets up the max number of concurrent workers for cloud fetch. Default is 10 - - useArrowNativeDecimal: Returns DECIMAL columns as native Arrow decimal128 (via GetArrowBatches); when scanned through database/sql they are returned as lossless strings. Default is false + - useArrowNativeDecimal: Returns DECIMAL columns as native Arrow decimal128 (via GetArrowBatches); when scanned through database/sql they are returned as lossless strings. Default is true (matching the Python and JDBC drivers). Set to false to receive DECIMAL as UTF8 strings on the wire. - authType: Specifies the desired authentication type. Valid values are: Pat, OauthM2M, OauthU2M - accessToken: Personal access token. Required if authType set to Pat - clientID: Specifies the client ID to use with OauthM2M diff --git a/internal/config/config.go b/internal/config/config.go index 6434eb15..45bb9995 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -242,7 +242,13 @@ type UserConfig struct { // connector copies it into Config.ArrowConfig.UseArrowNativeDecimal when it is // assembled. The name intentionally differs from ArrowConfig's field so the // promoted selector Config.UseArrowNativeDecimal stays unambiguous. - UseArrowNativeDecimalDSN bool + // + // It is a *bool so the connector can distinguish "not present in the DSN" + // (nil -> keep the ArrowConfig default, which is now native-on) from an + // explicit useArrowNativeDecimal=false (which must override the default and + // win). A plain bool could not tell those apart and would silently reset the + // default to false on the sql.Open(dsn) path. + UseArrowNativeDecimalDSN *bool CloudFetchConfig // UseKernel selects the SEA-via-kernel backend instead of Thrift. See the // WithUseKernel connector option for the build requirements. DSN: useKernel=true. @@ -272,6 +278,12 @@ func (ucfg UserConfig) DeepCopy() UserConfig { } + var nativeDecimalDSN *bool + if ucfg.UseArrowNativeDecimalDSN != nil { + v := *ucfg.UseArrowNativeDecimalDSN + nativeDecimalDSN = &v + } + return UserConfig{ Protocol: ucfg.Protocol, Host: ucfg.Host, @@ -292,7 +304,7 @@ func (ucfg UserConfig) DeepCopy() UserConfig { Transport: ucfg.Transport, UseLz4Compression: ucfg.UseLz4Compression, EnableMetricViewMetadata: ucfg.EnableMetricViewMetadata, - UseArrowNativeDecimalDSN: ucfg.UseArrowNativeDecimalDSN, + UseArrowNativeDecimalDSN: nativeDecimalDSN, CloudFetchConfig: ucfg.CloudFetchConfig, EnableTelemetry: ucfg.EnableTelemetry, TelemetryBatchSize: ucfg.TelemetryBatchSize, @@ -445,7 +457,10 @@ func ParseDSN(dsn string) (UserConfig, error) { if err != nil { return UserConfig{}, err } - ucfg.UseArrowNativeDecimalDSN = useArrowNativeDecimal + // Record only when the param is present so the connector can tell an + // explicit override (including =false) from "unspecified", which keeps + // the native-on default. + ucfg.UseArrowNativeDecimalDSN = &useArrowNativeDecimal } // Kernel backend parameters @@ -672,6 +687,7 @@ type ArrowConfig struct { func (ucfg ArrowConfig) WithDefaults() ArrowConfig { ucfg.UseArrowBatches = true + ucfg.UseArrowNativeDecimal = true ucfg.UseArrowNativeTimestamp = true ucfg.UseArrowNativeComplexTypes = true diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c4160c25..2a74a439 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -13,6 +13,8 @@ import ( "github.com/databricks/databricks-sql-go/internal/cli_service" ) +func boolPtr(b bool) *bool { return &b } + func TestParseConfig(t *testing.T) { type args struct { dsn string @@ -271,7 +273,7 @@ func TestParseConfig(t *testing.T) { RetryMax: 4, RetryWaitMin: 1 * time.Second, RetryWaitMax: 30 * time.Second, - UseArrowNativeDecimalDSN: true, + UseArrowNativeDecimalDSN: boolPtr(true), CloudFetchConfig: defCloudConfig, }, wantURL: "https://example.cloud.databricks.com:8000/sql/1.0/endpoints/12346a5b5b0e123b", diff --git a/internal/rows/arrowbased/arrowRows_test.go b/internal/rows/arrowbased/arrowRows_test.go index 713ab314..157a37db 100644 --- a/internal/rows/arrowbased/arrowRows_test.go +++ b/internal/rows/arrowbased/arrowRows_test.go @@ -968,6 +968,9 @@ func TestArrowRowScanner(t *testing.T) { config := config.WithDefaults() config.UseArrowNativeComplexTypes = false + // all_types.json carries the decimal column string-encoded (legacy + // wire format), so scan it via the string path. + config.ArrowConfig.UseArrowNativeDecimal = false d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil) assert.Nil(t, err) @@ -991,6 +994,9 @@ func TestArrowRowScanner(t *testing.T) { config := config.WithDefaults() config.UseArrowNativeComplexTypes = false + // all_types.json carries the decimal column string-encoded (legacy wire + // format), so scan it via the string path. + config.ArrowConfig.UseArrowNativeDecimal = false d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil) assert.Nil(t, err) @@ -1023,6 +1029,9 @@ func TestArrowRowScanner(t *testing.T) { config := config.WithDefaults() config.UseArrowNativeComplexTypes = false + // all_types.json carries the decimal column string-encoded (legacy wire + // format), so scan it via the string path. + config.ArrowConfig.UseArrowNativeDecimal = false d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil) assert.Nil(t, err)