feat: default DECIMAL to native Arrow decimal128 on the wire (parity with Python/JDBC)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Release History
 
 ## Unreleased
+- **Default `useArrowNativeDecimal` to `true`** so the Thrift backend requests DECIMAL columns as native Arrow `decimal128` on the wire (a compact fixed-width binary encoding) instead of UTF8 strings, matching the `databricks-sql-python` and `databricks-jdbc` drivers and reducing decimal-heavy result payloads. Scanning through `database/sql` is unchanged — DECIMAL is still returned as a lossless, scale-applied string (Go's `driver.Value` has no decimal type). **Breaking for `GetArrowBatches` consumers only:** DECIMAL columns now arrive as `arrow.Decimal128` arrays rather than string arrays; pass `WithArrowNativeDecimal(false)` (or DSN `useArrowNativeDecimal=false`) to restore the previous string wire format. The kernel backend already renders DECIMAL exactly and is unaffected.
 - Improve telemetry error reporting: driver failures are now categorized by cause instead of reported as a generic error (databricks/databricks-sql-go#414, #415, #417, #419, #424)
 
 ## v1.14.0 (2026-07-13)

README.md

@@ -251,7 +251,7 @@ session parameter on both backends.
 
 | DSN parameter | Connector option | Protocol | Default | Description |
 |---|---|---|---|---|
-| `useArrowNativeDecimal` | `WithArrowNativeDecimal` | Thrift only (inert on kernel) | `false` | Thrift: return DECIMAL as native Arrow `decimal128` (lossless string when scanned via `database/sql`). The kernel path already renders DECIMAL as the exact string regardless. |
+| `useArrowNativeDecimal` | `WithArrowNativeDecimal` | Thrift only (inert on kernel) | `true` | Thrift: return DECIMAL as native Arrow `decimal128` on the wire (lossless string when scanned via `database/sql`), matching the Python and JDBC drivers. Set to `false` for the legacy UTF8-string wire format. The kernel path already renders DECIMAL as the exact string regardless. |
 | | `WithKernelDecimalAsFloat(b)` | SEA only | `false` | Scan top-level DECIMAL as lossy `float64` instead of the exact string. |
 
 See [Cloud Fetch](#cloud-fetch), [TLS](#tls), and [Proxy](#proxy) for the remaining

connector.go

@@ -282,8 +282,13 @@ func withUserConfig(ucfg config.UserConfig) ConnOption {
 		c.UserConfig = ucfg
 		// The useArrowNativeDecimal DSN parameter is carried on UserConfig (all
 		// ParseDSN can return) but is consumed from ArrowConfig. This is the one
-		// place that bridges the two.
-		c.ArrowConfig.UseArrowNativeDecimal = ucfg.UseArrowNativeDecimalDSN
+		// place that bridges the two. Only override the ArrowConfig default when
+		// the DSN actually specified the parameter; a nil carrier means the DSN
+		// was silent, so the native-on default is preserved (an explicit
+		// useArrowNativeDecimal=false still wins).
+		if ucfg.UseArrowNativeDecimalDSN != nil {
+			c.ArrowConfig.UseArrowNativeDecimal = *ucfg.UseArrowNativeDecimalDSN
+		}
 	}
 }
 
@@ -506,13 +511,20 @@ func WithEnableMetricViewMetadata(enable bool) ConnOption {
 }
 
 // WithArrowNativeDecimal controls whether DECIMAL columns are returned as native
-// Arrow decimal128 values. Default is false, in which case the server returns
-// DECIMAL columns as strings.
+// Arrow decimal128 values. Default is true, matching the databricks-sql-python
+// and databricks-jdbc drivers, which request DECIMAL as native Arrow decimal128
+// on the wire (a compact fixed-width binary encoding) rather than UTF8 strings.
 //
 // When enabled, DECIMAL columns retrieved via GetArrowBatches carry the native
 // arrow.Decimal128 type. When scanned through the standard database/sql Rows
 // interface, DECIMAL values are returned as lossless, scale-applied strings to
-// avoid the precision loss that a float64 would introduce.
+// avoid the precision loss that a float64 would introduce (Go's driver.Value
+// contract has no decimal type, so a string is the faithful scalar there).
+//
+// Pass false to restore the legacy behavior in which the server serializes
+// DECIMAL columns as UTF8 strings; GetArrowBatches then yields string-typed
+// columns. The kernel backend always renders DECIMAL exactly and is unaffected
+// by this option.
 //
 // See https://github.com/databricks/databricks-sql-go/issues/274.
 func WithArrowNativeDecimal(useNativeDecimal bool) ConnOption {

connector_test.go

@@ -265,7 +265,7 @@ func TestNewConnector(t *testing.T) {
 		assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
 	})
 
-	t.Run("Connector test WithArrowNativeDecimal disabled by default", func(t *testing.T) {
+	t.Run("Connector test WithArrowNativeDecimal enabled by default", func(t *testing.T) {
 		host := "databricks-host"
 		accessToken := "token"
 		httpPath := "http-path"
@@ -276,6 +276,20 @@ func TestNewConnector(t *testing.T) {
 		)
 		assert.Nil(t, err)
 
+		coni, ok := con.(*connector)
+		require.True(t, ok)
+		assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
+	})
+
+	t.Run("Connector test WithArrowNativeDecimal explicitly disabled", func(t *testing.T) {
+		con, err := NewConnector(
+			WithServerHostname("databricks-host"),
+			WithAccessToken("token"),
+			WithHTTPPath("http-path"),
+			WithArrowNativeDecimal(false),
+		)
+		assert.Nil(t, err)
+
 		coni, ok := con.(*connector)
 		require.True(t, ok)
 		assert.False(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
@@ -295,6 +309,30 @@ func TestNewConnector(t *testing.T) {
 		assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
 	})
 
+	t.Run("Connector test useArrowNativeDecimal=false DSN param overrides the native-on default", func(t *testing.T) {
+		// An explicit =false in the DSN must win over the new native-on default.
+		ucfg, err := config.ParseDSN("token:supersecret@databricks-host:443/sql/1.0/endpoints/abc?useArrowNativeDecimal=false")
+		require.NoError(t, err)
+		con, err := NewConnector(withUserConfig(ucfg))
+		require.NoError(t, err)
+
+		coni, ok := con.(*connector)
+		require.True(t, ok)
+		assert.False(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
+	})
+
+	t.Run("Connector test DSN without useArrowNativeDecimal keeps the native-on default", func(t *testing.T) {
+		// A silent DSN must not reset the default back to false.
+		ucfg, err := config.ParseDSN("token:supersecret@databricks-host:443/sql/1.0/endpoints/abc")
+		require.NoError(t, err)
+		con, err := NewConnector(withUserConfig(ucfg))
+		require.NoError(t, err)
+
+		coni, ok := con.(*connector)
+		require.True(t, ok)
+		assert.True(t, coni.cfg.ArrowConfig.UseArrowNativeDecimal)
+	})
+
 	t.Run("Connector test WithTransport sets HTTPClient in CloudFetchConfig", func(t *testing.T) {
 		host := "databricks-host"
 		accessToken := "token"

doc.go

@@ -39,7 +39,7 @@ Supported optional connection parameters can be specified in param=value and inc
   - userAgentEntry: Used to identify partners. Set as a string with format <isv-name+product-name>
   - useCloudFetch: Used to enable cloud fetch for the query execution. Default is true
   - maxDownloadThreads: Sets up the max number of concurrent workers for cloud fetch. Default is 10
-  - useArrowNativeDecimal: Returns DECIMAL columns as native Arrow decimal128 (via GetArrowBatches); when scanned through database/sql they are returned as lossless strings. Default is false
+  - useArrowNativeDecimal: Returns DECIMAL columns as native Arrow decimal128 (via GetArrowBatches); when scanned through database/sql they are returned as lossless strings. Default is true (matching the Python and JDBC drivers). Set to false to receive DECIMAL as UTF8 strings on the wire.
   - authType: Specifies the desired authentication type. Valid values are: Pat, OauthM2M, OauthU2M
   - accessToken: Personal access token. Required if authType set to Pat
   - clientID: Specifies the client ID to use with OauthM2M

internal/config/config.go

@@ -242,7 +242,13 @@ type UserConfig struct {
 	// connector copies it into Config.ArrowConfig.UseArrowNativeDecimal when it is
 	// assembled. The name intentionally differs from ArrowConfig's field so the
 	// promoted selector Config.UseArrowNativeDecimal stays unambiguous.
-	UseArrowNativeDecimalDSN bool
+	//
+	// It is a *bool so the connector can distinguish "not present in the DSN"
+	// (nil -> keep the ArrowConfig default, which is now native-on) from an
+	// explicit useArrowNativeDecimal=false (which must override the default and
+	// win). A plain bool could not tell those apart and would silently reset the
+	// default to false on the sql.Open(dsn) path.
+	UseArrowNativeDecimalDSN *bool
 	CloudFetchConfig
 	// UseKernel selects the SEA-via-kernel backend instead of Thrift. See the
 	// WithUseKernel connector option for the build requirements. DSN: useKernel=true.
@@ -272,6 +278,12 @@ func (ucfg UserConfig) DeepCopy() UserConfig {
 
 	}
 
+	var nativeDecimalDSN *bool
+	if ucfg.UseArrowNativeDecimalDSN != nil {
+		v := *ucfg.UseArrowNativeDecimalDSN
+		nativeDecimalDSN = &v
+	}
+
 	return UserConfig{
 		Protocol:                 ucfg.Protocol,
 		Host:                     ucfg.Host,
@@ -292,7 +304,7 @@ func (ucfg UserConfig) DeepCopy() UserConfig {
 		Transport:                ucfg.Transport,
 		UseLz4Compression:        ucfg.UseLz4Compression,
 		EnableMetricViewMetadata: ucfg.EnableMetricViewMetadata,
-		UseArrowNativeDecimalDSN: ucfg.UseArrowNativeDecimalDSN,
+		UseArrowNativeDecimalDSN: nativeDecimalDSN,
 		CloudFetchConfig:         ucfg.CloudFetchConfig,
 		EnableTelemetry:          ucfg.EnableTelemetry,
 		TelemetryBatchSize:       ucfg.TelemetryBatchSize,
@@ -445,7 +457,10 @@ func ParseDSN(dsn string) (UserConfig, error) {
 		if err != nil {
 			return UserConfig{}, err
 		}
-		ucfg.UseArrowNativeDecimalDSN = useArrowNativeDecimal
+		// Record only when the param is present so the connector can tell an
+		// explicit override (including =false) from "unspecified", which keeps
+		// the native-on default.
+		ucfg.UseArrowNativeDecimalDSN = &useArrowNativeDecimal
 	}
 
 	// Kernel backend parameters
@@ -672,6 +687,7 @@ type ArrowConfig struct {
 
 func (ucfg ArrowConfig) WithDefaults() ArrowConfig {
 	ucfg.UseArrowBatches = true
+	ucfg.UseArrowNativeDecimal = true
 	ucfg.UseArrowNativeTimestamp = true
 	ucfg.UseArrowNativeComplexTypes = true
 

internal/config/config_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/databricks/databricks-sql-go/internal/cli_service"
 )
 
+func boolPtr(b bool) *bool { return &b }
+
 func TestParseConfig(t *testing.T) {
 	type args struct {
 		dsn string
@@ -271,7 +273,7 @@ func TestParseConfig(t *testing.T) {
 				RetryMax:                 4,
 				RetryWaitMin:             1 * time.Second,
 				RetryWaitMax:             30 * time.Second,
-				UseArrowNativeDecimalDSN: true,
+				UseArrowNativeDecimalDSN: boolPtr(true),
 				CloudFetchConfig:         defCloudConfig,
 			},
 			wantURL: "https://example.cloud.databricks.com:8000/sql/1.0/endpoints/12346a5b5b0e123b",

internal/rows/arrowbased/arrowRows_test.go

@@ -968,6 +968,9 @@ func TestArrowRowScanner(t *testing.T) {
 
 			config := config.WithDefaults()
 			config.UseArrowNativeComplexTypes = false
+			// all_types.json carries the decimal column string-encoded (legacy
+			// wire format), so scan it via the string path.
+			config.ArrowConfig.UseArrowNativeDecimal = false
 			d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil)
 			assert.Nil(t, err)
 
@@ -991,6 +994,9 @@ func TestArrowRowScanner(t *testing.T) {
 
 		config := config.WithDefaults()
 		config.UseArrowNativeComplexTypes = false
+		// all_types.json carries the decimal column string-encoded (legacy wire
+		// format), so scan it via the string path.
+		config.ArrowConfig.UseArrowNativeDecimal = false
 		d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil)
 		assert.Nil(t, err)
 
@@ -1023,6 +1029,9 @@ func TestArrowRowScanner(t *testing.T) {
 
 		config := config.WithDefaults()
 		config.UseArrowNativeComplexTypes = false
+		// all_types.json carries the decimal column string-encoded (legacy wire
+		// format), so scan it via the string path.
+		config.ArrowConfig.UseArrowNativeDecimal = false
 		d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background(), nil)
 		assert.Nil(t, err)