Render nested scalars without encoding/json

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Upgrade go-oidc to v3.21.0 (go-jose v4.1.6) and x/oauth2 to v0.36.0 to address go-jose/v3 security findings (databricks/databricks-sql-go#526).
 - Reduce kernel-backend JSON rendering allocations for nested Boolean and signed-integer values.
+- Render nested ARRAY / MAP / STRUCT scalars without `encoding/json` on both backends, avoiding the per-value overhead of Go 1.27's json/v2-backed `encoding/json`.
 - **Behavior change:** Native Arrow TIMESTAMP_NTZ results now preserve their wall-clock fields and report `TIMESTAMP_NTZ` on both backends. Returned `time.Time` values use UTC instead of the session location; their represented instant and `.UTC()` value are unchanged (databricks/databricks-sql-go#506). On Thrift, this requires a server-supplied Arrow schema with native timestamps; without it, NTZ still reports `TIMESTAMP` and uses session-location handling. String timestamps (`UseArrowNativeTimestamp=false`, internal) and non-Arrow values also retain legacy parsing.
 
 ## v1.18.0 (2026-10-24)

internal/arrowscan/arrowscan.go

@@ -12,7 +12,7 @@
 //   - map         → {"k0":v0,"k1":v1,...}   (keys stringified)
 //   - struct      → {"field0":v0,...}
 //   - nested NULL → null
-//   - time.Time   → quoted .String()  (matches the Thrift marshal() special-case)
+//   - time.Time   → quoted .String()  (matches the Thrift marshalScalar special-case)
 //   - nested decimal → exact scale-applied JSON number literal (never a lossy
 //     float64), matching Thrift's marshalScalar → ValueString
 //   - float32     → native float32 (not widened to float64), so JSON renders
@@ -30,6 +30,7 @@ import (
 	"github.com/apache/arrow/go/v12/arrow"
 	"github.com/apache/arrow/go/v12/arrow/array"
 	"github.com/databricks/databricks-sql-go/internal/decimalfmt"
+	"github.com/databricks/databricks-sql-go/internal/jsonfmt"
 )
 
 // ScanCell extracts one cell as a driver.Value. Scalars map to their Go value:
@@ -581,13 +582,14 @@ func writeStructJSON(b *strings.Builder, s *array.Struct, row int, loc *time.Loc
 // not already a JSON string (numbers/bools become "7"/"true" keys).
 func writeJSONKey(b *strings.Builder, v any) {
 	// A time.Time key renders as its quoted .String() (same special-case as
-	// writeScalarJSON / the Thrift marshal()), not RFC3339.
+	// writeScalarJSON / the Thrift marshalScalar), not RFC3339.
 	if t, ok := v.(time.Time); ok {
 		kb, _ := json.Marshal(t.String())
 		b.Write(kb)
 		return
 	}
-	kb, err := json.Marshal(v)
+	var scratch [32]byte
+	kb, err := jsonfmt.Append(scratch[:0], v)
 	if err != nil {
 		// Unmarshalable key value — fall back to its stringified form, quoted.
 		kb, _ = json.Marshal(fmt.Sprintf("%v", v))
@@ -601,8 +603,9 @@ func writeJSONKey(b *strings.Builder, v any) {
 	b.WriteByte('"')
 }
 
-// writeScalarJSON writes a scalar leaf, mirroring the Thrift marshal(): a
-// time.Time becomes a quoted .String(); everything else uses json.Marshal.
+// writeScalarJSON writes a scalar leaf, mirroring the Thrift marshalScalar: a
+// time.Time becomes a quoted .String(); everything else renders as json.Marshal
+// would.
 func writeScalarJSON(b *strings.Builder, v any) error {
 	if v == nil {
 		b.WriteString("null")
@@ -614,7 +617,8 @@ func writeScalarJSON(b *strings.Builder, v any) error {
 		b.WriteByte('"')
 		return nil
 	}
-	vb, err := json.Marshal(v)
+	var scratch [32]byte
+	vb, err := jsonfmt.Append(scratch[:0], v)
 	if err != nil {
 		return err
 	}

internal/jsonfmt/jsonfmt.go

@@ -0,0 +1,152 @@
+// Package jsonfmt renders scalar values as JSON, byte-identical to
+// encoding/json.Marshal, for the nested-type renderers shared by the Thrift
+// (arrowbased) and kernel (arrowscan) result paths.
+//
+// Those renderers marshal one scalar per nested element, so per-call overhead
+// dominates. Go 1.27 routes encoding/json through the json/v2 engine by default,
+// which made that per-call cost markedly higher; formatting the common scalar
+// types directly avoids it. Anything not handled here falls back to json.Marshal,
+// so output never diverges from it.
+package jsonfmt
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"unicode/utf8"
+)
+
+// Append appends the JSON encoding of v to dst, exactly as json.Marshal(v)
+// would render it, and returns the extended slice.
+func Append(dst []byte, v any) ([]byte, error) {
+	switch t := v.(type) {
+	case nil:
+		return append(dst, "null"...), nil
+	case bool:
+		return strconv.AppendBool(dst, t), nil
+	case int:
+		return strconv.AppendInt(dst, int64(t), 10), nil
+	case int8:
+		return strconv.AppendInt(dst, int64(t), 10), nil
+	case int16:
+		return strconv.AppendInt(dst, int64(t), 10), nil
+	case int32:
+		return strconv.AppendInt(dst, int64(t), 10), nil
+	case int64:
+		return strconv.AppendInt(dst, t, 10), nil
+	case uint8:
+		return strconv.AppendUint(dst, uint64(t), 10), nil
+	case uint16:
+		return strconv.AppendUint(dst, uint64(t), 10), nil
+	case uint32:
+		return strconv.AppendUint(dst, uint64(t), 10), nil
+	case uint64:
+		return strconv.AppendUint(dst, t, 10), nil
+	case float32:
+		if !isFinite(float64(t)) {
+			break
+		}
+		return appendFloat(dst, float64(t), 32), nil
+	case float64:
+		if !isFinite(t) {
+			break
+		}
+		return appendFloat(dst, t, 64), nil
+	case string:
+		if needsEscape(t) {
+			break
+		}
+		dst = append(dst, '"')
+		dst = append(dst, t...)
+		return append(dst, '"'), nil
+	}
+	// NaN/Inf (json.Marshal's error), strings needing escapes, []byte (base64),
+	// and any other type.
+	b, err := json.Marshal(v)
+	if err != nil {
+		return dst, err
+	}
+	if len(dst) == 0 {
+		return b, nil // skip copying json.Marshal's fresh slice
+	}
+	return append(dst, b...), nil
+}
+
+// Marshal is Append into a fresh slice, a drop-in for json.Marshal on scalars.
+func Marshal(v any) ([]byte, error) {
+	return Append(nil, v)
+}
+
+func isFinite(f float64) bool {
+	return !math.IsInf(f, 0) && !math.IsNaN(f)
+}
+
+// appendFloat mirrors encoding/json's ES6-style float formatting: plain decimal
+// unless the magnitude is below 1e-6 or at least 1e21, with exponents unpadded.
+func appendFloat(dst []byte, f float64, bits int) []byte {
+	abs := math.Abs(f)
+	format := byte('f')
+	// Compare at the value's own width so float32 cutoffs match exactly.
+	if abs != 0 {
+		if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
+			format = 'e'
+		}
+	}
+	dst = strconv.AppendFloat(dst, f, format, -1, bits)
+	if format == 'e' {
+		// e-09 -> e-9
+		n := len(dst)
+		if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' {
+			dst[n-2] = dst[n-1]
+			dst = dst[:n-1]
+		}
+	}
+	return dst
+}
+
+// needsEscape reports whether json.Marshal would escape any part of s. It copies
+// ASCII verbatim except control characters, the quote/backslash and the
+// HTML-escaped <, >, &; and valid multi-byte UTF-8 verbatim except U+2028/U+2029.
+// Everything else (including invalid UTF-8) takes the fallback.
+func needsEscape(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if !plainASCII[s[i]] {
+			if s[i] < utf8.RuneSelf {
+				return true
+			}
+			return needsEscapeUTF8(s[i:])
+		}
+	}
+	return false
+}
+
+// needsEscapeUTF8 is needsEscape for the rest of s from its first non-ASCII byte.
+func needsEscapeUTF8(s string) bool {
+	for i := 0; i < len(s); {
+		if c := s[i]; c < utf8.RuneSelf {
+			if !plainASCII[c] {
+				return true
+			}
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r == utf8.RuneError && size == 1 || r == 0x2028 || r == 0x2029 {
+			return true
+		}
+		i += size
+	}
+	return false
+}
+
+// plainASCII marks the ASCII bytes json.Marshal copies verbatim. A table lookup
+// keeps the scan faster than json.Marshal's own on long strings.
+var plainASCII = func() (t [256]bool) {
+	for c := 0x20; c <= 0x7f; c++ {
+		t[c] = true
+	}
+	for _, c := range `"\<>&` {
+		t[c] = false
+	}
+	return t
+}()

internal/jsonfmt/jsonfmt_test.go

@@ -0,0 +1,130 @@
+package jsonfmt
+
+import (
+	"encoding/json"
+	"math"
+	"math/rand"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// assertParity fails unless Marshal(v) matches json.Marshal(v) byte-for-byte,
+// including whether it errors.
+func assertParity(t *testing.T, v any) {
+	t.Helper()
+	want, wantErr := json.Marshal(v)
+	got, gotErr := Marshal(v)
+	if (wantErr != nil) != (gotErr != nil) || wantErr != nil && gotErr.Error() != wantErr.Error() {
+		t.Fatalf("%#v: error mismatch: got %v, want %v", v, gotErr, wantErr)
+	}
+	if wantErr == nil && string(got) != string(want) {
+		t.Fatalf("%#v: got %s, want %s", v, got, want)
+	}
+}
+
+func TestMarshalMatchesEncodingJSON(t *testing.T) {
+	values := []any{
+		nil, true, false,
+		0, -1, math.MaxInt, math.MinInt,
+		int8(math.MinInt8), int8(math.MaxInt8), int16(math.MinInt16), int32(math.MinInt32), int64(math.MinInt64), int64(math.MaxInt64),
+		uint8(255), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64),
+		0.0, math.Copysign(0, -1), 1.0, -1.5, 3.14, 0.1, 1e20, 1e21, 1e-6, 1e-7, 9.999999e-7, 123456789.123456789,
+		math.MaxFloat64, math.SmallestNonzeroFloat64, -math.MaxFloat64,
+		float32(0), float32(math.Copysign(0, -1)), float32(3.14), float32(0.1), float32(1e21), float32(1e-6), float32(1e-7),
+		float32(math.MaxFloat32), float32(math.SmallestNonzeroFloat32),
+		math.NaN(), math.Inf(1), math.Inf(-1), float32(math.NaN()), float32(math.Inf(-1)),
+		"", "alpha", "with space", "a\"b", `back\slash`, "<tag>", "a&b", "tab\there", "nl\n", "\x00", "\x7f",
+		"héllo", "日本語", "

", "\xff\xfe", "emoji 😀",
+		[]byte("abc"), []byte(nil),
+	}
+	for _, v := range values {
+		assertParity(t, v)
+	}
+}
+
+// Every single-byte string, so the escape set can't drift from json.Marshal's.
+func TestMarshalStringBytes(t *testing.T) {
+	for c := 0; c < 256; c++ {
+		assertParity(t, string([]byte{byte(c)}))
+		assertParity(t, "x"+string([]byte{byte(c)})+"y")
+	}
+}
+
+// Every rune (surrogates encode as U+FFFD), so the UTF-8 fast path can't drift
+// from json.Marshal.
+func TestMarshalEveryRune(t *testing.T) {
+	for r := rune(0); r <= utf8.MaxRune; r++ {
+		assertParity(t, "x"+string(r))
+	}
+}
+
+// Random mixes of ASCII, multi-byte runes, U+2028/U+2029 and raw invalid bytes.
+func TestMarshalRandomUnicodeStrings(t *testing.T) {
+	r := rand.New(rand.NewSource(3)) //nolint:gosec // G404: deterministic test inputs, not security
+	pieces := []string{"a", "Z", " ", "<", "&", "\"", "\\", "\n", "\x7f"}
+	for _, cp := range []rune{0xe9, 0xdf, 0x65e5, 0x1f600, 0x2028, 0x2029, 0xa0, 0xfeff, 0xffff} {
+		pieces = append(pieces, string(cp))
+	}
+	for i := 0; i < 100000; i++ {
+		var sb strings.Builder
+		for j, n := 0, r.Intn(12); j < n; j++ {
+			switch r.Intn(4) {
+			case 0:
+				var raw [1]byte // possibly invalid UTF-8
+				_, _ = r.Read(raw[:])
+				sb.Write(raw[:])
+			case 1:
+				sb.WriteRune(r.Int31n(utf8.MaxRune + 1))
+			default:
+				sb.WriteString(pieces[r.Intn(len(pieces))])
+			}
+		}
+		assertParity(t, sb.String())
+	}
+}
+
+// Random bit patterns reach exponent cutoffs, subnormals, and rounding cases a
+// hand-picked list misses.
+func TestMarshalRandomFloats(t *testing.T) {
+	r := rand.New(rand.NewSource(1)) //nolint:gosec // G404: deterministic test inputs, not security
+	for i := 0; i < 200000; i++ {
+		assertParity(t, math.Float64frombits(r.Uint64()))
+		assertParity(t, math.Float32frombits(r.Uint32()))
+		assertParity(t, r.NormFloat64()*math.Pow(10, float64(r.Intn(50)-25)))
+		assertParity(t, float32(r.NormFloat64()*math.Pow(10, float64(r.Intn(30)-15))))
+	}
+}
+
+func TestAppendKeepsPrefix(t *testing.T) {
+	got, err := Append([]byte("["), int32(7))
+	if err != nil || string(got) != "[7" {
+		t.Fatalf("got %q, %v", got, err)
+	}
+	got, err = Append([]byte("["), []byte("abc"))
+	if err != nil || string(got) != `["YWJj"` {
+		t.Fatalf("got %q, %v", got, err)
+	}
+}
+
+func BenchmarkMarshalScalars(b *testing.B) {
+	// The last three take json.Marshal's fallback.
+	values := []any{int32(42), int64(-9000), 3.14, 0.30000000000000004, float32(0.5), true, "alpha0", "café crème",
+		`a"b`, []byte("abc"), math.NaN()}
+	b.Run("jsonfmt", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			for _, v := range values {
+				_, _ = Marshal(v)
+			}
+		}
+	})
+	b.Run("encoding_json", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			for _, v := range values {
+				_, _ = json.Marshal(v)
+			}
+		}
+	})
+}

internal/rows/arrowbased/arrowscan_parity_test.go

@@ -120,6 +120,33 @@ func TestArrowbasedKernelRenderParity(t *testing.T) {
 			b.FieldBuilder(0).(*array.Float32Builder).Append(0.1)
 			return b.NewArray()
 		}},
+		{"list_double", func() arrow.Array {
+			b := array.NewListBuilder(pool, arrow.PrimitiveTypes.Float64)
+			b.Append(true)
+			b.ValueBuilder().(*array.Float64Builder).AppendValues([]float64{0.1, -2.5, 1e-7, 1e21, 123456789.125, 0.30000000000000004}, nil)
+			return b.NewArray()
+		}},
+		// String leaves and map keys with UTF-8, HTML-escaped characters, U+2028
+		// and invalid UTF-8.
+		{"list_string_utf8_html", func() arrow.Array {
+			b := array.NewListBuilder(pool, arrow.BinaryTypes.String)
+			b.Append(true)
+			b.ValueBuilder().(*array.StringBuilder).AppendValues([]string{
+				"café", "日本語", "😀", "<b>&amp;</b>", "a" + string(rune(0x2028)) + "b", "bad\xffutf8", "del\x7f",
+			}, nil)
+			return b.NewArray()
+		}},
+		{"map_utf8_key", func() arrow.Array {
+			b := array.NewMapBuilder(pool, arrow.BinaryTypes.String, arrow.BinaryTypes.String, false)
+			b.Append(true)
+			kb := b.KeyBuilder().(*array.StringBuilder)
+			ib := b.ItemBuilder().(*array.StringBuilder)
+			kb.Append("clé")
+			ib.Append("naïve")
+			kb.Append("a<b")
+			ib.Append("Größe & Ελληνικά")
+			return b.NewArray()
+		}},
 		{"nested_decimal", func() arrow.Array {
 			dt := arrow.StructOf(arrow.Field{Name: "d", Type: &arrow.Decimal128Type{Precision: 5, Scale: 2}})
 			b := array.NewStructBuilder(pool, dt)