diff --git a/CHANGELOG.md b/CHANGELOG.md index 066fac4e..0c3ba196 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Upgrade go-oidc to v3.21.0 (go-jose v4.1.6) and x/oauth2 to v0.36.0 to address go-jose/v3 security findings (databricks/databricks-sql-go#526). - Reduce kernel-backend JSON rendering allocations for nested Boolean and signed-integer values. +- Render nested ARRAY / MAP / STRUCT scalars without `encoding/json` on both backends, avoiding the per-value overhead of Go 1.27's json/v2-backed `encoding/json`. - **Behavior change:** Native Arrow TIMESTAMP_NTZ results now preserve their wall-clock fields and report `TIMESTAMP_NTZ` on both backends. Returned `time.Time` values use UTC instead of the session location; their represented instant and `.UTC()` value are unchanged (databricks/databricks-sql-go#506). On Thrift, this requires a server-supplied Arrow schema with native timestamps; without it, NTZ still reports `TIMESTAMP` and uses session-location handling. String timestamps (`UseArrowNativeTimestamp=false`, internal) and non-Arrow values also retain legacy parsing. ## v1.18.0 (2026-10-24) diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index cbec7a1b..a201610c 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -12,7 +12,7 @@ // - map → {"k0":v0,"k1":v1,...} (keys stringified) // - struct → {"field0":v0,...} // - nested NULL → null -// - time.Time → quoted .String() (matches the Thrift marshal() special-case) +// - time.Time → quoted .String() (matches the Thrift marshalScalar special-case) // - nested decimal → exact scale-applied JSON number literal (never a lossy // float64), matching Thrift's marshalScalar → ValueString // - float32 → native float32 (not widened to float64), so JSON renders @@ -30,6 +30,7 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" "github.com/databricks/databricks-sql-go/internal/decimalfmt" + "github.com/databricks/databricks-sql-go/internal/jsonfmt" ) // ScanCell extracts one cell as a driver.Value. Scalars map to their Go value: @@ -581,13 +582,14 @@ func writeStructJSON(b *strings.Builder, s *array.Struct, row int, loc *time.Loc // not already a JSON string (numbers/bools become "7"/"true" keys). func writeJSONKey(b *strings.Builder, v any) { // A time.Time key renders as its quoted .String() (same special-case as - // writeScalarJSON / the Thrift marshal()), not RFC3339. + // writeScalarJSON / the Thrift marshalScalar), not RFC3339. if t, ok := v.(time.Time); ok { kb, _ := json.Marshal(t.String()) b.Write(kb) return } - kb, err := json.Marshal(v) + var scratch [32]byte + kb, err := jsonfmt.Append(scratch[:0], v) if err != nil { // Unmarshalable key value — fall back to its stringified form, quoted. kb, _ = json.Marshal(fmt.Sprintf("%v", v)) @@ -601,8 +603,9 @@ func writeJSONKey(b *strings.Builder, v any) { b.WriteByte('"') } -// writeScalarJSON writes a scalar leaf, mirroring the Thrift marshal(): a -// time.Time becomes a quoted .String(); everything else uses json.Marshal. +// writeScalarJSON writes a scalar leaf, mirroring the Thrift marshalScalar: a +// time.Time becomes a quoted .String(); everything else renders as json.Marshal +// would. func writeScalarJSON(b *strings.Builder, v any) error { if v == nil { b.WriteString("null") @@ -614,7 +617,8 @@ func writeScalarJSON(b *strings.Builder, v any) error { b.WriteByte('"') return nil } - vb, err := json.Marshal(v) + var scratch [32]byte + vb, err := jsonfmt.Append(scratch[:0], v) if err != nil { return err } diff --git a/internal/jsonfmt/jsonfmt.go b/internal/jsonfmt/jsonfmt.go new file mode 100644 index 00000000..af76c229 --- /dev/null +++ b/internal/jsonfmt/jsonfmt.go @@ -0,0 +1,152 @@ +// Package jsonfmt renders scalar values as JSON, byte-identical to +// encoding/json.Marshal, for the nested-type renderers shared by the Thrift +// (arrowbased) and kernel (arrowscan) result paths. +// +// Those renderers marshal one scalar per nested element, so per-call overhead +// dominates. Go 1.27 routes encoding/json through the json/v2 engine by default, +// which made that per-call cost markedly higher; formatting the common scalar +// types directly avoids it. Anything not handled here falls back to json.Marshal, +// so output never diverges from it. +package jsonfmt + +import ( + "encoding/json" + "math" + "strconv" + "unicode/utf8" +) + +// Append appends the JSON encoding of v to dst, exactly as json.Marshal(v) +// would render it, and returns the extended slice. +func Append(dst []byte, v any) ([]byte, error) { + switch t := v.(type) { + case nil: + return append(dst, "null"...), nil + case bool: + return strconv.AppendBool(dst, t), nil + case int: + return strconv.AppendInt(dst, int64(t), 10), nil + case int8: + return strconv.AppendInt(dst, int64(t), 10), nil + case int16: + return strconv.AppendInt(dst, int64(t), 10), nil + case int32: + return strconv.AppendInt(dst, int64(t), 10), nil + case int64: + return strconv.AppendInt(dst, t, 10), nil + case uint8: + return strconv.AppendUint(dst, uint64(t), 10), nil + case uint16: + return strconv.AppendUint(dst, uint64(t), 10), nil + case uint32: + return strconv.AppendUint(dst, uint64(t), 10), nil + case uint64: + return strconv.AppendUint(dst, t, 10), nil + case float32: + if !isFinite(float64(t)) { + break + } + return appendFloat(dst, float64(t), 32), nil + case float64: + if !isFinite(t) { + break + } + return appendFloat(dst, t, 64), nil + case string: + if needsEscape(t) { + break + } + dst = append(dst, '"') + dst = append(dst, t...) + return append(dst, '"'), nil + } + // NaN/Inf (json.Marshal's error), strings needing escapes, []byte (base64), + // and any other type. + b, err := json.Marshal(v) + if err != nil { + return dst, err + } + if len(dst) == 0 { + return b, nil // skip copying json.Marshal's fresh slice + } + return append(dst, b...), nil +} + +// Marshal is Append into a fresh slice, a drop-in for json.Marshal on scalars. +func Marshal(v any) ([]byte, error) { + return Append(nil, v) +} + +func isFinite(f float64) bool { + return !math.IsInf(f, 0) && !math.IsNaN(f) +} + +// appendFloat mirrors encoding/json's ES6-style float formatting: plain decimal +// unless the magnitude is below 1e-6 or at least 1e21, with exponents unpadded. +func appendFloat(dst []byte, f float64, bits int) []byte { + abs := math.Abs(f) + format := byte('f') + // Compare at the value's own width so float32 cutoffs match exactly. + if abs != 0 { + if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) { + format = 'e' + } + } + dst = strconv.AppendFloat(dst, f, format, -1, bits) + if format == 'e' { + // e-09 -> e-9 + n := len(dst) + if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' { + dst[n-2] = dst[n-1] + dst = dst[:n-1] + } + } + return dst +} + +// needsEscape reports whether json.Marshal would escape any part of s. It copies +// ASCII verbatim except control characters, the quote/backslash and the +// HTML-escaped <, >, &; and valid multi-byte UTF-8 verbatim except U+2028/U+2029. +// Everything else (including invalid UTF-8) takes the fallback. +func needsEscape(s string) bool { + for i := 0; i < len(s); i++ { + if !plainASCII[s[i]] { + if s[i] < utf8.RuneSelf { + return true + } + return needsEscapeUTF8(s[i:]) + } + } + return false +} + +// needsEscapeUTF8 is needsEscape for the rest of s from its first non-ASCII byte. +func needsEscapeUTF8(s string) bool { + for i := 0; i < len(s); { + if c := s[i]; c < utf8.RuneSelf { + if !plainASCII[c] { + return true + } + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 || r == 0x2028 || r == 0x2029 { + return true + } + i += size + } + return false +} + +// plainASCII marks the ASCII bytes json.Marshal copies verbatim. A table lookup +// keeps the scan faster than json.Marshal's own on long strings. +var plainASCII = func() (t [256]bool) { + for c := 0x20; c <= 0x7f; c++ { + t[c] = true + } + for _, c := range `"\<>&` { + t[c] = false + } + return t +}() diff --git a/internal/jsonfmt/jsonfmt_test.go b/internal/jsonfmt/jsonfmt_test.go new file mode 100644 index 00000000..6c073bde --- /dev/null +++ b/internal/jsonfmt/jsonfmt_test.go @@ -0,0 +1,130 @@ +package jsonfmt + +import ( + "encoding/json" + "math" + "math/rand" + "strings" + "testing" + "unicode/utf8" +) + +// assertParity fails unless Marshal(v) matches json.Marshal(v) byte-for-byte, +// including whether it errors. +func assertParity(t *testing.T, v any) { + t.Helper() + want, wantErr := json.Marshal(v) + got, gotErr := Marshal(v) + if (wantErr != nil) != (gotErr != nil) || wantErr != nil && gotErr.Error() != wantErr.Error() { + t.Fatalf("%#v: error mismatch: got %v, want %v", v, gotErr, wantErr) + } + if wantErr == nil && string(got) != string(want) { + t.Fatalf("%#v: got %s, want %s", v, got, want) + } +} + +func TestMarshalMatchesEncodingJSON(t *testing.T) { + values := []any{ + nil, true, false, + 0, -1, math.MaxInt, math.MinInt, + int8(math.MinInt8), int8(math.MaxInt8), int16(math.MinInt16), int32(math.MinInt32), int64(math.MinInt64), int64(math.MaxInt64), + uint8(255), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), + 0.0, math.Copysign(0, -1), 1.0, -1.5, 3.14, 0.1, 1e20, 1e21, 1e-6, 1e-7, 9.999999e-7, 123456789.123456789, + math.MaxFloat64, math.SmallestNonzeroFloat64, -math.MaxFloat64, + float32(0), float32(math.Copysign(0, -1)), float32(3.14), float32(0.1), float32(1e21), float32(1e-6), float32(1e-7), + float32(math.MaxFloat32), float32(math.SmallestNonzeroFloat32), + math.NaN(), math.Inf(1), math.Inf(-1), float32(math.NaN()), float32(math.Inf(-1)), + "", "alpha", "with space", "a\"b", `back\slash`, "", "a&b", "tab\there", "nl\n", "\x00", "\x7f", + "héllo", "日本語", "

", "\xff\xfe", "emoji 😀", + []byte("abc"), []byte(nil), + } + for _, v := range values { + assertParity(t, v) + } +} + +// Every single-byte string, so the escape set can't drift from json.Marshal's. +func TestMarshalStringBytes(t *testing.T) { + for c := 0; c < 256; c++ { + assertParity(t, string([]byte{byte(c)})) + assertParity(t, "x"+string([]byte{byte(c)})+"y") + } +} + +// Every rune (surrogates encode as U+FFFD), so the UTF-8 fast path can't drift +// from json.Marshal. +func TestMarshalEveryRune(t *testing.T) { + for r := rune(0); r <= utf8.MaxRune; r++ { + assertParity(t, "x"+string(r)) + } +} + +// Random mixes of ASCII, multi-byte runes, U+2028/U+2029 and raw invalid bytes. +func TestMarshalRandomUnicodeStrings(t *testing.T) { + r := rand.New(rand.NewSource(3)) //nolint:gosec // G404: deterministic test inputs, not security + pieces := []string{"a", "Z", " ", "<", "&", "\"", "\\", "\n", "\x7f"} + for _, cp := range []rune{0xe9, 0xdf, 0x65e5, 0x1f600, 0x2028, 0x2029, 0xa0, 0xfeff, 0xffff} { + pieces = append(pieces, string(cp)) + } + for i := 0; i < 100000; i++ { + var sb strings.Builder + for j, n := 0, r.Intn(12); j < n; j++ { + switch r.Intn(4) { + case 0: + var raw [1]byte // possibly invalid UTF-8 + _, _ = r.Read(raw[:]) + sb.Write(raw[:]) + case 1: + sb.WriteRune(r.Int31n(utf8.MaxRune + 1)) + default: + sb.WriteString(pieces[r.Intn(len(pieces))]) + } + } + assertParity(t, sb.String()) + } +} + +// Random bit patterns reach exponent cutoffs, subnormals, and rounding cases a +// hand-picked list misses. +func TestMarshalRandomFloats(t *testing.T) { + r := rand.New(rand.NewSource(1)) //nolint:gosec // G404: deterministic test inputs, not security + for i := 0; i < 200000; i++ { + assertParity(t, math.Float64frombits(r.Uint64())) + assertParity(t, math.Float32frombits(r.Uint32())) + assertParity(t, r.NormFloat64()*math.Pow(10, float64(r.Intn(50)-25))) + assertParity(t, float32(r.NormFloat64()*math.Pow(10, float64(r.Intn(30)-15)))) + } +} + +func TestAppendKeepsPrefix(t *testing.T) { + got, err := Append([]byte("["), int32(7)) + if err != nil || string(got) != "[7" { + t.Fatalf("got %q, %v", got, err) + } + got, err = Append([]byte("["), []byte("abc")) + if err != nil || string(got) != `["YWJj"` { + t.Fatalf("got %q, %v", got, err) + } +} + +func BenchmarkMarshalScalars(b *testing.B) { + // The last three take json.Marshal's fallback. + values := []any{int32(42), int64(-9000), 3.14, 0.30000000000000004, float32(0.5), true, "alpha0", "café crème", + `a"b`, []byte("abc"), math.NaN()} + b.Run("jsonfmt", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, v := range values { + _, _ = Marshal(v) + } + } + }) + b.Run("encoding_json", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, v := range values { + _, _ = json.Marshal(v) + } + } + }) +} diff --git a/internal/rows/arrowbased/arrowscan_parity_test.go b/internal/rows/arrowbased/arrowscan_parity_test.go index 495fde95..f60eaefe 100644 --- a/internal/rows/arrowbased/arrowscan_parity_test.go +++ b/internal/rows/arrowbased/arrowscan_parity_test.go @@ -120,6 +120,33 @@ func TestArrowbasedKernelRenderParity(t *testing.T) { b.FieldBuilder(0).(*array.Float32Builder).Append(0.1) return b.NewArray() }}, + {"list_double", func() arrow.Array { + b := array.NewListBuilder(pool, arrow.PrimitiveTypes.Float64) + b.Append(true) + b.ValueBuilder().(*array.Float64Builder).AppendValues([]float64{0.1, -2.5, 1e-7, 1e21, 123456789.125, 0.30000000000000004}, nil) + return b.NewArray() + }}, + // String leaves and map keys with UTF-8, HTML-escaped characters, U+2028 + // and invalid UTF-8. + {"list_string_utf8_html", func() arrow.Array { + b := array.NewListBuilder(pool, arrow.BinaryTypes.String) + b.Append(true) + b.ValueBuilder().(*array.StringBuilder).AppendValues([]string{ + "café", "日本語", "😀", "&", "a" + string(rune(0x2028)) + "b", "bad\xffutf8", "del\x7f", + }, nil) + return b.NewArray() + }}, + {"map_utf8_key", func() arrow.Array { + b := array.NewMapBuilder(pool, arrow.BinaryTypes.String, arrow.BinaryTypes.String, false) + b.Append(true) + kb := b.KeyBuilder().(*array.StringBuilder) + ib := b.ItemBuilder().(*array.StringBuilder) + kb.Append("clé") + ib.Append("naïve") + kb.Append("a