From 7dcc984a9cadafbde7f708c83f093ec9c55e7438 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 30 Oct 2026 20:41:41 +0000 Subject: [PATCH 1/3] Reduce allocations in nested Arrow JSON rendering Signed-off-by: Vu Anh Phung --- CHANGELOG.md | 1 + internal/arrowscan/arrowscan.go | 22 ++++++ internal/arrowscan/json_scalars_test.go | 101 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 internal/arrowscan/json_scalars_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a4ccd0..1db080fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Reduce kernel-backend JSON rendering allocations for nested Boolean and signed-integer values. - **Behavior change:** Native Arrow TIMESTAMP_NTZ results now preserve their wall-clock fields and report `TIMESTAMP_NTZ` on both backends. Returned `time.Time` values use UTC instead of the session location; their represented instant and `.UTC()` value are unchanged (databricks/databricks-sql-go#506). On Thrift, this requires a server-supplied Arrow schema with native timestamps; without it, NTZ still reports `TIMESTAMP` and uses session-location handling. String timestamps (`UseArrowNativeTimestamp=false`, internal) and non-Arrow values also retain legacy parsing. ## v1.18.0 (2026-10-24) diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index 8ff76aba..cbec7a1b 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -23,6 +23,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -433,6 +434,21 @@ func writeJSON(b *strings.Builder, col arrow.Array, row int, loc *time.Location, return nil } switch c := col.(type) { + case *array.Boolean: + if c.Value(row) { + b.WriteString("true") + } else { + b.WriteString("false") + } + return nil + case *array.Int8: + return writeIntJSON(b, int64(c.Value(row))) + case *array.Int16: + return writeIntJSON(b, int64(c.Value(row))) + case *array.Int32: + return writeIntJSON(b, int64(c.Value(row))) + case *array.Int64: + return writeIntJSON(b, c.Value(row)) case *array.List: // Use the offset-aware ValueOffsets, NOT Offsets()[row]: arrow-go's // Offsets() returns the full un-sliced buffer, so a List with a non-zero @@ -479,6 +495,12 @@ func writeJSON(b *strings.Builder, col arrow.Array, row int, loc *time.Location, } } +func writeIntJSON(b *strings.Builder, v int64) error { + var scratch [20]byte // Sign and decimal digits of math.MinInt64 fit. + b.Write(strconv.AppendInt(scratch[:0], v, 10)) + return nil +} + func writeListJSON(b *strings.Builder, values arrow.Array, start, end int, loc *time.Location, keys *StructKeyCache) error { b.WriteByte('[') for i := start; i < end; i++ { diff --git a/internal/arrowscan/json_scalars_test.go b/internal/arrowscan/json_scalars_test.go new file mode 100644 index 00000000..2a5bc53e --- /dev/null +++ b/internal/arrowscan/json_scalars_test.go @@ -0,0 +1,101 @@ +package arrowscan + +import ( + "strings" + "testing" + + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/memory" +) + +type nestedScalarJSONFixture struct { + name string + dtype arrow.DataType + input string + want string +} + +func nestedScalarJSONFixtures() []nestedScalarJSONFixture { + return []nestedScalarJSONFixture{ + {"boolean", arrow.ListOf(arrow.FixedWidthTypes.Boolean), `[true,false,null]`, `[true,false,null]`}, + {"int8", arrow.ListOf(arrow.PrimitiveTypes.Int8), `[-128,-1,0,1,127,null]`, `[-128,-1,0,1,127,null]`}, + {"int16", arrow.ListOf(arrow.PrimitiveTypes.Int16), `[-32768,-1,0,1,32767,null]`, `[-32768,-1,0,1,32767,null]`}, + {"int32", arrow.ListOf(arrow.PrimitiveTypes.Int32), `[-2147483648,-1,0,1,2147483647,null]`, `[-2147483648,-1,0,1,2147483647,null]`}, + {"int64", arrow.ListOf(arrow.PrimitiveTypes.Int64), `[-9223372036854775808,-1,0,1,9223372036854775807,null]`, `[-9223372036854775808,-1,0,1,9223372036854775807,null]`}, + { + "map_boolean", arrow.MapOf(arrow.BinaryTypes.String, arrow.FixedWidthTypes.Boolean), + `[{"key":"t","value":true},{"key":"f","value":false},{"key":"n","value":null}]`, + `{"t":true,"f":false,"n":null}`, + }, + { + "map_int64", arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64), + `[{"key":"quote\"&<>","value":-9223372036854775808},{"key":"max","value":9223372036854775807},{"key":"n","value":null}]`, + `{"quote\"\u0026\u003c\u003e":-9223372036854775808,"max":9223372036854775807,"n":null}`, + }, + { + "struct", arrow.StructOf( + arrow.Field{Name: "flag", Type: arrow.FixedWidthTypes.Boolean}, + arrow.Field{Name: "tiny", Type: arrow.PrimitiveTypes.Int8}, + arrow.Field{Name: "small", Type: arrow.PrimitiveTypes.Int16}, + arrow.Field{Name: "int", Type: arrow.PrimitiveTypes.Int32}, + arrow.Field{Name: "big", Type: arrow.PrimitiveTypes.Int64}, + arrow.Field{Name: "missing", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, + arrow.Field{Name: "s", Type: arrow.BinaryTypes.String}, + ), + `{"flag":false,"tiny":-128,"small":-32768,"int":-2147483648,"big":-9223372036854775808,"missing":null,"s":"quote\"&<>"}`, + `{"flag":false,"tiny":-128,"small":-32768,"int":-2147483648,"big":-9223372036854775808,"missing":null,"s":"quote\"\u0026\u003c\u003e"}`, + }, + } +} + +func TestScanCellNestedScalarJSON(t *testing.T) { + for _, tc := range nestedScalarJSONFixtures() { + t.Run(tc.name, func(t *testing.T) { + input := "[" + tc.input + ",null," + tc.input + "]" + full, _, err := array.FromJSON(memory.DefaultAllocator, tc.dtype, strings.NewReader(input), array.WithUseNumber()) + if err != nil { + t.Fatal(err) + } + defer full.Release() + sliced := array.NewSlice(full, 1, 3) + defer sliced.Release() + for _, cache := range []*StructKeyCache{nil, NewStructKeyCache()} { + for _, col := range []arrow.Array{full, sliced} { + for row := 0; row < col.Len(); row++ { + var want any = tc.want + if row+col.Data().Offset() == 1 { + want = nil + } + got, err := ScanCellCached(col, row, nil, cache) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("offset %d row %d: got %v, want %v", col.Data().Offset(), row, got, want) + } + } + } + } + }) + } +} + +func BenchmarkScanCellNestedScalarJSON(b *testing.B) { + for _, tc := range nestedScalarJSONFixtures() { + b.Run(tc.name, func(b *testing.B) { + col, _, err := array.FromJSON(memory.DefaultAllocator, tc.dtype, strings.NewReader("["+tc.input+"]"), array.WithUseNumber()) + if err != nil { + b.Fatal(err) + } + defer col.Release() + cache := NewStructKeyCache() + b.ReportAllocs() + for b.Loop() { + if _, err := ScanCellCached(col, 0, nil, cache); err != nil { + b.Fatal(err) + } + } + }) + } +} From c7bc016f5fc90485235e37ffc4e3976a69e093cf Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 30 Oct 2026 22:10:49 +0000 Subject: [PATCH 2/3] Test nested scalar rendering parity across backends Signed-off-by: Vu Anh Phung --- .../rows/arrowbased/arrowscan_parity_test.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal/rows/arrowbased/arrowscan_parity_test.go b/internal/rows/arrowbased/arrowscan_parity_test.go index aeaaab30..495fde95 100644 --- a/internal/rows/arrowbased/arrowscan_parity_test.go +++ b/internal/rows/arrowbased/arrowscan_parity_test.go @@ -1,6 +1,7 @@ package arrowbased import ( + "strings" "testing" "time" @@ -253,6 +254,30 @@ func TestArrowbasedKernelRenderParity(t *testing.T) { name string build func() arrow.Array }{ + {"sliced_struct_scalar_leaves", func() arrow.Array { + dt := arrow.StructOf( + arrow.Field{Name: "flag", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}, + arrow.Field{Name: "tiny", Type: arrow.PrimitiveTypes.Int8, Nullable: true}, + arrow.Field{Name: "small", Type: arrow.PrimitiveTypes.Int16, Nullable: true}, + arrow.Field{Name: "int", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + arrow.Field{Name: "big", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, + ) + input := `[ + {"flag":true,"tiny":1,"small":2,"int":3,"big":4}, + {"flag":true,"tiny":-128,"small":-32768,"int":-2147483648,"big":-9223372036854775808}, + {"flag":false,"tiny":127,"small":32767,"int":2147483647,"big":9223372036854775807}, + {"flag":true,"tiny":-1,"small":-1,"int":-1,"big":-1}, + {"flag":false,"tiny":0,"small":0,"int":0,"big":0}, + {"flag":null,"tiny":null,"small":null,"int":null,"big":null}, + null + ]` + full, _, err := array.FromJSON(pool, dt, strings.NewReader(input), array.WithUseNumber()) + if err != nil { + t.Fatal(err) + } + defer full.Release() + return array.NewSlice(full, 1, int64(full.Len())) + }}, {"sliced_list", func() arrow.Array { b := array.NewListBuilder(pool, arrow.PrimitiveTypes.Int64) vb := b.ValueBuilder().(*array.Int64Builder) From 32970eb2cefcbc47de258d04b2ea63be95b831e6 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 30 Oct 2026 22:14:00 +0000 Subject: [PATCH 3/3] Drop redundant nested scalar JSON test file Signed-off-by: Vu Anh Phung --- internal/arrowscan/json_scalars_test.go | 101 ------------------------ 1 file changed, 101 deletions(-) delete mode 100644 internal/arrowscan/json_scalars_test.go diff --git a/internal/arrowscan/json_scalars_test.go b/internal/arrowscan/json_scalars_test.go deleted file mode 100644 index 2a5bc53e..00000000 --- a/internal/arrowscan/json_scalars_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package arrowscan - -import ( - "strings" - "testing" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" -) - -type nestedScalarJSONFixture struct { - name string - dtype arrow.DataType - input string - want string -} - -func nestedScalarJSONFixtures() []nestedScalarJSONFixture { - return []nestedScalarJSONFixture{ - {"boolean", arrow.ListOf(arrow.FixedWidthTypes.Boolean), `[true,false,null]`, `[true,false,null]`}, - {"int8", arrow.ListOf(arrow.PrimitiveTypes.Int8), `[-128,-1,0,1,127,null]`, `[-128,-1,0,1,127,null]`}, - {"int16", arrow.ListOf(arrow.PrimitiveTypes.Int16), `[-32768,-1,0,1,32767,null]`, `[-32768,-1,0,1,32767,null]`}, - {"int32", arrow.ListOf(arrow.PrimitiveTypes.Int32), `[-2147483648,-1,0,1,2147483647,null]`, `[-2147483648,-1,0,1,2147483647,null]`}, - {"int64", arrow.ListOf(arrow.PrimitiveTypes.Int64), `[-9223372036854775808,-1,0,1,9223372036854775807,null]`, `[-9223372036854775808,-1,0,1,9223372036854775807,null]`}, - { - "map_boolean", arrow.MapOf(arrow.BinaryTypes.String, arrow.FixedWidthTypes.Boolean), - `[{"key":"t","value":true},{"key":"f","value":false},{"key":"n","value":null}]`, - `{"t":true,"f":false,"n":null}`, - }, - { - "map_int64", arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64), - `[{"key":"quote\"&<>","value":-9223372036854775808},{"key":"max","value":9223372036854775807},{"key":"n","value":null}]`, - `{"quote\"\u0026\u003c\u003e":-9223372036854775808,"max":9223372036854775807,"n":null}`, - }, - { - "struct", arrow.StructOf( - arrow.Field{Name: "flag", Type: arrow.FixedWidthTypes.Boolean}, - arrow.Field{Name: "tiny", Type: arrow.PrimitiveTypes.Int8}, - arrow.Field{Name: "small", Type: arrow.PrimitiveTypes.Int16}, - arrow.Field{Name: "int", Type: arrow.PrimitiveTypes.Int32}, - arrow.Field{Name: "big", Type: arrow.PrimitiveTypes.Int64}, - arrow.Field{Name: "missing", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, - arrow.Field{Name: "s", Type: arrow.BinaryTypes.String}, - ), - `{"flag":false,"tiny":-128,"small":-32768,"int":-2147483648,"big":-9223372036854775808,"missing":null,"s":"quote\"&<>"}`, - `{"flag":false,"tiny":-128,"small":-32768,"int":-2147483648,"big":-9223372036854775808,"missing":null,"s":"quote\"\u0026\u003c\u003e"}`, - }, - } -} - -func TestScanCellNestedScalarJSON(t *testing.T) { - for _, tc := range nestedScalarJSONFixtures() { - t.Run(tc.name, func(t *testing.T) { - input := "[" + tc.input + ",null," + tc.input + "]" - full, _, err := array.FromJSON(memory.DefaultAllocator, tc.dtype, strings.NewReader(input), array.WithUseNumber()) - if err != nil { - t.Fatal(err) - } - defer full.Release() - sliced := array.NewSlice(full, 1, 3) - defer sliced.Release() - for _, cache := range []*StructKeyCache{nil, NewStructKeyCache()} { - for _, col := range []arrow.Array{full, sliced} { - for row := 0; row < col.Len(); row++ { - var want any = tc.want - if row+col.Data().Offset() == 1 { - want = nil - } - got, err := ScanCellCached(col, row, nil, cache) - if err != nil { - t.Fatal(err) - } - if got != want { - t.Errorf("offset %d row %d: got %v, want %v", col.Data().Offset(), row, got, want) - } - } - } - } - }) - } -} - -func BenchmarkScanCellNestedScalarJSON(b *testing.B) { - for _, tc := range nestedScalarJSONFixtures() { - b.Run(tc.name, func(b *testing.B) { - col, _, err := array.FromJSON(memory.DefaultAllocator, tc.dtype, strings.NewReader("["+tc.input+"]"), array.WithUseNumber()) - if err != nil { - b.Fatal(err) - } - defer col.Release() - cache := NewStructKeyCache() - b.ReportAllocs() - for b.Loop() { - if _, err := ScanCellCached(col, 0, nil, cache); err != nil { - b.Fatal(err) - } - } - }) - } -}