perf: reduce node snapshot and hash traversal overhead

datamodel/low/node_lines_snapshot_test.go

@@ -0,0 +1,135 @@
+package low
+
+import (
+	"fmt"
+	"slices"
+	"sync"
+	"testing"
+
+	"github.com/pb33f/go-yaml"
+	"github.com/pb33f/testify/assert"
+	"github.com/pb33f/testify/require"
+)
+
+func TestNodeMapGetNodeLines(t *testing.T) {
+	cases := []struct {
+		name   string
+		values map[int]any
+	}{
+		{name: "nil nodes"},
+		{name: "empty", values: map[int]any{}},
+		{name: "single", values: map[int]any{4: &yaml.Node{}}},
+		{
+			name: "mixed",
+			values: map[int]any{
+				-2: &yaml.Node{},
+				0:  []*yaml.Node{{}, {}},
+				7:  (*yaml.Node)(nil),
+				9:  []*yaml.Node(nil),
+				11: []*yaml.Node{},
+				13: "ignored",
+				14: nil,
+			},
+		},
+		{name: "invalid only", values: map[int]any{5: "invalid", 9: 42}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			nm := &NodeMap{}
+			if tc.values != nil {
+				nm.Nodes = &NodeLines{}
+				for k, v := range tc.values {
+					nm.Nodes.Store(k, v)
+				}
+			}
+			got := nm.GetNodeLines()
+			want := []int{}
+			for k := range nm.GetNodes() {
+				want = append(want, k)
+			}
+			slices.Sort(got)
+			slices.Sort(want)
+			assert.Equal(t, want, got)
+			require.NotEmpty(t, got)
+			got[0] = 999
+			fresh := nm.GetNodeLines()
+			slices.Sort(fresh)
+			assert.Equal(t, want, fresh, "snapshot must not alias storage")
+		})
+	}
+}
+
+func TestNodeMapGetNodeLinesMutation(t *testing.T) {
+	nm := &NodeMap{Nodes: &NodeLines{}}
+	nm.Nodes.Store(2, "discard")
+	nm.AddNode(3, &yaml.Node{})
+	nm.Nodes.Store(2, &yaml.Node{})
+	nm.AddNode(3, &yaml.Node{})
+	before := nm.GetNodeLines()
+	slices.Sort(before)
+	nm.Nodes.Store(2, "now invalid")
+	nm.Nodes.Store(7, (*yaml.Node)(nil))
+	nm.AddNode(9, &yaml.Node{})
+	after := nm.GetNodeLines()
+	slices.Sort(after)
+	assert.Equal(t, []int{2, 3}, before)
+	assert.Equal(t, []int{3, 7, 9}, after)
+}
+
+func TestNodeMapGetNodeLinesConcurrent(t *testing.T) {
+	nm := &NodeMap{Nodes: &NodeLines{}}
+	var wg sync.WaitGroup
+	for worker := 0; worker < 4; worker++ {
+		wg.Add(1)
+		go func(worker int) {
+			defer wg.Done()
+			for i := 0; i < 1000; i++ {
+				nm.Nodes.Store(worker*1000+i, &yaml.Node{})
+				lines := nm.GetNodeLines()
+				seen := make(map[int]bool, len(lines))
+				for _, line := range lines {
+					assert.False(t, seen[line], "duplicate line %d", line)
+					seen[line] = true
+				}
+			}
+		}(worker)
+	}
+	wg.Wait()
+	assert.Len(t, nm.GetNodeLines(), 4000, "missing writes")
+}
+
+func BenchmarkNodeMapLineNumbers(b *testing.B) {
+	for _, size := range []int{1, 32, 1024} {
+		b.Run(fmt.Sprint(size), func(b *testing.B) {
+			nm := &NodeMap{Nodes: &NodeLines{}}
+			for i := 0; i < size; i++ {
+				nm.Nodes.Store(i, &yaml.Node{})
+			}
+			nm.GetNodes()
+			b.Run("map", func(b *testing.B) {
+				b.ReportAllocs()
+				for b.Loop() {
+					n := 0
+					for range nm.GetNodes() {
+						n++
+					}
+					if n != size {
+						b.Fatal(n)
+					}
+				}
+			})
+			b.Run("lines", func(b *testing.B) {
+				b.ReportAllocs()
+				for b.Loop() {
+					n := 0
+					for range nm.GetNodeLines() {
+						n++
+					}
+					if n != size {
+						b.Fatal(n)
+					}
+				}
+			})
+		})
+	}
+}

datamodel/low/node_map.go

@@ -158,6 +158,30 @@ func (nm *NodeMap) GetNodes() map[int][]*yaml.Node {
 	return composed
 }
 
+// GetNodeLines returns an owned snapshot of the line numbers in GetNodes, in
+// unspecified order. Only *yaml.Node and []*yaml.Node values contribute lines,
+// including typed nil values. If no such values exist, the result is []int{0}.
+// Later writes to Nodes do not change the returned slice.
+func (nm *NodeMap) GetNodeLines() []int {
+	if nm.Nodes == nil {
+		return []int{0}
+	}
+	nm.Nodes.mu.Lock()
+	defer nm.Nodes.mu.Unlock()
+	index := nm.Nodes.index()
+	lines := make([]int, 0, len(index))
+	for line, value := range index {
+		switch value.(type) {
+		case *yaml.Node, []*yaml.Node:
+			lines = append(lines, line)
+		}
+	}
+	if len(lines) == 0 {
+		lines = append(lines, 0)
+	}
+	return lines
+}
+
 // ExtractNodes will iterate over a *yaml.Node and extract all nodes with a line number into a map
 func (nm *NodeMap) ExtractNodes(node *yaml.Node, recurse bool) {
 	if node == nil {

index/origin_identity_test.go

@@ -0,0 +1,180 @@
+package index
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pb33f/go-yaml"
+	"github.com/pb33f/testify/assert"
+	"github.com/pb33f/testify/require"
+)
+
+func originIdentityIndex(path string, nodes ...*yaml.Node) *SpecIndex {
+	idx := &SpecIndex{specAbsolutePath: path}
+	for _, n := range nodes {
+		for len(idx.nodeLines) <= n.Line {
+			idx.nodeLines = append(idx.nodeLines, nil)
+		}
+		idx.nodeLines[n.Line] = append(idx.nodeLines[n.Line], nodeLineEntry{column: int32(n.Column), node: n})
+	}
+	sortNodeLines(idx.nodeLines)
+	return idx
+}
+
+func originIdentityValue(line int, children int) *yaml.Node {
+	n := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Line: line, Column: 2}
+	for i := 0; i < children; i++ {
+		n.Content = append(n.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: fmt.Sprint(i), Line: line + i + 1, Column: 4})
+	}
+	return n
+}
+
+func TestOriginIdentityEquivalent(t *testing.T) {
+	key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "root", Line: 1, Column: 1}
+	value := originIdentityValue(2, 2)
+	extKey := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "external", Line: 20, Column: 1}
+	extValue := originIdentityValue(30, 2)
+	rootIndex := originIdentityIndex("root.yaml", key, value)
+	keyIndex := originIdentityIndex("key.yaml", extKey)
+	valueIndex := originIdentityIndex("value.yaml", extValue)
+	r := &Rolodex{rootIndex: rootIndex, indexes: []*SpecIndex{keyIndex, valueIndex}}
+	rootOrigin := &NodeOrigin{Node: key, Line: 1, Column: 1, AbsoluteLocation: "root.yaml", Index: rootIndex}
+	externalOrigin := &NodeOrigin{Node: extKey, Line: 20, Column: 1, AbsoluteLocation: "key.yaml", Index: keyIndex}
+	rootWithValue := *rootOrigin
+	rootWithValue.AbsoluteLocationValue = "value.yaml"
+	rootWithValue.LineValue, rootWithValue.ColumnValue, rootWithValue.ValueNode = 30, 2, extValue
+	externalWithValue := *externalOrigin
+	externalWithValue.AbsoluteLocationValue = "value.yaml"
+	externalWithValue.LineValue, externalWithValue.ColumnValue, externalWithValue.ValueNode = 30, 2, extValue
+	clone := *value
+	mismatch := *value
+	mismatch.Value = "different"
+	missing := &yaml.Node{Kind: yaml.ScalarNode, Value: "missing", Line: 99, Column: 99}
+	refs := []struct {
+		name  string
+		node  *yaml.Node
+		value string
+	}{
+		{name: "none"},
+		{name: "node only", node: key},
+		{name: "value only", value: "#/schemas/Item"},
+		{name: "both", node: key, value: "#/schemas/Item"},
+	}
+	cases := []struct {
+		name       string
+		key, value *yaml.Node
+		want       [4]*NodeOrigin
+		panics     [4]bool
+	}{
+		{
+			name:  "root pointer",
+			key:   key,
+			value: value,
+			want:  [4]*NodeOrigin{rootOrigin, rootOrigin, rootOrigin, rootOrigin},
+		},
+		{
+			name:  "root clone",
+			key:   key,
+			value: &clone,
+			want:  [4]*NodeOrigin{rootOrigin, rootOrigin, rootOrigin, rootOrigin},
+		},
+		{
+			name:  "different content",
+			key:   key,
+			value: &mismatch,
+			want:  [4]*NodeOrigin{nil, nil, nil, rootOrigin},
+		},
+		{
+			name:  "external pointer",
+			key:   extKey,
+			value: extValue,
+			want:  [4]*NodeOrigin{&externalWithValue, externalOrigin, externalOrigin, externalOrigin},
+		},
+		{
+			name:  "root to external",
+			key:   key,
+			value: extValue,
+			want:  [4]*NodeOrigin{&rootWithValue, rootOrigin, rootOrigin, rootOrigin},
+		},
+		{
+			name:  "nil key",
+			value: value,
+		},
+		{
+			name: "nil value",
+			key:  key,
+			want: [4]*NodeOrigin{rootOrigin, rootOrigin, rootOrigin, rootOrigin},
+		},
+		{
+			name:  "missing value",
+			key:   key,
+			value: missing,
+			want:  [4]*NodeOrigin{nil, nil, nil, rootOrigin},
+		},
+		{
+			name:   "missing key",
+			key:    missing,
+			value:  extValue,
+			panics: [4]bool{true, true, true, false},
+		},
+	}
+	for _, tc := range cases {
+		for i, ref := range refs {
+			t.Run(tc.name+"/"+ref.name, func(t *testing.T) {
+				ClearHashCache()
+				t.Cleanup(ClearHashCache)
+				call := func() { assert.Equal(t, tc.want[i], r.FindNodeOriginWithValue(tc.key, tc.value, ref.node, ref.value)) }
+				if tc.panics[i] {
+					assert.Panics(t, call)
+					return
+				}
+				assert.NotPanics(t, call)
+			})
+		}
+	}
+}
+
+func TestOriginIdentityPopulatesHashCache(t *testing.T) {
+	for _, external := range []bool{false, true} {
+		t.Run(fmt.Sprintf("external=%t", external), func(t *testing.T) {
+			key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key", Line: 1, Column: 1}
+			value := originIdentityValue(2, hashCacheThreshold)
+			r := &Rolodex{rootIndex: originIdentityIndex("root.yaml", key, value)}
+			if external {
+				r.rootIndex = originIdentityIndex("root.yaml")
+				r.indexes = []*SpecIndex{originIdentityIndex("key.yaml", key), originIdentityIndex("value.yaml", value)}
+			}
+			ClearHashCache()
+			t.Cleanup(ClearHashCache)
+			require.NotNil(t, r.FindNodeOriginWithValue(key, value, nil, ""))
+			cached, ok := nodeHashCache.Load(value)
+			require.True(t, ok, "origin lookup should warm cache-eligible values")
+			value.Content[0].Value = "changed"
+			ClearHashCache()
+			assert.NotEqual(t, cached, HashNode(value), "invalidation must expose mutation")
+		})
+	}
+}
+
+func BenchmarkOriginIdentity(b *testing.B) {
+	for _, external := range []bool{false, true} {
+		for _, children := range []int{0, 2, hashCacheThreshold} {
+			b.Run(fmt.Sprintf("external=%t/children=%d", external, children), func(b *testing.B) {
+				key := &yaml.Node{Value: "key", Line: 1, Column: 1}
+				value := originIdentityValue(2, children)
+				r := &Rolodex{rootIndex: originIdentityIndex("root.yaml", key, value)}
+				if external {
+					r.rootIndex = originIdentityIndex("root.yaml")
+					r.indexes = []*SpecIndex{originIdentityIndex("external.yaml", key, value)}
+				}
+				b.ReportAllocs()
+				b.ResetTimer()
+				for b.Loop() {
+					if r.FindNodeOriginWithValue(key, value, nil, "") == nil {
+						b.Fatal("missing origin")
+					}
+				}
+			})
+		}
+	}
+}

index/search_rolodex.go

@@ -21,6 +21,13 @@ func (r *Rolodex) FindNodeOriginWithValue(key, value, refNode *yaml.Node, refVal
 	if value != nil {
 		if keyOrigin != nil && keyOrigin.AbsoluteLocation == r.GetRootIndex().specAbsolutePath {
 			valueOrigin = r.GetRootIndex().FindNodeOrigin(value)
+			if valueOrigin != nil && valueOrigin.Node == value {
+				// only cache-eligible nodes benefit from warming an identity match.
+				if len(value.Content) >= hashCacheThreshold {
+					HashNode(value)
+				}
+				return keyOrigin
+			}
 			valueHash = HashNode(value)
 			if refNode != nil && refValue != "" {
 				return keyOrigin
@@ -52,10 +59,10 @@ func (r *Rolodex) FindNodeOriginWithValue(key, value, refNode *yaml.Node, refVal
 						return keyOrigin
 					}
 
-					valueHash = HashNode(value)
-					nHash := HashNode(n.Node)
-
-					if valueHash == nHash {
+					if n.Node != value || len(value.Content) >= hashCacheThreshold {
+						valueHash = HashNode(value)
+					}
+					if n.Node == value || valueHash == HashNode(n.Node) {
 						if keyOrigin.AbsoluteLocation != n.AbsoluteLocation {
 							if refNode == nil && refValue == "" {
 								keyOrigin.AbsoluteLocationValue = n.AbsoluteLocation