fix(doc): build relationship edge tables once per model, uncharged to the visit budget

changes/unreleased/relationship-tables-uncharged.fixed.md

@@ -0,0 +1 @@
+- **Relationship queries no longer spend their visit budget building edge tables.** `RelatedElements`, `WhereRelated` and relationship-derived columns build the edge table of a relationship kind by scanning every declaration in the workspace, and each declaration scanned was charged to the query's visit budget — so on a model past ~100,000 declarations every relationship query failed with `visit-budget` before traversing anything (a migrated TMT requirements-mapping document, 101,014 declarations, needed about 200 visits for its rows). The scan is a fixed cost of the model, not of the query: it is now memoized per model in `queryexec.Context.Related`, shared by every query a document (or a linked set of documents) evaluates, and left uncharged; the budget still bounds the traversal itself, paying one visit per element reached, and `visit-budget` is still the typed failure when that is exceeded.

docs/manual/query-cookbook.md

@@ -783,7 +783,10 @@ Direction is from the relationship's own point of view — `outgoing` follows
 it as declared, `incoming` follows it backwards. Traversal is breadth-first
 to `maxDepth` (unbounded when omitted or `null`), deduplicated, in
 declaration order, and bounded by a visit budget so a pathological model
-terminates with a typed error rather than hanging.
+terminates with a typed error rather than hanging. The budget pays only for
+the elements reached: the edge table a relationship kind reads is built once
+per model (again after an edit), from every declaration in the workspace, and is not charged to it —
+so a matrix over a large model costs what its rows relate to, not the model's size.
 
 ### Connections
 

internal/doc/docir/evaluate.go

@@ -59,6 +59,7 @@ func EvaluateSet(
 		}
 		collectCrossAnchors(plan.Content(), external)
 	}
+	context = sharingRelationshipTables(context)
 	documents := make([]*Document, 0, len(plans))
 	for _, plan := range plans {
 		document, err := evaluate(plan, context, options, text, external[plan.Name()])
@@ -83,6 +84,7 @@ func evaluate(
 	if context.Index == nil || context.Resolver == nil || context.Model == nil {
 		return nil, &Error{Kind: ErrorInvalidContext, Document: plan.Name()}
 	}
+	context = sharingRelationshipTables(context)
 	referenced := referencedAnchors(plan.Content())
 	for anchor := range external {
 		referenced[anchor] = true
@@ -106,6 +108,16 @@ func evaluate(
 	}, nil
 }
 
+// sharingRelationshipTables gives a context without relationship tables its
+// own, so every query of the documents evaluated under it builds each kind's
+// edges once.
+func sharingRelationshipTables(context queryexec.Context) queryexec.Context {
+	if context.Related == nil {
+		context.Related = queryexec.NewRelationshipTables()
+	}
+	return context
+}
+
 // collectCrossAnchors records, per target document, the anchors that other
 // documents' reference runs require it to emit.
 func collectCrossAnchors(planned []docplan.Content, external map[string]map[string]bool) {

internal/doc/queryexec/execute.go

@@ -27,6 +27,9 @@ type Context struct {
 	// Roots are the objects the session holds directly, each under its label,
 	// in the order Objects enumerates them.
 	Roots []Root
+	// Related memoizes relationship edge tables across the executions sharing
+	// this context; nil builds them once per execution.
+	Related *RelationshipTables
 }
 
 // Root is one object a session holds directly, under its label (`Demo::car`, `#7`).
@@ -80,7 +83,7 @@ type executor struct {
 	program    map[string]queryplan.Definition
 	budget     *visitBudget
 	calls      *visitBudget
-	related    *relationshipTables
+	related    *RelationshipTables
 	derived    *derivedValues
 	depthLeft  int
 	stack      []string
@@ -118,6 +121,10 @@ func Execute(program *queryplan.Program, context Context, bindings Bindings, opt
 	for _, compiledDefinition := range definitions {
 		compiled[compiledDefinition.Name()] = compiledDefinition
 	}
+	related := context.Related
+	if related == nil {
+		related = NewRelationshipTables()
+	}
 	execution := &executor{
 		definition: definition,
 		context:    context,
@@ -126,7 +133,7 @@ func Execute(program *queryplan.Program, context Context, bindings Bindings, opt
 		program:    compiled,
 		budget:     &visitBudget{remaining: budget},
 		calls:      &visitBudget{remaining: calls},
-		related:    newRelationshipTables(),
+		related:    related,
 		derived:    &derivedValues{},
 		depthLeft:  depth,
 		stack:      []string{definition.Name()},

internal/doc/queryexec/related.go

@@ -47,14 +47,30 @@ type relationshipEdges struct {
 	incoming map[symbols.ElementKey][]*symbols.Symbol
 }
 
-// relationshipTables caches the per-kind edge tables of one execution, shared
-// across invoked queries the way the visit budget is.
-type relationshipTables struct {
-	entries map[string]*relationshipEdges
+// RelationshipTables memoizes one model's per-kind edge tables across the
+// executions whose Context shares them. Not safe for concurrent use.
+type RelationshipTables struct {
+	index      *symbols.Index
+	generation uint64
+	model      *semantics.Model
+	entries    map[string]*relationshipEdges
 }
 
-func newRelationshipTables() *relationshipTables {
-	return &relationshipTables{entries: make(map[string]*relationshipEdges)}
+// NewRelationshipTables returns empty tables for a Context to carry.
+func NewRelationshipTables() *RelationshipTables {
+	return &RelationshipTables{entries: make(map[string]*relationshipEdges)}
+}
+
+// lookup returns the cached tables of one kind, first discarding every entry
+// built against another index or model, or the index before an edit.
+func (t *RelationshipTables) lookup(kind string, context Context) (*relationshipEdges, bool) {
+	generation := context.Index.Generation()
+	if t.index != context.Index || t.generation != generation || t.model != context.Model {
+		t.index, t.generation, t.model = context.Index, generation, context.Model
+		t.entries = make(map[string]*relationshipEdges)
+	}
+	edges, ok := t.entries[kind]
+	return edges, ok
 }
 
 // relationshipWalk is the validated relationshipKind, direction and maxDepth
@@ -143,11 +159,7 @@ func (e *executor) traverseRelated(
 		if walk.maxDepth.reached(next.depth) {
 			continue
 		}
-		neighbors, err := e.relatedNeighbors(expression, walk.kind, walk.direction, next.sym)
-		if err != nil {
-			return err
-		}
-		for _, neighbor := range neighbors {
+		for _, neighbor := range e.relatedNeighbors(walk.kind, walk.direction, next.sym) {
 			key := symbols.KeyOf(neighbor)
 			if _, duplicate := seen[key]; duplicate {
 				continue
@@ -182,19 +194,16 @@ func supportedRelationship(kind string) bool {
 // sym in the given direction, in declaration order. Outgoing lineage reads
 // sym's own declared relationships; every other combination reads the edge
 // tables built from the workspace's declarations.
-func (e *executor) relatedNeighbors(expression queryplan.Expression, kind, direction string, sym *symbols.Symbol) ([]*symbols.Symbol, error) {
+func (e *executor) relatedNeighbors(kind, direction string, sym *symbols.Symbol) []*symbols.Symbol {
 	if relKind, lineage := lineageKinds[kind]; lineage && direction == directionOutgoing {
-		return e.lineageTargets(sym, relKind), nil
-	}
-	edges, err := e.relationshipEdges(expression, kind)
-	if err != nil {
-		return nil, err
+		return e.lineageTargets(sym, relKind)
 	}
+	edges := e.relationshipEdges(kind)
 	table := edges.outgoing
 	if direction == directionIncoming {
 		table = edges.incoming
 	}
-	return table[symbols.KeyOf(sym)], nil
+	return table[symbols.KeyOf(sym)]
 }
 
 // lineageTargets resolves the targets of sym's declared relationships of the
@@ -212,47 +221,36 @@ func (e *executor) lineageTargets(sym *symbols.Symbol, kind ast.RelationshipKind
 	return out
 }
 
-// relationshipEdges returns the edge tables for one relationship kind,
-// building them on first use by scanning the workspace's documents in sorted
-// name order and each document's symbols in declaration order. Every
-// declaration examined charges the shared visit budget; a built table is
-// cached, so later traversals of the same kind read it for free.
-func (e *executor) relationshipEdges(expression queryplan.Expression, kind string) (*relationshipEdges, error) {
-	if cached, ok := e.related.entries[kind]; ok {
-		return cached, nil
+// relationshipEdges returns one kind's edge tables, built on first use by scanning
+// the workspace's documents in name order. The scan is memoized and uncharged;
+// only the elements a traversal reaches pay the visit budget.
+func (e *executor) relationshipEdges(kind string) *relationshipEdges {
+	if cached, ok := e.related.lookup(kind, e.context); ok {
+		return cached
 	}
 	edges := &relationshipEdges{
 		outgoing: make(map[symbols.ElementKey][]*symbols.Symbol),
 		incoming: make(map[symbols.ElementKey][]*symbols.Symbol),
 	}
 	for _, document := range e.context.Index.WorkspaceDocuments() {
-		if err := e.scanScope(expression, edges, kind, e.context.Index.DocumentRoot(document)); err != nil {
-			return nil, err
-		}
+		e.scanScope(edges, kind, e.context.Index.DocumentRoot(document))
 	}
 	e.related.entries[kind] = edges
-	return edges, nil
+	return edges
 }
 
 // scanScope records the edges of one relationship kind that the declarations
-// in scope and its nested scopes state, charging the visit budget per
-// declaration examined.
-func (e *executor) scanScope(expression queryplan.Expression, edges *relationshipEdges, kind string, scope *symbols.Scope) error {
+// in scope and its nested scopes state.
+func (e *executor) scanScope(edges *relationshipEdges, kind string, scope *symbols.Scope) {
 	if scope == nil {
-		return nil
+		return
 	}
 	for _, member := range scope.AllMembers() {
-		if !e.consumeVisit() {
-			return e.budgetError(expression)
-		}
 		e.scanSymbol(edges, kind, member)
 	}
 	for _, child := range scope.Children() {
-		if err := e.scanScope(expression, edges, kind, child); err != nil {
-			return err
-		}
+		e.scanScope(edges, kind, child)
 	}
-	return nil
 }
 
 // scanSymbol records the edges the given symbol's declaration states: the

internal/doc/queryexec/related_column_test.go

@@ -288,12 +288,24 @@ func TestExecuteRelatedColumnReportsItsColumnInErrors(t *testing.T) {
 	}
 }
 
-func TestExecuteRelatedColumnChargesTableConstructionToTheVisitBudget(t *testing.T) {
+func TestExecuteRelatedColumnLeavesTableConstructionUncharged(t *testing.T) {
 	fixture := loadExecutionFixtureFile(t, traceMatrixFixture)
-	// Building the satisfaction table scans the workspace, so a tiny budget
-	// fails even for a requirement nothing satisfies.
-	_, err := fixture.traced(t, ElementValue(fixture.symbol(t, "dataRequirement")),
-		"satisfaction", "incoming", 1, "count", Options{VisitBudget: 3})
+	// Building the satisfaction table scans the workspace uncharged, so a
+	// requirement nothing satisfies is counted within a budget of one.
+	none, err := fixture.traced(t, ElementValue(fixture.symbol(t, "dataRequirement")),
+		"satisfaction", "incoming", 1, "count", Options{VisitBudget: 1})
+	if err != nil {
+		t.Fatalf("unsatisfied: %v", err)
+	}
+	assertColumn(t, cellsByColumn(t, none), "related", [][]string{{"0"}})
+	// The two satisfiers reached are what the budget pays for.
+	mass := ElementValue(fixture.symbol(t, "massRequirement"))
+	both, err := fixture.traced(t, mass, "satisfaction", "incoming", 1, "count", Options{VisitBudget: 2})
+	if err != nil {
+		t.Fatalf("exact budget: %v", err)
+	}
+	assertColumn(t, cellsByColumn(t, both), "related", [][]string{{"2"}})
+	_, err = fixture.traced(t, mass, "satisfaction", "incoming", 1, "count", Options{VisitBudget: 1})
 	execution := executionError(t, err, ErrorVisitBudget)
 	if execution.Property != "related" {
 		t.Fatalf("error column = %q, want related", execution.Property)

internal/doc/queryexec/related_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 
 	"github.com/Open-MBEE/OpenSysML/internal/semantic/symbols"
+	"github.com/Open-MBEE/OpenSysML/internal/syntax/parser"
+	"github.com/Open-MBEE/OpenSysML/internal/syntax/source"
 )
 
 func (f executionFixture) related(
@@ -200,15 +202,111 @@ func TestExecuteRelatedConsumesTheVisitBudget(t *testing.T) {
 	}
 }
 
-func TestExecuteRelatedChargesTableConstructionToTheVisitBudget(t *testing.T) {
+func TestExecuteRelatedLeavesTableConstructionUncharged(t *testing.T) {
 	fixture := loadExecutionFixtureFile(t, "testdata/tmt_relationships.sysml")
-	// Building the edge table scans the workspace, so a tiny budget fails even
-	// when the source has no matching edges.
-	_, err := fixture.related(t, fixture.symbol(t, "Subsystem"), "connection", "incoming", 1,
-		Options{VisitBudget: 3})
-	var executionError *Error
-	if !errors.As(err, &executionError) || executionError.Kind != ErrorVisitBudget {
-		t.Fatalf("visit budget error = %v", err)
+	// Building the edge table scans every declaration in the workspace, yet a
+	// source with no matching edges reaches nothing and so pays nothing.
+	none, err := fixture.related(t, fixture.symbol(t, "Subsystem"), "connection", "incoming", 1,
+		Options{VisitBudget: 1})
+	if err != nil {
+		t.Fatalf("no edges: %v", err)
+	}
+	if len(none.Rows()) != 0 {
+		t.Fatalf("rows = %v, want none", rowNames(none))
+	}
+	// Only the elements reached are charged: the two subsetters fit a budget of
+	// exactly two, however many declarations the scan examined.
+	both, err := fixture.related(t, fixture.symbol(t, "instruments"), "subsetting", "incoming", 1,
+		Options{VisitBudget: 2})
+	if err != nil {
+		t.Fatalf("exact budget: %v", err)
+	}
+	if names := rowNames(both); len(names) != 2 ||
+		names[0] != "Observatory::iris" || names[1] != "Observatory::modhis" {
+		t.Fatalf("rows = %v", names)
+	}
+}
+
+func TestExecuteSharesRelationshipTablesThroughTheContext(t *testing.T) {
+	fixture := loadExecutionFixtureFile(t, "testdata/tmt_relationships.sysml")
+	tables := NewRelationshipTables()
+	context := Context{Index: fixture.index, Resolver: fixture.resolver, Model: fixture.model, Related: tables}
+	bindings := func(source string) Bindings {
+		return Bindings{
+			"source":    {ElementValue(fixture.symbol(t, source))},
+			"kind":      {StringValue("subsetting")},
+			"direction": {StringValue("incoming")},
+			"maxDepth":  {IntegerValue(1)},
+		}
+	}
+	program := fixture.program(t, "Related")
+	if _, err := Execute(program, context, bindings("instruments"), Options{}); err != nil {
+		t.Fatalf("first execution: %v", err)
+	}
+	built, ok := tables.entries["subsetting"]
+	if !ok {
+		t.Fatal("the first execution must leave its subsetting table in the context")
+	}
+	if _, err := Execute(program, context, bindings("Subsystem"), Options{}); err != nil {
+		t.Fatalf("second execution: %v", err)
+	}
+	if tables.entries["subsetting"] != built {
+		t.Fatal("a second execution under the same context must reuse the built table")
+	}
+	// Tables built against another model are discarded rather than trusted.
+	other := loadExecutionFixtureFile(t, "testdata/tmt_relationships.sysml")
+	_, err := Execute(other.program(t, "Related"),
+		Context{Index: other.index, Resolver: other.resolver, Model: other.model, Related: tables},
+		Bindings{
+			"source":    {ElementValue(other.symbol(t, "instruments"))},
+			"kind":      {StringValue("subsetting")},
+			"direction": {StringValue("incoming")},
+			"maxDepth":  {IntegerValue(1)},
+		}, Options{})
+	if err != nil {
+		t.Fatalf("other model: %v", err)
+	}
+	if tables.entries["subsetting"] == built || tables.index != other.index {
+		t.Fatal("tables built against another index must be rebuilt")
+	}
+}
+
+func TestExecuteRebuildsRelationshipTablesAfterAnIndexEdit(t *testing.T) {
+	fixture := loadExecutionFixtureFile(t, "testdata/tmt_relationships.sysml")
+	context := Context{Index: fixture.index, Resolver: fixture.resolver, Model: fixture.model, Related: NewRelationshipTables()}
+	subsetters := func(step string) []string {
+		t.Helper()
+		rows, err := Execute(fixture.program(t, "Related"), context, Bindings{
+			"source":    {ElementValue(fixture.symbol(t, "instruments"))},
+			"kind":      {StringValue("subsetting")},
+			"direction": {StringValue("incoming")},
+			"maxDepth":  {IntegerValue(1)},
+		}, Options{})
+		if err != nil {
+			t.Fatalf("%s: %v", step, err)
+		}
+		return rowNames(rows)
+	}
+	if names := subsetters("before the edit"); len(names) != 2 {
+		t.Fatalf("rows before the edit = %v", names)
+	}
+
+	// The same index, edited in place: a document declaring one more subsetter
+	// is added, then removed again. The tables follow both edits.
+	edit := "edit.sysml"
+	p := parser.New(source.New(edit, []byte("package Edit { part nfiraos :> Observatory::instruments; }")))
+	root := p.ParseFile()
+	if len(p.Diagnostics) > 0 {
+		t.Fatalf("parse edit: %v", p.Diagnostics)
+	}
+	fixture.index.AddDocument(edit, root)
+	fixture.index.ExpandWildcardImports()
+	if names := subsetters("after adding a subsetter"); len(names) != 3 || names[0] != "Edit::nfiraos" {
+		t.Fatalf("rows after adding a subsetter = %v", names)
+	}
+	fixture.index.RemoveDocument(edit)
+	if names := subsetters("after removing it again"); len(names) != 2 {
+		t.Fatalf("rows after removing the subsetter = %v", names)
 	}
 }
 

internal/doc/queryexec/where_related_test.go

@@ -285,7 +285,13 @@ func TestExecuteWhereRelatedChargesTheVisitBudget(t *testing.T) {
 	pair := append(leaf, ElementValue(fixture.symbol(t, "OpticalSubsystem")))
 	_, err = fixture.filtered(t, pair, "specialization", "outgoing", 3, true, Options{VisitBudget: 1})
 	executionError(t, err, ErrorVisitBudget)
-	// Building an edge table charges each declaration scanned, as RelatedElements does.
-	_, err = fixture.filtered(t, leaf, "satisfaction", "incoming", 1, false, Options{VisitBudget: 3})
-	executionError(t, err, ErrorVisitBudget)
+	// Building an edge table is not charged, as for RelatedElements: a row
+	// nothing satisfies is kept without spending a visit.
+	kept, err = fixture.filtered(t, leaf, "satisfaction", "incoming", 1, false, Options{VisitBudget: 1})
+	if err != nil {
+		t.Fatalf("uncharged table: %v", err)
+	}
+	if got := rowNames(kept); len(got) != 1 || got[0] != "Observatory::MirrorAssembly" {
+		t.Fatalf("kept = %v", got)
+	}
 }