perf(repl): let a submission's parse trees go before its analysis

changes/unreleased/release-preparse-trees.fixed.md

@@ -0,0 +1 @@
+- A submission's parse trees are let go once its files are accepted, instead of being held through the analysis for the root namespaces an unresolved reference would be looked up by (#1441); a reference left unresolved parses the text again. Validating a generated 12 086-element model peaks at 164 MiB of resident memory instead of 184, beside 0.11.0's 164 on the same toolchain (152 against 147 under `GOGC=25`), with the load's time and allocations unchanged.

internal/frontend/repl/resolve_walk_test.go

@@ -4,18 +4,14 @@ import (
 	"fmt"
 	"strings"
 	"testing"
-
-	"github.com/Open-MBEE/OpenSysML/internal/syntax/parser"
-	"github.com/Open-MBEE/OpenSysML/internal/syntax/source"
 )
 
-func resolvedParses(n int) []parsed {
+func resolvedSources(n int) []SourceFile {
 	var sb strings.Builder
 	for i := 0; i < n; i++ {
 		fmt.Fprintf(&sb, "package P%d { attribute a%d = %d; }\n", i, i, i)
 	}
-	root := parser.New(source.New("typed", []byte(sb.String()))).ParseFile()
-	return []parsed{{root: root}}
+	return []SourceFile{{Name: "typed", Text: sb.String()}}
 }
 
 // A submission that leaves nothing unresolved looks for no root namespace, so
@@ -24,9 +20,9 @@ func TestResolvingNothingDoesNotWalkTheText(t *testing.T) {
 	s, dir := workspaceSession(t)
 	search := &rootSearch{find: s.workspaceIndex().Declaring, where: "under " + dir, onDemand: true}
 	cost := func(n int) float64 {
-		parses := resolvedParses(n)
+		sources := resolvedSources(n)
 		return testing.AllocsPerRun(20, func() {
-			s.resolveReferences(Result{}, parses, search)
+			s.resolveReferences(Result{}, &submissionRoots{pending: sources}, search)
 		})
 	}
 	if small, large := cost(50), cost(100); large > small {

internal/frontend/repl/session.go

@@ -1139,7 +1139,8 @@ func (s *Session) submitEach(files []SourceFile) (res Result, byFile [][]string,
 	// A root namespace the submission leaves unresolved may be declared by a
 	// file beside a loaded one or under the workspace; it joins the submission.
 	search := s.searchFor(load, files)
-	if ws, more := s.pullReferences(s.resolveReferences(res, parses, search), parses, search); ws.loaded || len(ws.notices) > 0 {
+	roots := &submissionRoots{pending: files}
+	if ws, more := s.pullReferences(s.resolveReferences(res, roots, search), roots, search); ws.loaded || len(ws.notices) > 0 {
 		if ws.loaded {
 			whole = append(whole, more...)
 			// A loaded file may have replaced an earlier snippet, moving the

internal/frontend/repl/workspace.go

@@ -70,12 +70,12 @@ func (s *Session) doWorkspace() []string {
 }
 
 // workspaceLoad is what resolving references did: the files it accepted into
-// the current generation, if any, their parses, what they declared, what
+// the current generation, if any, their sources, what they declared, what
 // accepting them dropped, and the lines to report about it.
 type workspaceLoad struct {
 	loaded   bool
 	files    []string
-	parses   []parsed
+	sources  []SourceFile
 	declared []string
 	drops    []dropReport
 	notices  []string
@@ -115,12 +115,12 @@ func (s *Session) searchFor(load bool, files []SourceFile) *rootSearch {
 	return nil
 }
 
-// resolveReferences loads, for a submission whose text parsed to parses and
-// analyzed to res, the files declaring the root namespaces its imports and
-// qualified references leave unresolved, found where search looks. Imports
-// come first: a qualified name an import would answer is not looked for until
-// the imports have been (import Lib::*; then Inner::Thing with Lib::Inner).
-func (s *Session) resolveReferences(res Result, parses []parsed, search *rootSearch) workspaceLoad {
+// resolveReferences loads, for a submission of sources that analyzed to res,
+// the files declaring the root namespaces its imports and qualified references
+// leave unresolved, found where search looks. Imports come first: a qualified
+// name an import would answer is not looked for until the imports have been
+// (import Lib::*; then Inner::Thing with Lib::Inner).
+func (s *Session) resolveReferences(res Result, roots *submissionRoots, search *rootSearch) workspaceLoad {
 	if search == nil {
 		return workspaceLoad{}
 	}
@@ -138,30 +138,58 @@ func (s *Session) resolveReferences(res Result, parses []parsed, search *rootSea
 		return workspaceLoad{}
 	}
 	unresolved := func(name string) bool { return referenceBegins(subjects, name) }
-	var imported []string
-	for _, pre := range parses {
-		imported = append(imported, project.ImportedRoots(pre.root)...)
-	}
+	imported, referenced := roots.names(s)
 	ws := s.loadDeclaring(s.undeclaredRoots(imported, unresolved), search)
 	if !ws.loaded {
 		// An import no file answers must not keep the qualified names waiting.
-		var referenced []string
-		for _, pre := range parses {
-			referenced = append(referenced, project.ReferencedRoots(pre.root)...)
-		}
 		more := s.loadDeclaring(s.undeclaredRoots(referenced, unresolved), search)
 		more.notices = append(ws.notices, more.notices...)
 		ws = more
 	}
 	return ws
 }
 
-// pullReferences carries on from what loading ws did for the text parsed to
-// parses: the session is rebuilt over the files loaded, and the root
+// submissionRoots is the root namespaces a submission's sources import and
+// reference, by name: a source is parsed once, on first need, and its tree let go.
+type submissionRoots struct {
+	pending    []SourceFile
+	imported   []string
+	referenced []string
+}
+
+// names is the imported and the referenced root names, parsing what is pending.
+func (r *submissionRoots) names(s *Session) (imported, referenced []string) {
+	if len(r.pending) > 0 {
+		for _, root := range s.parseRoots(r.pending) {
+			r.imported = append(r.imported, project.ImportedRoots(root)...)
+			r.referenced = append(r.referenced, project.ReferencedRoots(root)...)
+		}
+		r.pending = nil
+	}
+	return r.imported, r.referenced
+}
+
+// add takes in the files a round loaded, to parse if a later round asks.
+func (r *submissionRoots) add(sources []SourceFile) { r.pending = append(r.pending, sources...) }
+
+// parseRoots parses the sources for the root namespaces they import and reference.
+func (s *Session) parseRoots(sources []SourceFile) []*ast.RootNamespace {
+	roots := make([]*ast.RootNamespace, len(sources))
+	model.ParallelFor(s.jobs, len(sources), func(i int) {
+		roots[i] = preparseWithKind(sources[i].Name, sources[i].Text, sources[i].Kind).root
+	})
+	return roots
+}
+
+// pullReferences carries on from what loading ws did for the submission whose
+// roots these are: the session is rebuilt over the files loaded, and the root
 // namespaces the text and those files leave unresolved in turn are loaded the
 // same way, round after round, until a round loads nothing. It reports what
 // the rounds did together, and what rebuilding did to the session.
-func (s *Session) pullReferences(ws workspaceLoad, parses []parsed, search *rootSearch) (all workspaceLoad, whole []string) {
+func (s *Session) pullReferences(ws workspaceLoad, roots *submissionRoots, search *rootSearch) (all workspaceLoad, whole []string) {
+	if roots == nil {
+		roots = &submissionRoots{}
+	}
 	// A root two files declare is noticed by every round that wants it; once is enough.
 	noticed := map[string]bool{}
 	notice := func(notices []string) {
@@ -183,8 +211,8 @@ func (s *Session) pullReferences(ws workspaceLoad, parses []parsed, search *root
 		whole = append(whole, more...)
 		all.notices = append(append(all.notices, dropNotices(ws.drops)...), more...)
 		notice(ws.notices)
-		parses = append(parses, ws.parses...)
-		ws = s.resolveReferences(s.generationResult(), parses, search)
+		roots.add(ws.sources)
+		ws = s.resolveReferences(s.generationResult(), roots, search)
 	}
 	notice(ws.notices)
 	return all, whole
@@ -318,13 +346,14 @@ func (s *Session) loadDeclaring(names []string, search *rootSearch) workspaceLoa
 		out.notices = append(out.notices, errPrefix+err.Error())
 		return out
 	}
-	out.parses = make([]parsed, len(read.files))
+	parses := make([]parsed, len(read.files))
 	model.ParallelFor(s.jobs, len(read.files), func(i int) {
-		out.parses[i] = preparseWithKind(read.files[i].Name, read.files[i].Text, read.files[i].Kind)
+		parses[i] = preparseWithKind(read.files[i].Name, read.files[i].Text, read.files[i].Kind)
 	})
+	out.sources = read.files
 	seen := map[string]bool{}
 	for i, f := range read.files {
-		names, dropped := s.acceptParsed(f.Name, f.key(), fileKeyOf(f.Of), f.Text, out.parses[i], f.Kind)
+		names, dropped := s.acceptParsed(f.Name, f.key(), fileKeyOf(f.Of), f.Text, parses[i], f.Kind)
 		s.snippets[len(s.snippets)-1].onDemand = search.onDemand
 		if f.Of != "" {
 			out.files = append(out.files, f.Of)