perf(store/indexer): defer ContentHash to lazy lookup; trim ExistingF…

CHANGELOG.md

@@ -18,6 +18,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`).
 - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61)
 - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65)
 - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up.
+- **store/indexer:** Defer `ContentHash` from the upfront existing-files map to a lazy per-path lookup. `ExistingFileMeta` further trimmed to `{SizeBytes, MtimeUnixNS}`; `ExistingFiles` / `ExistingFilesForPaths` SQL projection narrowed to `(path, size_bytes, mtime_unix_ns)`. New `Store.LookupFileContentHash(repoID, path)` is consulted only on a (size,mtime) mismatch via a `hashLookup func(rel) (string, bool)` callback threaded into `processFileTask`. The fast path of a no-op repo-wide update never allocates one hex string per file. New `BenchmarkIndexerNoOpUpdateRepoWide_FileScaling` sweeps 500/2000/5000 files. A/B at 2000 files (count=3 × benchtime=8x median): B/op 1854 KB → 1411 KB (~24% less); allocs/op 28,930 → 20,930 (~28% fewer); ns/op 23.0 → 22.7 ms (within noise — FS walk dominates).
 - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66)
 - **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default).
 - **store/resolver:** Schema-backed dot-suffix path for the dominant 2-dot dst_name case (e.g. `pkg.Sub.Func`) via persisted `symbols.dot_tail3` (migration 018) + partial index `idx_symbols_repo_dot_tail3`. New `resolveEdgesByDotTail3` runs as a schema-backed prelude inside `resolveEdgesByDotSuffix` (indexed equality JOIN against `dot_tail3` for dst_names with exactly 2 dots and no slash), leaving only ≥3-dot dst_names to the original `LIKE '%.' || dst_name` fallback. Measured on new `BenchmarkResolveEdgesByDotSuffixLargeScale` (40k symbols, count=3 × benchtime=2x median, A/B): ns/op 73,216,905,800 (73.2 s) → 74,127,650 (74 ms) — **~988× faster**. Symbol insert SQL bumped to 18 columns (batch row cap 58→55).
@@ -33,6 +34,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`).
 - **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44)
 - **store/bench:** Add chooser + callers/callees benchmarks; retry `applyPragmas` on `SQLITE_BUSY`. (#57)
 - **store/bench:** Add per-strategy resolver microbenchmarks (slash-suffix, dot-tail2, dot-suffix). (#59)
+- **store/bench:** Add per-component write-path microbenchmarks: `BenchmarkStoreReplaceFileGraph_EdgeHeavy` (5000 edges, 1 src) isolates per-edge insert; `_TokenHeavy` (200 syms × ~25 tokens) isolates `execTokenTriplesInsert`; `_FileScaling` (16×40 syms vs 80×8 syms at constant total) surfaces per-file batch overhead. Measured: ~5.4 allocs/edge and ~6 allocs/token (interface{} boxing floor on `[]any` row args), ~100 allocs/file batch overhead. Hot loops verified clean (SQL cached, args reused, no per-row string ops); no localized hotspot above noise.
 - **store:** Speed up `FindDeadCode` with new `idx_refs_repo_symbol_id` / `idx_refs_repo_context_symbol_id` indexes. (#54)
 - **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45)
 - **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49)

internal/indexer/indexer.go

@@ -292,9 +292,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err
 				}
 				return
 			}
+			hashLookup := func(rel string) (string, bool, error) {
+				return i.store.LookupFileContentHash(ctxRun, repo.ID, rel)
+			}
 			for task := range tasks {
 				prev, hasPrev := existing[task.rel]
-				res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy)
+				res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy, hashLookup)
 				select {
 				case results <- res:
 				case <-ctxRun.Done():
@@ -694,7 +697,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err
 	return summary, nil
 }
 
-func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string) fileResult {
+func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string, hashLookup func(rel string) (string, bool, error)) fileResult {
 	result := fileResult{task: task}
 	started := time.Now()
 	defer func() {
@@ -742,9 +745,16 @@ func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFile
 		result.hashDur = time.Since(hashStarted)
 	}
 	result.hash = hash
-	if hasPrev && !force && prev.ContentHash == hash {
-		result.action = "touch"
-		return result
+	if hasPrev && !force && hashLookup != nil {
+		prevHash, ok, err := hashLookup(task.rel)
+		if err != nil {
+			result.err = err
+			return result
+		}
+		if ok && prevHash == hash {
+			result.action = "touch"
+			return result
+		}
 	}
 
 	parsed := graph.ParsedFile{Language: task.language}

internal/indexer/indexer_bench_test.go

@@ -87,6 +87,45 @@ func BenchmarkIndexerNoOpUpdateRepoWide(b *testing.B) {
 	}
 }
 
+// BenchmarkIndexerNoOpUpdateRepoWide_FileScaling exposes the per-row cost of
+// the existing-files map by sweeping fixture sizes. Surfaces wins or
+// regressions in the change-detection floor as file count grows; the slim
+// `ExistingFileMeta` (no `ContentHash`) keeps per-row alloc bounded so the
+// curve stays roughly linear in N rather than amplified by per-file string
+// allocs.
+func BenchmarkIndexerNoOpUpdateRepoWide_FileScaling(b *testing.B) {
+	ctx := context.Background()
+	cases := []int{500, 2000, 5000}
+	for _, files := range cases {
+		b.Run(fmt.Sprintf("files=%d", files), func(b *testing.B) {
+			repoRoot := b.TempDir()
+			createGoFixtureRepo(b, repoRoot, files)
+			dbPath := filepath.Join(b.TempDir(), "bench-noop-update-scaling.sqlite")
+			s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()})
+			if err != nil {
+				b.Fatalf("store.Open() error = %v", err)
+			}
+			defer s.Close()
+			idx := New(s, parser.NewRegistry(goparser.New()), nil)
+			if _, err := idx.Index(ctx, Options{RepoRoot: repoRoot}); err != nil {
+				b.Fatalf("Index() error = %v", err)
+			}
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				summary, err := idx.Update(ctx, Options{RepoRoot: repoRoot})
+				if err != nil {
+					b.Fatalf("Update() error = %v", err)
+				}
+				if summary.FilesChanged != 0 {
+					b.Fatalf("expected 0 files changed on no-op update, got %d", summary.FilesChanged)
+				}
+			}
+		})
+	}
+}
+
 func BenchmarkIndexerUpdateOneFile(b *testing.B) {
 	ctx := context.Background()
 	repoRoot := b.TempDir()

internal/store/store.go

@@ -749,25 +749,26 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) {
 }
 
 // ExistingFileMeta is the slim per-row value the change-detection path
-// actually consumes. It intentionally omits the `Path` (held by the map
-// key), `ID`, `Language`, and `IsDeleted` fields — none are read by the
-// indexer worker, and dropping them shrinks the per-entry footprint of the
-// repo-wide existing-files map. On a 2k/5k-file no-op the trim is the
-// dominant remaining alloc on this path.
+// actually consumes upfront. It carries only the (size, mtime) pair —
+// the fields the indexer's fast path checks first. `content_sha256` is
+// deliberately NOT in this struct: it's only consulted in the slow
+// branch (size/mtime mismatch) and is fetched lazily via
+// `LookupFileContentHash` so a repo-wide no-op load doesn't allocate
+// one hex string per file.
 type ExistingFileMeta struct {
 	SizeBytes   int64
 	MtimeUnixNS int64
-	ContentHash string
 }
 
 // ExistingFiles returns active (non-deleted) file records for the repo,
 // keyed by path with a slim `ExistingFileMeta` value. The projection is
-// the change-detection minimum (`size_bytes`, `mtime_unix_ns`,
-// `content_sha256`); `is_deleted = 0` is applied server-side so tombstone
-// rows never reach the Go map.
+// the fast-path minimum (`size_bytes`, `mtime_unix_ns`); `is_deleted = 0`
+// is applied server-side so tombstone rows never reach the Go map. The
+// content hash is fetched on demand via `LookupFileContentHash` only
+// when (size, mtime) differs from disk.
 func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]ExistingFileMeta, error) {
 	rows, err := s.db.QueryContext(ctx, `
-		SELECT path, size_bytes, mtime_unix_ns, content_sha256
+		SELECT path, size_bytes, mtime_unix_ns
 		FROM files
 		WHERE repo_id = ? AND is_deleted = 0
 	`, repoID)
@@ -794,7 +795,7 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [
 		chunk := paths[start:end]
 		placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",")
 		query := `
-			SELECT path, size_bytes, mtime_unix_ns, content_sha256
+			SELECT path, size_bytes, mtime_unix_ns
 			FROM files
 			WHERE repo_id = ? AND is_deleted = 0 AND path IN (` + placeholders + `)
 		`
@@ -814,6 +815,27 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [
 	return out, nil
 }
 
+// LookupFileContentHash returns the stored `content_sha256` for the given
+// active file path, or ("", false, nil) if no live row exists. Used by the
+// indexer's slow-path "touch vs. replace" decision after a (size, mtime)
+// mismatch — folding this into a per-file query keeps the upfront
+// `ExistingFiles` map free of per-row hex strings.
+func (s *Store) LookupFileContentHash(ctx context.Context, repoID int64, path string) (string, bool, error) {
+	var hash string
+	err := s.db.QueryRowContext(ctx, `
+		SELECT content_sha256
+		FROM files
+		WHERE repo_id = ? AND is_deleted = 0 AND path = ?
+	`, repoID, path).Scan(&hash)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return hash, true, nil
+}
+
 // scanExistingFileMetasInto scans rows directly into `dst` so callers can
 // reuse a pre-allocated destination map across chunked queries instead of
 // paying for a per-chunk map alloc + merge-loop.
@@ -822,7 +844,7 @@ func scanExistingFileMetasInto(rows *sql.Rows, dst map[string]ExistingFileMeta)
 	for rows.Next() {
 		var path string
 		var meta ExistingFileMeta
-		if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS, &meta.ContentHash); err != nil {
+		if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS); err != nil {
 			return err
 		}
 		dst[path] = meta

internal/store/store_bench_test.go

@@ -144,6 +144,171 @@ func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy(b *testing.B) {
 	}
 }
 
+// BenchmarkStoreReplaceFileGraph_EdgeHeavy isolates the per-edge insert
+// cost path: 1 source symbol, many edges, no refs, no tokens. If a
+// per-edge `dst_name`/args allocation hotspot exists, this surfaces it
+// without the noise of the symbol-token / FTS / file-token paths.
+func BenchmarkStoreReplaceFileGraph_EdgeHeavy(b *testing.B) {
+	ctx := context.Background()
+	profile := sqliteBenchProfile()
+	b.Logf("sqlite_driver=%s", store.SQLiteDriverName())
+	b.Logf("sqlite_profile=%s", profile)
+
+	dbPath := filepath.Join(b.TempDir(), "store-edges.sqlite")
+	s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile})
+	if err != nil {
+		b.Fatalf("store.OpenWithOptions() error = %v", err)
+	}
+	b.Cleanup(func() { _ = s.Close() })
+
+	repoRoot := b.TempDir()
+	repo, err := s.UpsertRepo(ctx, repoRoot)
+	if err != nil {
+		b.Fatalf("UpsertRepo() error = %v", err)
+	}
+
+	const numEdges = 5000
+	parsed := graph.ParsedFile{Language: "go"}
+	parsed.Symbols = []graph.Symbol{{
+		Language:      "go",
+		Kind:          "function",
+		Name:          "Src",
+		QualifiedName: "bench.Src",
+		Range:         graph.Position{StartLine: 1, StartCol: 1, EndLine: 1, EndCol: 10},
+		StableKey:     "go:bench.Src:0",
+	}}
+	parsed.Edges = make([]graph.Edge, 0, numEdges)
+	for e := 0; e < numEdges; e++ {
+		parsed.Edges = append(parsed.Edges, graph.Edge{
+			DstName:  fmt.Sprintf("bench.Dst_%d", e),
+			Kind:     "calls",
+			Evidence: "bench",
+			Line:     e + 1,
+		})
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/edges.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil {
+			b.Fatalf("ReplaceFileGraph() error = %v", err)
+		}
+	}
+}
+
+// BenchmarkStoreReplaceFileGraph_TokenHeavy isolates the per-token
+// (`execTokenTriplesInsert`) path: small symbol count, large per-symbol
+// token cardinality. Surfaces alloc/CPU cost in token batching loops.
+func BenchmarkStoreReplaceFileGraph_TokenHeavy(b *testing.B) {
+	ctx := context.Background()
+	profile := sqliteBenchProfile()
+	b.Logf("sqlite_driver=%s", store.SQLiteDriverName())
+	b.Logf("sqlite_profile=%s", profile)
+
+	dbPath := filepath.Join(b.TempDir(), "store-tokens.sqlite")
+	s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile})
+	if err != nil {
+		b.Fatalf("store.OpenWithOptions() error = %v", err)
+	}
+	b.Cleanup(func() { _ = s.Close() })
+
+	repoRoot := b.TempDir()
+	repo, err := s.UpsertRepo(ctx, repoRoot)
+	if err != nil {
+		b.Fatalf("UpsertRepo() error = %v", err)
+	}
+
+	// Long doc summary + signature drives the tokenizer to many tokens per
+	// symbol. 200 symbols × ~25 tokens/symbol ≈ 5k symbol_tokens rows.
+	docs := "this is a long descriptive doc summary covering multiple subjects and keywords for benchmark purposes including alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu"
+	parsed := graph.ParsedFile{Language: "go"}
+	const numSymbols = 200
+	parsed.Symbols = make([]graph.Symbol, 0, numSymbols)
+	for i := 0; i < numSymbols; i++ {
+		name := fmt.Sprintf("TokenFn%d", i)
+		qname := fmt.Sprintf("bench.%s", name)
+		parsed.Symbols = append(parsed.Symbols, graph.Symbol{
+			Language:      "go",
+			Kind:          "function",
+			Name:          name,
+			QualifiedName: qname,
+			Signature:     fmt.Sprintf("func %s(ctx context.Context, request *Request, options []Option) (*Response, error)", name),
+			DocSummary:    docs,
+			Range:         graph.Position{StartLine: i + 1, StartCol: 1, EndLine: i + 1, EndCol: 10},
+			StableKey:     fmt.Sprintf("go:%s:%d", qname, i),
+		})
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/tokens.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil {
+			b.Fatalf("ReplaceFileGraph() error = %v", err)
+		}
+	}
+}
+
+// BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling reveals
+// per-batch / per-file overhead by holding total symbol count constant
+// (~640 symbols) but varying file count (16 files × 40 syms vs
+// 80 files × 8 syms). If per-file overhead (file upsert, deleteFileGraphs
+// preamble, args slice setup) dominates over per-symbol cost, the
+// many-files variant is meaningfully slower.
+func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling(b *testing.B) {
+	cases := []struct {
+		name     string
+		numFiles int
+		perFile  int
+	}{
+		{"16files_40syms", 16, 40},
+		{"80files_8syms", 80, 8},
+	}
+	for _, tc := range cases {
+		b.Run(tc.name, func(b *testing.B) {
+			ctx := context.Background()
+			profile := sqliteBenchProfile()
+			dbPath := filepath.Join(b.TempDir(), "store-batch-scaling.sqlite")
+			s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile})
+			if err != nil {
+				b.Fatalf("store.OpenWithOptions() error = %v", err)
+			}
+			b.Cleanup(func() { _ = s.Close() })
+
+			repoRoot := b.TempDir()
+			repo, err := s.UpsertRepo(ctx, repoRoot)
+			if err != nil {
+				b.Fatalf("UpsertRepo() error = %v", err)
+			}
+
+			parsed := heavyParsedFile(tc.perFile, 2, 2)
+			inputs := make([]store.ReplaceFileGraphInput, 0, tc.numFiles)
+			for j := 0; j < tc.numFiles; j++ {
+				inputs = append(inputs, store.ReplaceFileGraphInput{
+					Path:        fmt.Sprintf("bench/scale_%d.go", j),
+					Language:    "go",
+					SizeBytes:   1024,
+					MtimeUnixNS: 0,
+					ContentHash: "",
+					Parsed:      parsed,
+				})
+			}
+			stats := &store.WriteStats{}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				hash := strconv.Itoa(i)
+				for j := range inputs {
+					inputs[j].MtimeUnixNS = int64(i)
+					inputs[j].ContentHash = hash
+				}
+				if _, err := s.ReplaceFileGraphsBatchWithStats(ctx, repo.ID, 1, inputs, stats); err != nil {
+					b.Fatalf("ReplaceFileGraphsBatchWithStats() error = %v", err)
+				}
+			}
+		})
+	}
+}
+
 func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64) {
 	b.Helper()
 	repoRoot := b.TempDir()