From 6498fe18eb93bfc01835c9c16f74f6e0628a54ea Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:12:18 +0200 Subject: [PATCH 1/2] perf(store/indexer): defer ContentHash to lazy lookup; trim ExistingFiles map row size --- CHANGELOG.md | 2 + internal/indexer/indexer.go | 19 ++- internal/indexer/indexer_bench_test.go | 39 ++++++ internal/store/store.go | 46 +++++-- internal/store/store_bench_test.go | 165 +++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f4c387..fcb5de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. +- **store/indexer:** Defer `ContentHash` from the upfront existing-files map to a lazy per-path lookup. `ExistingFileMeta` further trimmed to `{SizeBytes, MtimeUnixNS}`; `ExistingFiles` / `ExistingFilesForPaths` SQL projection narrowed to `(path, size_bytes, mtime_unix_ns)`. New `Store.LookupFileContentHash(repoID, path)` is consulted only on a (size,mtime) mismatch via a `hashLookup func(rel) (string, bool)` callback threaded into `processFileTask`. The fast path of a no-op repo-wide update never allocates one hex string per file. New `BenchmarkIndexerNoOpUpdateRepoWide_FileScaling` sweeps 500/2000/5000 files. A/B at 2000 files (count=3 × benchtime=8x median): B/op 1854 KB → 1411 KB (~24% less); allocs/op 28,930 → 20,930 (~28% fewer); ns/op 23.0 → 22.7 ms (within noise — FS walk dominates). - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default). - **store/resolver:** Schema-backed dot-suffix path for the dominant 2-dot dst_name case (e.g. `pkg.Sub.Func`) via persisted `symbols.dot_tail3` (migration 018) + partial index `idx_symbols_repo_dot_tail3`. New `resolveEdgesByDotTail3` runs as a schema-backed prelude inside `resolveEdgesByDotSuffix` (indexed equality JOIN against `dot_tail3` for dst_names with exactly 2 dots and no slash), leaving only ≥3-dot dst_names to the original `LIKE '%.' || dst_name` fallback. Measured on new `BenchmarkResolveEdgesByDotSuffixLargeScale` (40k symbols, count=3 × benchtime=2x median, A/B): ns/op 73,216,905,800 (73.2 s) → 74,127,650 (74 ms) — **~988× faster**. Symbol insert SQL bumped to 18 columns (batch row cap 58→55). @@ -33,6 +34,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44) - **store/bench:** Add chooser + callers/callees benchmarks; retry `applyPragmas` on `SQLITE_BUSY`. (#57) - **store/bench:** Add per-strategy resolver microbenchmarks (slash-suffix, dot-tail2, dot-suffix). (#59) +- **store/bench:** Add per-component write-path microbenchmarks: `BenchmarkStoreReplaceFileGraph_EdgeHeavy` (5000 edges, 1 src) isolates per-edge insert; `_TokenHeavy` (200 syms × ~25 tokens) isolates `execTokenTriplesInsert`; `_FileScaling` (16×40 syms vs 80×8 syms at constant total) surfaces per-file batch overhead. Measured: ~5.4 allocs/edge and ~6 allocs/token (interface{} boxing floor on `[]any` row args), ~100 allocs/file batch overhead. Hot loops verified clean (SQL cached, args reused, no per-row string ops); no localized hotspot above noise. - **store:** Speed up `FindDeadCode` with new `idx_refs_repo_symbol_id` / `idx_refs_repo_context_symbol_id` indexes. (#54) - **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45) - **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index a7fa29a..bba49d0 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -292,9 +292,16 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } return } + hashLookup := func(rel string) (string, bool) { + h, ok, err := i.store.LookupFileContentHash(ctxRun, repo.ID, rel) + if err != nil { + return "", false + } + return h, ok + } for task := range tasks { prev, hasPrev := existing[task.rel] - res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy) + res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy, hashLookup) select { case results <- res: case <-ctxRun.Done(): @@ -694,7 +701,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, nil } -func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string) fileResult { +func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string, hashLookup func(rel string) (string, bool)) fileResult { result := fileResult{task: task} started := time.Now() defer func() { @@ -742,9 +749,11 @@ func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFile result.hashDur = time.Since(hashStarted) } result.hash = hash - if hasPrev && !force && prev.ContentHash == hash { - result.action = "touch" - return result + if hasPrev && !force && hashLookup != nil { + if prevHash, ok := hashLookup(task.rel); ok && prevHash == hash { + result.action = "touch" + return result + } } parsed := graph.ParsedFile{Language: task.language} diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index 260a41e..f2eb0c3 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -87,6 +87,45 @@ func BenchmarkIndexerNoOpUpdateRepoWide(b *testing.B) { } } +// BenchmarkIndexerNoOpUpdateRepoWide_FileScaling exposes the per-row cost of +// the existing-files map by sweeping fixture sizes. Surfaces wins or +// regressions in the change-detection floor as file count grows; the slim +// `ExistingFileMeta` (no `ContentHash`) keeps per-row alloc bounded so the +// curve stays roughly linear in N rather than amplified by per-file string +// allocs. +func BenchmarkIndexerNoOpUpdateRepoWide_FileScaling(b *testing.B) { + ctx := context.Background() + cases := []int{500, 2000, 5000} + for _, files := range cases { + b.Run(fmt.Sprintf("files=%d", files), func(b *testing.B) { + repoRoot := b.TempDir() + createGoFixtureRepo(b, repoRoot, files) + dbPath := filepath.Join(b.TempDir(), "bench-noop-update-scaling.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()}) + if err != nil { + b.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, Options{RepoRoot: repoRoot}); err != nil { + b.Fatalf("Index() error = %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + summary, err := idx.Update(ctx, Options{RepoRoot: repoRoot}) + if err != nil { + b.Fatalf("Update() error = %v", err) + } + if summary.FilesChanged != 0 { + b.Fatalf("expected 0 files changed on no-op update, got %d", summary.FilesChanged) + } + } + }) + } +} + func BenchmarkIndexerUpdateOneFile(b *testing.B) { ctx := context.Background() repoRoot := b.TempDir() diff --git a/internal/store/store.go b/internal/store/store.go index ef4e2f8..14e368f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -749,25 +749,26 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { } // ExistingFileMeta is the slim per-row value the change-detection path -// actually consumes. It intentionally omits the `Path` (held by the map -// key), `ID`, `Language`, and `IsDeleted` fields — none are read by the -// indexer worker, and dropping them shrinks the per-entry footprint of the -// repo-wide existing-files map. On a 2k/5k-file no-op the trim is the -// dominant remaining alloc on this path. +// actually consumes upfront. It carries only the (size, mtime) pair — +// the fields the indexer's fast path checks first. `content_sha256` is +// deliberately NOT in this struct: it's only consulted in the slow +// branch (size/mtime mismatch) and is fetched lazily via +// `LookupFileContentHash` so a repo-wide no-op load doesn't allocate +// one hex string per file. type ExistingFileMeta struct { SizeBytes int64 MtimeUnixNS int64 - ContentHash string } // ExistingFiles returns active (non-deleted) file records for the repo, // keyed by path with a slim `ExistingFileMeta` value. The projection is -// the change-detection minimum (`size_bytes`, `mtime_unix_ns`, -// `content_sha256`); `is_deleted = 0` is applied server-side so tombstone -// rows never reach the Go map. +// the fast-path minimum (`size_bytes`, `mtime_unix_ns`); `is_deleted = 0` +// is applied server-side so tombstone rows never reach the Go map. The +// content hash is fetched on demand via `LookupFileContentHash` only +// when (size, mtime) differs from disk. func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]ExistingFileMeta, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT path, size_bytes, mtime_unix_ns, content_sha256 + SELECT path, size_bytes, mtime_unix_ns FROM files WHERE repo_id = ? AND is_deleted = 0 `, repoID) @@ -794,7 +795,7 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ chunk := paths[start:end] placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") query := ` - SELECT path, size_bytes, mtime_unix_ns, content_sha256 + SELECT path, size_bytes, mtime_unix_ns FROM files WHERE repo_id = ? AND is_deleted = 0 AND path IN (` + placeholders + `) ` @@ -814,6 +815,27 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ return out, nil } +// LookupFileContentHash returns the stored `content_sha256` for the given +// active file path, or ("", false, nil) if no live row exists. Used by the +// indexer's slow-path "touch vs. replace" decision after a (size, mtime) +// mismatch — folding this into a per-file query keeps the upfront +// `ExistingFiles` map free of per-row hex strings. +func (s *Store) LookupFileContentHash(ctx context.Context, repoID int64, path string) (string, bool, error) { + var hash string + err := s.db.QueryRowContext(ctx, ` + SELECT content_sha256 + FROM files + WHERE repo_id = ? AND is_deleted = 0 AND path = ? + `, repoID, path).Scan(&hash) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return hash, true, nil +} + // scanExistingFileMetasInto scans rows directly into `dst` so callers can // reuse a pre-allocated destination map across chunked queries instead of // paying for a per-chunk map alloc + merge-loop. @@ -822,7 +844,7 @@ func scanExistingFileMetasInto(rows *sql.Rows, dst map[string]ExistingFileMeta) for rows.Next() { var path string var meta ExistingFileMeta - if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS, &meta.ContentHash); err != nil { + if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS); err != nil { return err } dst[path] = meta diff --git a/internal/store/store_bench_test.go b/internal/store/store_bench_test.go index 3da6504..a681413 100644 --- a/internal/store/store_bench_test.go +++ b/internal/store/store_bench_test.go @@ -144,6 +144,171 @@ func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy(b *testing.B) { } } +// BenchmarkStoreReplaceFileGraph_EdgeHeavy isolates the per-edge insert +// cost path: 1 source symbol, many edges, no refs, no tokens. If a +// per-edge `dst_name`/args allocation hotspot exists, this surfaces it +// without the noise of the symbol-token / FTS / file-token paths. +func BenchmarkStoreReplaceFileGraph_EdgeHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-edges.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + const numEdges = 5000 + parsed := graph.ParsedFile{Language: "go"} + parsed.Symbols = []graph.Symbol{{ + Language: "go", + Kind: "function", + Name: "Src", + QualifiedName: "bench.Src", + Range: graph.Position{StartLine: 1, StartCol: 1, EndLine: 1, EndCol: 10}, + StableKey: "go:bench.Src:0", + }} + parsed.Edges = make([]graph.Edge, 0, numEdges) + for e := 0; e < numEdges; e++ { + parsed.Edges = append(parsed.Edges, graph.Edge{ + DstName: fmt.Sprintf("bench.Dst_%d", e), + Kind: "calls", + Evidence: "bench", + Line: e + 1, + }) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/edges.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil { + b.Fatalf("ReplaceFileGraph() error = %v", err) + } + } +} + +// BenchmarkStoreReplaceFileGraph_TokenHeavy isolates the per-token +// (`execTokenTriplesInsert`) path: small symbol count, large per-symbol +// token cardinality. Surfaces alloc/CPU cost in token batching loops. +func BenchmarkStoreReplaceFileGraph_TokenHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-tokens.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + // Long doc summary + signature drives the tokenizer to many tokens per + // symbol. 200 symbols × ~25 tokens/symbol ≈ 5k symbol_tokens rows. + docs := "this is a long descriptive doc summary covering multiple subjects and keywords for benchmark purposes including alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu" + parsed := graph.ParsedFile{Language: "go"} + const numSymbols = 200 + parsed.Symbols = make([]graph.Symbol, 0, numSymbols) + for i := 0; i < numSymbols; i++ { + name := fmt.Sprintf("TokenFn%d", i) + qname := fmt.Sprintf("bench.%s", name) + parsed.Symbols = append(parsed.Symbols, graph.Symbol{ + Language: "go", + Kind: "function", + Name: name, + QualifiedName: qname, + Signature: fmt.Sprintf("func %s(ctx context.Context, request *Request, options []Option) (*Response, error)", name), + DocSummary: docs, + Range: graph.Position{StartLine: i + 1, StartCol: 1, EndLine: i + 1, EndCol: 10}, + StableKey: fmt.Sprintf("go:%s:%d", qname, i), + }) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/tokens.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil { + b.Fatalf("ReplaceFileGraph() error = %v", err) + } + } +} + +// BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling reveals +// per-batch / per-file overhead by holding total symbol count constant +// (~640 symbols) but varying file count (16 files × 40 syms vs +// 80 files × 8 syms). If per-file overhead (file upsert, deleteFileGraphs +// preamble, args slice setup) dominates over per-symbol cost, the +// many-files variant is meaningfully slower. +func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling(b *testing.B) { + cases := []struct { + name string + numFiles int + perFile int + }{ + {"16files_40syms", 16, 40}, + {"80files_8syms", 80, 8}, + } + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + dbPath := filepath.Join(b.TempDir(), "store-batch-scaling.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + parsed := heavyParsedFile(tc.perFile, 2, 2) + inputs := make([]store.ReplaceFileGraphInput, 0, tc.numFiles) + for j := 0; j < tc.numFiles; j++ { + inputs = append(inputs, store.ReplaceFileGraphInput{ + Path: fmt.Sprintf("bench/scale_%d.go", j), + Language: "go", + SizeBytes: 1024, + MtimeUnixNS: 0, + ContentHash: "", + Parsed: parsed, + }) + } + stats := &store.WriteStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + hash := strconv.Itoa(i) + for j := range inputs { + inputs[j].MtimeUnixNS = int64(i) + inputs[j].ContentHash = hash + } + if _, err := s.ReplaceFileGraphsBatchWithStats(ctx, repo.ID, 1, inputs, stats); err != nil { + b.Fatalf("ReplaceFileGraphsBatchWithStats() error = %v", err) + } + } + }) + } +} + func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64) { b.Helper() repoRoot := b.TempDir() From 10da57f83f703ee0d8b22d39e469e7d9d2ca45f5 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:19:07 +0200 Subject: [PATCH 2/2] fix(indexer): propagate hashLookup errors through processFileTask --- internal/indexer/indexer.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index bba49d0..b01170b 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -292,12 +292,8 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } return } - hashLookup := func(rel string) (string, bool) { - h, ok, err := i.store.LookupFileContentHash(ctxRun, repo.ID, rel) - if err != nil { - return "", false - } - return h, ok + hashLookup := func(rel string) (string, bool, error) { + return i.store.LookupFileContentHash(ctxRun, repo.ID, rel) } for task := range tasks { prev, hasPrev := existing[task.rel] @@ -701,7 +697,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, nil } -func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string, hashLookup func(rel string) (string, bool)) fileResult { +func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string, hashLookup func(rel string) (string, bool, error)) fileResult { result := fileResult{task: task} started := time.Now() defer func() { @@ -750,7 +746,12 @@ func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFile } result.hash = hash if hasPrev && !force && hashLookup != nil { - if prevHash, ok := hashLookup(task.rel); ok && prevHash == hash { + prevHash, ok, err := hashLookup(task.rel) + if err != nil { + result.err = err + return result + } + if ok && prevHash == hash { result.action = "touch" return result }