From 7ed6efc473c4b42cdbfb140da1e0b3f1e8878893 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:42:41 +0200 Subject: [PATCH 01/59] perf: batch ReplaceFileGraph writes during indexing (#20) * perf: batch ReplaceFileGraph writes during indexing * perf: reduce indexing DB roundtrips (batch file IDs, set-based deletes) --- internal/indexer/indexer.go | 82 +++++++++-- internal/store/store.go | 277 ++++++++++++++++++++++++------------ 2 files changed, 256 insertions(+), 103 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 413e007..822b971 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -228,9 +228,11 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err }() const metadataBatchSize = 300 + const replaceBatchSize = 20 markSeenBatch := make([]string, 0, metadataBatchSize) touchBatch := make([]store.FileMetadataUpdate, 0, metadataBatchSize) parseFailedBatch := make([]store.FileMetadataUpdate, 0, metadataBatchSize) + replaceBatch := make([]store.ReplaceFileGraphInput, 0, replaceBatchSize) changedPathSet := make(map[string]struct{}, 64) flushMarkSeen := func() error { @@ -263,6 +265,26 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err parseFailedBatch = parseFailedBatch[:0] return nil } + flushReplace := func() error { + if len(replaceBatch) == 0 { + return nil + } + fileIDs, err := i.store.ReplaceFileGraphsBatch(ctx, repo.ID, scanID, replaceBatch) + if err != nil { + return err + } + if !embedding.IsNoop(i.embedder) { + for idx, input := range replaceBatch { + fileID := int64(0) + if idx < len(fileIDs) { + fileID = fileIDs[idx] + } + i.embedFileSymbolsWithFileID(ctx, repo.ID, fileID, input.Parsed) + } + } + replaceBatch = replaceBatch[:0] + return nil + } var runErr error var parseMS int64 @@ -332,21 +354,27 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err summary.FilesSkipped++ case "replace": writeStart := time.Now() - if err := i.store.ReplaceFileGraph(ctx, repo.ID, scanID, res.task.rel, res.parsed.Language, res.task.info.Size(), res.task.info.ModTime().UnixNano(), res.hash, res.parsed); err != nil { - runErr = err - cancel() - continue - } - if !embedding.IsNoop(i.embedder) { - i.embedFileSymbols(ctx, repo.ID, res.task.rel, res.parsed) - } - writeMS += time.Since(writeStart).Milliseconds() + replaceBatch = append(replaceBatch, store.ReplaceFileGraphInput{ + Path: res.task.rel, + Language: res.parsed.Language, + SizeBytes: res.task.info.Size(), + MtimeUnixNS: res.task.info.ModTime().UnixNano(), + ContentHash: res.hash, + Parsed: res.parsed, + }) changedPathSet[res.task.rel] = struct{}{} summary.FilesChanged++ summary.FilesIndexed++ coverage := summary.LanguageCoverage[coverageLanguage] coverage.Indexed++ summary.LanguageCoverage[coverageLanguage] = coverage + if len(replaceBatch) >= replaceBatchSize { + if err := flushReplace(); err != nil { + runErr = err + cancel() + } + } + writeMS += time.Since(writeStart).Milliseconds() case "parse_failed": writeStart := time.Now() parseFailedBatch = append(parseFailedBatch, store.FileMetadataUpdate{ @@ -383,6 +411,14 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } writeMS += time.Since(writeStart).Milliseconds() } + if runErr == nil { + writeStart := time.Now() + if err := flushReplace(); err != nil { + runErr = err + cancel() + } + writeMS += time.Since(writeStart).Milliseconds() + } if runErr == nil { writeStart := time.Now() if err := flushTouch(); err != nil { @@ -689,3 +725,31 @@ func (i *Indexer) embedFileSymbols(ctx context.Context, repoID int64, relPath st _ = i.store.UpsertSymbolEmbeddings(ctx, repoID, fileID, "", symbolMap) } } + +func (i *Indexer) embedFileSymbolsWithFileID(ctx context.Context, repoID int64, fileID int64, parsed graph.ParsedFile) { + if len(parsed.Symbols) == 0 || fileID == 0 { + return + } + + texts := make([]string, len(parsed.Symbols)) + keys := make([]string, len(parsed.Symbols)) + for j, sym := range parsed.Symbols { + texts[j] = embedding.FormatSymbolText(sym.Kind, sym.QualifiedName, sym.Signature, sym.DocSummary) + keys[j] = sym.StableKey + } + + vectors, err := i.embedder.EmbedBatch(ctx, texts) + if err != nil { + return // best-effort: don't fail indexing on embedding errors + } + + symbolMap := make(map[string][]float32, len(keys)) + for j, key := range keys { + if vectors[j] != nil { + symbolMap[key] = vectors[j] + } + } + if len(symbolMap) > 0 { + _ = i.store.UpsertSymbolEmbeddings(ctx, repoID, fileID, "", symbolMap) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index a216bb3..e0c638a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -113,6 +113,15 @@ type ExportEdge struct { Line int `json:"line"` } +type ReplaceFileGraphInput struct { + Path string + Language string + SizeBytes int64 + MtimeUnixNS int64 + ContentHash string + Parsed graph.ParsedFile +} + func Open(path string) (*Store, error) { return OpenWithOptions(path, OpenOptions{}) } @@ -662,12 +671,27 @@ func (s *Store) MarkFilesParseFailedBatch(ctx context.Context, repoID, scanID in } func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path, language string, sizeBytes, mtimeUnixNS int64, contentHash string, parsed graph.ParsedFile) error { + _, err := s.ReplaceFileGraphsBatch(ctx, repoID, scanID, []ReplaceFileGraphInput{{ + Path: path, + Language: language, + SizeBytes: sizeBytes, + MtimeUnixNS: mtimeUnixNS, + ContentHash: contentHash, + Parsed: parsed, + }}) + return err +} + +func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64, inputs []ReplaceFileGraphInput) ([]int64, error) { + if len(inputs) == 0 { + return nil, nil + } tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return err + return nil, err } - now := time.Now().UTC().Format(time.RFC3339) - if _, err := tx.ExecContext(ctx, ` + + upsertFileStmt, err := tx.PrepareContext(ctx, ` INSERT INTO files(repo_id, path, language, size_bytes, mtime_unix_ns, content_sha256, parse_state, last_scan_id, indexed_at, is_deleted) VALUES(?, ?, ?, ?, ?, ?, 'indexed', ?, ?, 0) ON CONFLICT(repo_id, path) @@ -680,19 +704,39 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path last_scan_id = excluded.last_scan_id, indexed_at = excluded.indexed_at, is_deleted = 0 - `, repoID, path, language, sizeBytes, mtimeUnixNS, contentHash, scanID, now); err != nil { + `) + if err != nil { _ = tx.Rollback() - return err + return nil, err } - var fileID int64 - if err := tx.QueryRowContext(ctx, `SELECT id FROM files WHERE repo_id = ? AND path = ?`, repoID, path).Scan(&fileID); err != nil { + defer upsertFileStmt.Close() + + selectFileIDStmt, err := tx.PrepareContext(ctx, `SELECT id FROM files WHERE repo_id = ? AND path = ?`) + if err != nil { _ = tx.Rollback() - return err + return nil, err } - if err := deleteFileGraph(ctx, tx, fileID); err != nil { + defer selectFileIDStmt.Close() + + deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, ` + DELETE FROM symbol_tokens + WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) + `) + if err != nil { _ = tx.Rollback() - return err + return nil, err } + defer deleteSymbolTokensStmt.Close() + + deleteSymbolFTSStmt, err := tx.PrepareContext(ctx, ` + DELETE FROM symbol_fts + WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) + `) + if err != nil { + _ = tx.Rollback() + return nil, err + } + defer deleteSymbolFTSStmt.Close() insertSymbolStmt, err := tx.PrepareContext(ctx, ` INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) @@ -700,7 +744,7 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path `) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertSymbolStmt.Close() @@ -710,14 +754,14 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path `) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertSymbolFTSStmt.Close() insertSymbolTokenStmt, err := tx.PrepareContext(ctx, `INSERT INTO symbol_tokens(symbol_id, token, weight) VALUES(?, ?, ?)`) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertSymbolTokenStmt.Close() @@ -727,7 +771,7 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path `) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertReferenceStmt.Close() @@ -737,21 +781,21 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path `) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertEdgeStmt.Close() insertImportStmt, err := tx.PrepareContext(ctx, `INSERT INTO file_imports(repo_id, file_id, import_path) VALUES(?, ?, ?)`) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertImportStmt.Close() insertFileTokenStmt, err := tx.PrepareContext(ctx, `INSERT INTO file_tokens(file_id, token, weight) VALUES(?, ?, ?)`) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertFileTokenStmt.Close() @@ -761,30 +805,137 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path `) if err != nil { _ = tx.Rollback() - return err + return nil, err } defer insertTestLinkStmt.Close() + now := time.Now().UTC().Format(time.RFC3339) + fileIDs := make([]int64, 0, len(inputs)) + for _, input := range inputs { + if _, err := upsertFileStmt.ExecContext(ctx, repoID, input.Path, input.Language, input.SizeBytes, input.MtimeUnixNS, input.ContentHash, scanID, now); err != nil { + _ = tx.Rollback() + return nil, err + } + var fileID int64 + if err := selectFileIDStmt.QueryRowContext(ctx, repoID, input.Path).Scan(&fileID); err != nil { + _ = tx.Rollback() + return nil, err + } + if err := deleteFileGraphWithStmts(ctx, tx, fileID, deleteSymbolTokensStmt, deleteSymbolFTSStmt); err != nil { + _ = tx.Rollback() + return nil, err + } + fileIDs = append(fileIDs, fileID) + if err := insertParsedFileGraph( + ctx, + tx, + repoID, + fileID, + input.Parsed, + insertSymbolStmt, + insertSymbolFTSStmt, + insertSymbolTokenStmt, + insertReferenceStmt, + insertEdgeStmt, + insertImportStmt, + insertFileTokenStmt, + insertTestLinkStmt, + ); err != nil { + _ = tx.Rollback() + return nil, err + } + } + + if err := tx.Commit(); err != nil { + return nil, err + } + return fileIDs, nil +} + +func deleteFileGraph(ctx context.Context, tx *sql.Tx, fileID int64) error { + deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, ` + DELETE FROM symbol_tokens + WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) + `) + if err != nil { + return err + } + defer deleteSymbolTokensStmt.Close() + + deleteSymbolFTSStmt, err := tx.PrepareContext(ctx, ` + DELETE FROM symbol_fts + WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) + `) + if err != nil { + return err + } + defer deleteSymbolFTSStmt.Close() + + return deleteFileGraphWithStmts(ctx, tx, fileID, deleteSymbolTokensStmt, deleteSymbolFTSStmt) +} + +func deleteFileGraphWithStmts(ctx context.Context, tx *sql.Tx, fileID int64, deleteSymbolTokensStmt, deleteSymbolFTSStmt *sql.Stmt) error { + if _, err := deleteSymbolTokensStmt.ExecContext(ctx, fileID); err != nil { + return err + } + if _, err := deleteSymbolFTSStmt.ExecContext(ctx, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM edges WHERE file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM references_tbl WHERE file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM file_imports WHERE file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM file_tokens WHERE file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM test_links WHERE test_file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM symbol_embeddings WHERE file_id = ?`, fileID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM symbols WHERE file_id = ?`, fileID); err != nil { + return err + } + return nil +} + +func insertParsedFileGraph( + ctx context.Context, + tx *sql.Tx, + repoID int64, + fileID int64, + parsed graph.ParsedFile, + insertSymbolStmt *sql.Stmt, + insertSymbolFTSStmt *sql.Stmt, + insertSymbolTokenStmt *sql.Stmt, + insertReferenceStmt *sql.Stmt, + insertEdgeStmt *sql.Stmt, + insertImportStmt *sql.Stmt, + insertFileTokenStmt *sql.Stmt, + insertTestLinkStmt *sql.Stmt, +) error { stableToID := map[string]int64{} for _, sym := range parsed.Symbols { res, err := insertSymbolStmt.ExecContext(ctx, repoID, fileID, sym.Language, sym.Kind, sym.Name, sym.QualifiedName, sym.ContainerName, sym.Signature, sym.Visibility, sym.Range.StartLine, sym.Range.StartCol, sym.Range.EndLine, sym.Range.EndCol, sym.DocSummary, sym.StableKey) if err != nil { - _ = tx.Rollback() return err } symbolID, err := res.LastInsertId() if err != nil { - _ = tx.Rollback() return err } stableToID[sym.StableKey] = symbolID if _, err := insertSymbolFTSStmt.ExecContext(ctx, repoID, symbolID, sym.Name, sym.QualifiedName, sym.Signature, sym.DocSummary); err != nil { - _ = tx.Rollback() return err } for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { if _, err := insertSymbolTokenStmt.ExecContext(ctx, symbolID, token, weight); err != nil { - _ = tx.Rollback() return err } } @@ -800,7 +951,6 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path contextID = contextSymbolID } if _, err := insertReferenceStmt.ExecContext(ctx, repoID, fileID, symbolID, ref.Kind, ref.Name, ref.QualifiedName, ref.Range.StartLine, ref.Range.StartCol, ref.Range.EndLine, ref.Range.EndCol, contextID); err != nil { - _ = tx.Rollback() return err } } @@ -810,19 +960,16 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path continue } if _, err := insertEdgeStmt.ExecContext(ctx, repoID, srcID, edge.DstName, edge.Kind, edge.Evidence, fileID, edge.Line); err != nil { - _ = tx.Rollback() return err } } for _, imp := range parsed.Imports { if _, err := insertImportStmt.ExecContext(ctx, repoID, fileID, imp); err != nil { - _ = tx.Rollback() return err } } for token, weight := range parsed.FileTokens { if _, err := insertFileTokenStmt.ExecContext(ctx, fileID, token, weight); err != nil { - _ = tx.Rollback() return err } } @@ -837,12 +984,10 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path for key := range targetKeySet { targetKeys = append(targetKeys, key) } - targetStableToID, err := s.resolveSymbolsByStableKeys(ctx, repoID, targetKeys) + targetStableToID, err := resolveSymbolsByStableKeysQuery(ctx, tx, repoID, targetKeys) if err != nil { - _ = tx.Rollback() return err } - for _, link := range parsed.TestLinks { var testSymbolID any var targetSymbolID any @@ -853,73 +998,9 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path targetSymbolID = id } if _, err := insertTestLinkStmt.ExecContext(ctx, repoID, fileID, testSymbolID, targetSymbolID, link.Reason, link.Score); err != nil { - _ = tx.Rollback() - return err - } - } - if err := tx.Commit(); err != nil { - return err - } - return nil -} - -func deleteFileGraph(ctx context.Context, tx *sql.Tx, fileID int64) error { - rows, err := tx.QueryContext(ctx, `SELECT id FROM symbols WHERE file_id = ?`, fileID) - if err != nil { - return err - } - var symbolIDs []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - _ = rows.Close() - return err - } - symbolIDs = append(symbolIDs, id) - } - _ = rows.Close() - - deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, `DELETE FROM symbol_tokens WHERE symbol_id = ?`) - if err != nil { - return err - } - defer deleteSymbolTokensStmt.Close() - - deleteSymbolFTSStmt, err := tx.PrepareContext(ctx, `DELETE FROM symbol_fts WHERE symbol_id = ?`) - if err != nil { - return err - } - defer deleteSymbolFTSStmt.Close() - - for _, symbolID := range symbolIDs { - if _, err := deleteSymbolTokensStmt.ExecContext(ctx, symbolID); err != nil { - return err - } - if _, err := deleteSymbolFTSStmt.ExecContext(ctx, symbolID); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `DELETE FROM edges WHERE file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM references_tbl WHERE file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM file_imports WHERE file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM file_tokens WHERE file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM test_links WHERE test_file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM symbol_embeddings WHERE file_id = ?`, fileID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM symbols WHERE file_id = ?`, fileID); err != nil { - return err - } return nil } @@ -1271,6 +1352,14 @@ func (s *Store) resolveUniqueSymbolsByNames(ctx context.Context, repoID int64, n } func (s *Store) resolveSymbolsByStableKeys(ctx context.Context, repoID int64, stableKeys []string) (map[string]int64, error) { + return resolveSymbolsByStableKeysQuery(ctx, s.db, repoID, stableKeys) +} + +type queryContexter interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +func resolveSymbolsByStableKeysQuery(ctx context.Context, q queryContexter, repoID int64, stableKeys []string) (map[string]int64, error) { out := map[string]int64{} if len(stableKeys) == 0 { return out, nil @@ -1286,7 +1375,7 @@ func (s *Store) resolveSymbolsByStableKeys(ctx context.Context, repoID int64, st for _, key := range stableKeys { args = append(args, key) } - rows, err := s.db.QueryContext(ctx, query, args...) + rows, err := q.QueryContext(ctx, query, args...) if err != nil { return nil, err } From 28e4e1b9356b8860fdef92aab487fbd357b3e51b Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:12:28 +0200 Subject: [PATCH 02/59] =?UTF-8?q?perf(index):=20add=20phase=20profiling=20?= =?UTF-8?q?+=20reduce=20ReplaceFileGraphsBatch=20over=E2=80=A6=20(#21)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(index): add phase profiling + reduce ReplaceFileGraphsBatch overhead (RETURNING id, prepared deletes) + add delete-path index * perf: batch file-graph deletes and add supporting file_id indexes --- internal/cli/app.go | 18 ++- internal/cli/commands.go | 2 + internal/indexer/indexer.go | 43 ++++++ .../schema/008_file_graph_delete_indexes.sql | 28 ++++ internal/store/store.go | 122 ++++++++++++------ 5 files changed, 176 insertions(+), 37 deletions(-) create mode 100644 internal/store/schema/008_file_graph_delete_indexes.sql diff --git a/internal/cli/app.go b/internal/cli/app.go index 9d4bafa..cabf3cf 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -456,6 +456,7 @@ func runInstall(stdout io.Writer) error { func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName string, args []string, update bool) error { fs := flag.NewFlagSet(cmdName, flag.ContinueOnError) fs.SetOutput(io.Discard) + force := fs.Bool("force", false, "re-index files even if unchanged") jsonl := false filtered := make([]string, 0, len(args)) for _, arg := range args { @@ -478,7 +479,7 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName } defer app.Close() _ = repo - opts := indexer.Options{RepoRoot: repo.RootPath} + opts := indexer.Options{RepoRoot: repo.RootPath, Force: *force} var summary store.ScanSummary if update { opts.ScanKind = "update" @@ -502,6 +503,21 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName }); err != nil { return err } + _ = writeJSONL(stdout, map[string]any{ + "type": "scan_phases", + "data": map[string]any{ + "existing_load_ms": summary.ExistingLoadMS, + "walk_ms": summary.WalkMS, + "process_wall_ms": summary.ProcessWallMS, + "write_ms": summary.WriteMS, + "write_metadata_ms": summary.WriteMetadataMS, + "write_replace_ms": summary.WriteReplaceMS, + "embed_ms": summary.EmbedMS, + "mark_missing_ms": summary.MarkMissingMS, + "resolve_ms": summary.ResolveMS, + "duration_ms": summary.DurationMS, + }, + }) return writeJSONL(stdout, map[string]any{ "type": "scan_stats", "data": stats, diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 484656c..a8071a9 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -119,6 +119,7 @@ func newCommandList() []*command { description: "index a repository", usageLines: []string{" index "}, flags: []commandFlag{ + {name: "--force", description: "re-index files even if unchanged"}, {name: "--jsonl", description: "stream line-delimited JSON events"}, }, examples: []string{ @@ -135,6 +136,7 @@ func newCommandList() []*command { description: "update only changed files", usageLines: []string{" update_graph "}, flags: []commandFlag{ + {name: "--force", description: "re-index files even if unchanged"}, {name: "--jsonl", description: "stream line-delimited JSON events"}, }, examples: []string{ diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 822b971..346d4f5 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -55,6 +55,9 @@ type fileResult struct { err error parseErr string processMS int64 + readMS int64 + hashMS int64 + parseMS int64 } func New(s *store.Store, registry *parser.Registry, embedder embedding.Embedder) *Indexer { @@ -121,6 +124,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } var existing map[string]store.FileRecord + existingLoadStarted := time.Now() if len(candidateSet) > 0 { paths := make([]string, 0, len(candidateSet)) for rel := range candidateSet { @@ -130,6 +134,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } else { existing, err = i.store.ExistingFiles(ctx, repo.ID) } + summary.ExistingLoadMS = time.Since(existingLoadStarted).Milliseconds() if err != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) return summary, err @@ -235,13 +240,19 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err replaceBatch := make([]store.ReplaceFileGraphInput, 0, replaceBatchSize) changedPathSet := make(map[string]struct{}, 64) + var writeMetadataMS int64 + var writeReplaceMS int64 + var embedMS int64 + flushMarkSeen := func() error { if len(markSeenBatch) == 0 { return nil } + started := time.Now() if err := i.store.MarkFilesSeenBatch(ctx, repo.ID, scanID, markSeenBatch); err != nil { return err } + writeMetadataMS += time.Since(started).Milliseconds() markSeenBatch = markSeenBatch[:0] return nil } @@ -249,9 +260,11 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if len(touchBatch) == 0 { return nil } + started := time.Now() if err := i.store.TouchFilesMetadataBatch(ctx, repo.ID, scanID, touchBatch); err != nil { return err } + writeMetadataMS += time.Since(started).Milliseconds() touchBatch = touchBatch[:0] return nil } @@ -259,9 +272,11 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if len(parseFailedBatch) == 0 { return nil } + started := time.Now() if err := i.store.MarkFilesParseFailedBatch(ctx, repo.ID, scanID, parseFailedBatch); err != nil { return err } + writeMetadataMS += time.Since(started).Milliseconds() parseFailedBatch = parseFailedBatch[:0] return nil } @@ -269,11 +284,14 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if len(replaceBatch) == 0 { return nil } + started := time.Now() fileIDs, err := i.store.ReplaceFileGraphsBatch(ctx, repo.ID, scanID, replaceBatch) if err != nil { return err } + writeReplaceMS += time.Since(started).Milliseconds() if !embedding.IsNoop(i.embedder) { + embedStarted := time.Now() for idx, input := range replaceBatch { fileID := int64(0) if idx < len(fileIDs) { @@ -281,6 +299,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } i.embedFileSymbolsWithFileID(ctx, repo.ID, fileID, input.Parsed) } + embedMS += time.Since(embedStarted).Milliseconds() } replaceBatch = replaceBatch[:0] return nil @@ -288,9 +307,16 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err var runErr error var parseMS int64 + var readMS int64 + var hashMS int64 + var adapterParseMS int64 var writeMS int64 + processWallStart := time.Now() for res := range results { parseMS += res.processMS + readMS += res.readMS + hashMS += res.hashMS + adapterParseMS += res.parseMS summary.FilesSeen++ coverageLanguage := coverageKey(res.task.language) coverage := summary.LanguageCoverage[coverageLanguage] @@ -438,8 +464,15 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err walkErr := <-producerErr summary.WalkMS = time.Since(walkStart).Milliseconds() + summary.ProcessWallMS = time.Since(processWallStart).Milliseconds() summary.ParseMS = parseMS + summary.ReadMS = readMS + summary.HashMS = hashMS + summary.AdapterParseMS = adapterParseMS summary.WriteMS = writeMS + summary.WriteMetadataMS = writeMetadataMS + summary.WriteReplaceMS = writeReplaceMS + summary.EmbedMS = embedMS if runErr == nil && walkErr != nil && walkErr != context.Canceled { runErr = walkErr } @@ -448,11 +481,13 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, runErr } + missingStarted := time.Now() deleted, err := i.store.MarkMissingDeleted(ctx, repo.ID, scanID) if err != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) return summary, err } + summary.MarkMissingMS = time.Since(missingStarted).Milliseconds() summary.FilesDeleted = deleted resolveStart := time.Now() if len(candidateSet) > 0 { @@ -511,18 +546,24 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, var content []byte var err error if task.adapter == nil { + hashStarted := time.Now() hash, err = hashFile(task.path) if err != nil { result.err = err return result } + result.hashMS = time.Since(hashStarted).Milliseconds() } else { + readStarted := time.Now() content, err = os.ReadFile(task.path) if err != nil { result.err = err return result } + result.readMS = time.Since(readStarted).Milliseconds() + hashStarted := time.Now() hash = hashContent(content) + result.hashMS = time.Since(hashStarted).Milliseconds() } result.hash = hash if hasPrev && !force && prev.ContentHash == hash { @@ -532,7 +573,9 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, parsed := graph.ParsedFile{Language: task.language} if task.adapter != nil { + parseStarted := time.Now() parsed, err = task.adapter.Parse(ctx, task.path, content) + result.parseMS = time.Since(parseStarted).Milliseconds() if err != nil { if parseErrorPolicy == "best_effort" { result.action = "parse_failed" diff --git a/internal/store/schema/008_file_graph_delete_indexes.sql b/internal/store/schema/008_file_graph_delete_indexes.sql new file mode 100644 index 0000000..1d3940f --- /dev/null +++ b/internal/store/schema/008_file_graph_delete_indexes.sql @@ -0,0 +1,28 @@ +-- Speed up per-file graph deletion during re-index/update. +-- These paths are hot for "changed file" indexing where we delete old graph rows by file_id. + +CREATE INDEX IF NOT EXISTS idx_edges_file_id +ON edges(file_id); + +CREATE INDEX IF NOT EXISTS idx_refs_file_id +ON references_tbl(file_id); + +CREATE INDEX IF NOT EXISTS idx_file_imports_file_id +ON file_imports(file_id); + +CREATE INDEX IF NOT EXISTS idx_test_links_test_file_id +ON test_links(test_file_id); + +-- These already exist in earlier migrations, but are essential for the batch delete path. +CREATE INDEX IF NOT EXISTS idx_symbols_file_id +ON symbols(file_id); + +CREATE INDEX IF NOT EXISTS idx_file_tokens_file_id +ON file_tokens(file_id); + +CREATE INDEX IF NOT EXISTS idx_symbol_embeddings_file +ON symbol_embeddings(file_id); + +-- Speed up edge resolution strategies that filter by (repo_id, name) and then by kind. +CREATE INDEX IF NOT EXISTS idx_symbols_repo_name_kind +ON symbols(repo_id, name, kind); diff --git a/internal/store/store.go b/internal/store/store.go index e0c638a..e446a70 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -66,9 +66,18 @@ type ScanSummary struct { ParseErrors int `json:"parse_errors,omitempty"` ParseSamples []string `json:"parse_samples,omitempty"` LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` + ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` WalkMS int64 `json:"walk_ms,omitempty"` + ProcessWallMS int64 `json:"process_wall_ms,omitempty"` ParseMS int64 `json:"parse_ms,omitempty"` + ReadMS int64 `json:"read_ms,omitempty"` + HashMS int64 `json:"hash_ms,omitempty"` + AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` WriteMS int64 `json:"write_ms,omitempty"` + WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` + WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` + EmbedMS int64 `json:"embed_ms,omitempty"` + MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` ResolveMS int64 `json:"resolve_ms,omitempty"` DurationMS int64 `json:"duration_ms"` } @@ -704,6 +713,7 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 last_scan_id = excluded.last_scan_id, indexed_at = excluded.indexed_at, is_deleted = 0 + RETURNING id `) if err != nil { _ = tx.Rollback() @@ -711,33 +721,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 } defer upsertFileStmt.Close() - selectFileIDStmt, err := tx.PrepareContext(ctx, `SELECT id FROM files WHERE repo_id = ? AND path = ?`) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer selectFileIDStmt.Close() - - deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, ` - DELETE FROM symbol_tokens - WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer deleteSymbolTokensStmt.Close() - - deleteSymbolFTSStmt, err := tx.PrepareContext(ctx, ` - DELETE FROM symbol_fts - WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id = ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer deleteSymbolFTSStmt.Close() - insertSymbolStmt, err := tx.PrepareContext(ctx, ` INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -812,20 +795,21 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 now := time.Now().UTC().Format(time.RFC3339) fileIDs := make([]int64, 0, len(inputs)) for _, input := range inputs { - if _, err := upsertFileStmt.ExecContext(ctx, repoID, input.Path, input.Language, input.SizeBytes, input.MtimeUnixNS, input.ContentHash, scanID, now); err != nil { - _ = tx.Rollback() - return nil, err - } var fileID int64 - if err := selectFileIDStmt.QueryRowContext(ctx, repoID, input.Path).Scan(&fileID); err != nil { - _ = tx.Rollback() - return nil, err - } - if err := deleteFileGraphWithStmts(ctx, tx, fileID, deleteSymbolTokensStmt, deleteSymbolFTSStmt); err != nil { + if err := upsertFileStmt.QueryRowContext(ctx, repoID, input.Path, input.Language, input.SizeBytes, input.MtimeUnixNS, input.ContentHash, scanID, now).Scan(&fileID); err != nil { _ = tx.Rollback() return nil, err } fileIDs = append(fileIDs, fileID) + } + + if err := deleteFileGraphsBatch(ctx, tx, fileIDs); err != nil { + _ = tx.Rollback() + return nil, err + } + + for idx, input := range inputs { + fileID := fileIDs[idx] if err := insertParsedFileGraph( ctx, tx, @@ -852,6 +836,72 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 return fileIDs, nil } +func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64) error { + if len(fileIDs) == 0 { + return nil + } + + const maxVars = 900 + + execInChunks := func(sqlPrefix, sqlSuffix string, ids []int64) error { + for start := 0; start < len(ids); start += maxVars { + end := start + maxVars + if end > len(ids) { + end = len(ids) + } + chunk := ids[start:end] + placeholders := strings.Repeat("?,", len(chunk)) + placeholders = strings.TrimSuffix(placeholders, ",") + query := sqlPrefix + placeholders + sqlSuffix + args := make([]any, 0, len(chunk)) + for _, id := range chunk { + args = append(args, id) + } + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return err + } + } + return nil + } + + // Dependent tables that reference symbols must be deleted before deleting symbols. + if err := execInChunks( + `DELETE FROM symbol_tokens WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (`, + `))`, + fileIDs, + ); err != nil { + return err + } + if err := execInChunks( + `DELETE FROM symbol_fts WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (`, + `))`, + fileIDs, + ); err != nil { + return err + } + + if err := execInChunks(`DELETE FROM edges WHERE file_id IN (`, `)`, fileIDs); err != nil { + return err + } + if err := execInChunks(`DELETE FROM references_tbl WHERE file_id IN (`, `)`, fileIDs); err != nil { + return err + } + if err := execInChunks(`DELETE FROM file_imports WHERE file_id IN (`, `)`, fileIDs); err != nil { + return err + } + if err := execInChunks(`DELETE FROM file_tokens WHERE file_id IN (`, `)`, fileIDs); err != nil { + return err + } + if err := execInChunks(`DELETE FROM test_links WHERE test_file_id IN (`, `)`, fileIDs); err != nil { + return err + } + if err := execInChunks(`DELETE FROM symbol_embeddings WHERE file_id IN (`, `)`, fileIDs); err != nil { + return err + } + + return execInChunks(`DELETE FROM symbols WHERE file_id IN (`, `)`, fileIDs) +} + func deleteFileGraph(ctx context.Context, tx *sql.Tx, fileID int64) error { deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, ` DELETE FROM symbol_tokens From d787242205e8e01c57291e44d8774bc22596ec2e Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:31:57 +0200 Subject: [PATCH 03/59] =?UTF-8?q?store:=20batch=20ResolveEdgesForPaths=20u?= =?UTF-8?q?pdates=20and=20add=20unresolved=20edge-nam=E2=80=A6=20(#22)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * store: batch ResolveEdgesForPaths updates and add unresolved edge-name index * store/schema: use partial unresolved-edge index; fix rows.Err handling --- .../schema/009_edge_resolution_indexes.sql | 5 + internal/store/store.go | 130 ++++++++++++++---- 2 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 internal/store/schema/009_edge_resolution_indexes.sql diff --git a/internal/store/schema/009_edge_resolution_indexes.sql b/internal/store/schema/009_edge_resolution_indexes.sql new file mode 100644 index 0000000..09c22c1 --- /dev/null +++ b/internal/store/schema/009_edge_resolution_indexes.sql @@ -0,0 +1,5 @@ +-- Supports ResolveEdges' multi-pass UPDATE strategies by tailoring the index to +-- the unresolved-edge working set (dst_symbol_id IS NULL). +CREATE INDEX IF NOT EXISTS idx_edges_repo_unresolved_name +ON edges(repo_id, dst_name) +WHERE dst_symbol_id IS NULL; diff --git a/internal/store/store.go b/internal/store/store.go index e446a70..90143d2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1184,19 +1184,26 @@ func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths [] if len(paths) == 0 { return nil } - const chunkSize = 300 - targets := make([]edgeTarget, 0, len(paths)) - seen := map[int64]struct{}{} - for start := 0; start < len(paths); start += chunkSize { - end := min(start+chunkSize, len(paths)) - chunk := paths[start:end] + uniquePaths := make([]string, 0, len(paths)) + seenPaths := make(map[string]struct{}, len(paths)) + for _, path := range paths { + if path == "" { + continue + } + if _, ok := seenPaths[path]; ok { + continue + } + seenPaths[path] = struct{}{} + uniquePaths = append(uniquePaths, path) + } + + const chunkSize = 400 + fileIDs := make([]int64, 0, len(uniquePaths)) + for start := 0; start < len(uniquePaths); start += chunkSize { + end := min(start+chunkSize, len(uniquePaths)) + chunk := uniquePaths[start:end] placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") - query := ` - SELECT e.id, e.dst_name - FROM edges e - JOIN files f ON f.id = e.file_id - WHERE e.repo_id = ? AND e.dst_symbol_id IS NULL AND f.path IN (` + placeholders + `) - ` + query := `SELECT id FROM files WHERE repo_id = ? AND path IN (` + placeholders + `)` args := make([]any, 0, len(chunk)+1) args = append(args, repoID) for _, path := range chunk { @@ -1206,17 +1213,43 @@ func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths [] if err != nil { return err } - chunkTargets, err := scanEdgeTargets(rows) + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return err + } + fileIDs = append(fileIDs, id) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + } + + targets := make([]edgeTarget, 0, len(fileIDs)) + for start := 0; start < len(fileIDs); start += chunkSize { + end := min(start+chunkSize, len(fileIDs)) + chunk := fileIDs[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := `SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND file_id IN (` + placeholders + `)` + args := make([]any, 0, len(chunk)+1) + args = append(args, repoID) + for _, id := range chunk { + args = append(args, id) + } + rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { return err } - for _, target := range chunkTargets { - if _, ok := seen[target.edgeID]; ok { - continue - } - seen[target.edgeID] = struct{}{} - targets = append(targets, target) + chunkTargets, err := scanEdgeTargets(rows) + if err != nil { + return err } + targets = append(targets, chunkTargets...) } return s.resolveEdgeTargets(ctx, repoID, targets) } @@ -1276,17 +1309,15 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] return err } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err + type edgeResolution struct { + edgeID int64 + dstID int64 } - updateStmt, err := tx.PrepareContext(ctx, `UPDATE edges SET dst_symbol_id = ? WHERE id = ?`) + resolutions := make([]edgeResolution, 0, len(targets)) + tx, err := s.db.BeginTx(ctx, nil) if err != nil { - _ = tx.Rollback() return err } - defer updateStmt.Close() - for _, target := range targets { dstID, ok := byQualified[target.dstName] if !ok { @@ -1297,11 +1328,56 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] if !ok || dstID == 0 { continue } - if _, err := updateStmt.ExecContext(ctx, dstID, target.edgeID); err != nil { + resolutions = append(resolutions, edgeResolution{edgeID: target.edgeID, dstID: dstID}) + } + + if len(resolutions) == 0 { + _ = tx.Rollback() + return nil + } + + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_edge_resolution(edge_id INTEGER PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + _ = tx.Rollback() + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_edge_resolution`); err != nil { + _ = tx.Rollback() + return err + } + + // Keep well under SQLite's default variable limit (999). + const maxPairsPerInsert = 400 + for start := 0; start < len(resolutions); start += maxPairsPerInsert { + end := min(start+maxPairsPerInsert, len(resolutions)) + chunk := resolutions[start:end] + var b strings.Builder + b.WriteString(`INSERT INTO tmp_edge_resolution(edge_id, dst_symbol_id) VALUES `) + args := make([]any, 0, len(chunk)*2) + for i, r := range chunk { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?)") + args = append(args, r.edgeID, r.dstID) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { _ = tx.Rollback() return err } } + + if _, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_edge_resolution t WHERE t.edge_id = edges.id) + WHERE edges.id IN (SELECT edge_id FROM tmp_edge_resolution) + `); err != nil { + _ = tx.Rollback() + return err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_edge_resolution`); err != nil { + _ = tx.Rollback() + return err + } if err := tx.Commit(); err != nil { return err } From 663cc774a16b2e645ecf7f2da8c240833c7e306f Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:53:15 +0200 Subject: [PATCH 04/59] =?UTF-8?q?perf:=20speed=20up=20ResolveEdges=20suffi?= =?UTF-8?q?x=20strategy=20and=20path-scoped=20updates;=20=E2=80=A6=20(#23)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: speed up ResolveEdges suffix strategy and path-scoped updates; add optional sqlite_cgo driver --- internal/indexer/indexer.go | 109 +++++++++++++--- internal/indexer/indexer_bench_test.go | 2 + internal/store/sqlite_driver_cgo.go | 9 ++ internal/store/sqlite_driver_info.go | 7 ++ internal/store/sqlite_driver_modernc.go | 9 ++ internal/store/store.go | 160 +++++++++++++++++++++++- internal/store/store_bench_test.go | 8 ++ 7 files changed, 279 insertions(+), 25 deletions(-) create mode 100644 internal/store/sqlite_driver_cgo.go create mode 100644 internal/store/sqlite_driver_info.go create mode 100644 internal/store/sqlite_driver_modernc.go diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 346d4f5..fe28811 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -122,15 +122,15 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err candidateSet[filepath.Clean(rel)] = struct{}{} } } + candidatePaths := make([]string, 0, len(candidateSet)) + for rel := range candidateSet { + candidatePaths = append(candidatePaths, rel) + } var existing map[string]store.FileRecord existingLoadStarted := time.Now() if len(candidateSet) > 0 { - paths := make([]string, 0, len(candidateSet)) - for rel := range candidateSet { - paths = append(paths, rel) - } - existing, err = i.store.ExistingFilesForPaths(ctx, repo.ID, paths) + existing, err = i.store.ExistingFilesForPaths(ctx, repo.ID, candidatePaths) } else { existing, err = i.store.ExistingFiles(ctx, repo.ID) } @@ -155,9 +155,55 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err results := make(chan fileResult, workerCount*2) producerErr := make(chan error, 1) walkStart := time.Now() + missingCandidatePaths := make([]string, 0, 16) go func() { defer close(tasks) + if len(candidateSet) > 0 { + for _, rel := range candidatePaths { + rel = filepath.Clean(rel) + if shouldIgnorePath(rel, opts.Exclude) { + continue + } + if shouldSkipFile(rel, opts.Include, opts.Exclude) { + continue + } + abs := filepath.Join(opts.RepoRoot, rel) + info, err := os.Stat(abs) + if err != nil { + if os.IsNotExist(err) { + missingCandidatePaths = append(missingCandidatePaths, rel) + continue + } + producerErr <- err + return + } + if info.IsDir() { + continue + } + adapter := i.registry.AdapterFor(rel) + language := "" + if adapter != nil { + language = adapter.Language() + } + task := fileTask{ + path: abs, + rel: rel, + info: info, + adapter: adapter, + language: language, + } + select { + case tasks <- task: + case <-ctxRun.Done(): + producerErr <- ctxRun.Err() + return + } + } + producerErr <- nil + return + } + producerErr <- filepath.WalkDir(opts.RepoRoot, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -176,11 +222,6 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if d.IsDir() { return nil } - if len(candidateSet) > 0 { - if _, ok := candidateSet[rel]; !ok { - return nil - } - } if shouldSkipFile(rel, opts.Include, opts.Exclude) { return nil } @@ -481,16 +522,29 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, runErr } - missingStarted := time.Now() - deleted, err := i.store.MarkMissingDeleted(ctx, repo.ID, scanID) - if err != nil { - _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) - return summary, err + if len(candidateSet) == 0 { + missingStarted := time.Now() + deleted, err := i.store.MarkMissingDeleted(ctx, repo.ID, scanID) + if err != nil { + _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) + return summary, err + } + summary.MarkMissingMS = time.Since(missingStarted).Milliseconds() + summary.FilesDeleted = deleted } - summary.MarkMissingMS = time.Since(missingStarted).Milliseconds() - summary.FilesDeleted = deleted resolveStart := time.Now() - if len(candidateSet) > 0 { + if len(missingCandidatePaths) > 0 { + deleted, err := i.store.MarkFilesDeletedBatch(ctx, repo.ID, scanID, missingCandidatePaths) + if err != nil { + _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) + return summary, err + } + summary.FilesDeleted += deleted + } + + if len(changedPathSet) == 0 { + summary.ResolveMS = 0 + } else if len(candidateSet) > 0 { changedPaths := make([]string, 0, len(changedPathSet)) for path := range changedPathSet { changedPaths = append(changedPaths, path) @@ -505,7 +559,9 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, resolveErr } } - summary.ResolveMS = time.Since(resolveStart).Milliseconds() + if len(changedPathSet) > 0 { + summary.ResolveMS = time.Since(resolveStart).Milliseconds() + } summary.DurationMS = time.Since(started).Milliseconds() summary.FilesTotal = summary.FilesSeen + summary.FilesDeleted if summary.FilesTotal > 0 { @@ -652,6 +708,21 @@ func shouldSkipFile(rel string, includes, excludes []string) bool { return !matchesAny(rel, includes) } +func shouldIgnorePath(rel string, excludes []string) bool { + current := filepath.Clean(rel) + for current != "." && current != "" { + if shouldSkipDir(current, excludes) { + return true + } + next := filepath.Dir(current) + if next == current { + break + } + current = next + } + return false +} + func matchesAny(path string, globs []string) bool { path = filepath.ToSlash(path) for _, glob := range globs { diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index d64ec72..15f57fc 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -17,6 +17,7 @@ func BenchmarkIndexerIndex(b *testing.B) { repoRoot := b.TempDir() createGoFixtureRepo(b, repoRoot, 80) dbDir := b.TempDir() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -40,6 +41,7 @@ func BenchmarkIndexerUpdateOneFile(b *testing.B) { repoRoot := b.TempDir() createGoFixtureRepo(b, repoRoot, 80) dbPath := filepath.Join(b.TempDir(), "bench-update.sqlite") + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) s, err := store.Open(dbPath) if err != nil { diff --git a/internal/store/sqlite_driver_cgo.go b/internal/store/sqlite_driver_cgo.go new file mode 100644 index 0000000..6a54809 --- /dev/null +++ b/internal/store/sqlite_driver_cgo.go @@ -0,0 +1,9 @@ +//go:build sqlite_cgo && cgo + +package store + +import ( + _ "github.com/mattn/go-sqlite3" +) + +const sqliteDriverName = "sqlite3" diff --git a/internal/store/sqlite_driver_info.go b/internal/store/sqlite_driver_info.go new file mode 100644 index 0000000..8fc2ac0 --- /dev/null +++ b/internal/store/sqlite_driver_info.go @@ -0,0 +1,7 @@ +package store + +// SQLiteDriverName reports the registered database/sql driver name used by this build. +// This is useful for benchmarks comparing the pure-Go and CGO-backed driver variants. +func SQLiteDriverName() string { + return sqliteDriverName +} diff --git a/internal/store/sqlite_driver_modernc.go b/internal/store/sqlite_driver_modernc.go new file mode 100644 index 0000000..722b144 --- /dev/null +++ b/internal/store/sqlite_driver_modernc.go @@ -0,0 +1,9 @@ +//go:build !sqlite_cgo + +package store + +import ( + _ "modernc.org/sqlite" +) + +const sqliteDriverName = "sqlite" diff --git a/internal/store/store.go b/internal/store/store.go index 90143d2..8752b7d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -18,8 +18,6 @@ import ( "strings" "time" - _ "modernc.org/sqlite" - "github.com/isink17/codegraph/internal/graph" "github.com/isink17/codegraph/internal/texttoken" ) @@ -139,7 +137,7 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, err } - db, err := sql.Open("sqlite", path) + db, err := sql.Open(sqliteDriverName, path) if err != nil { return nil, err } @@ -1091,6 +1089,39 @@ func (s *Store) MarkMissingDeleted(ctx context.Context, repoID, scanID int64) (i return int(n), nil } +func (s *Store) MarkFilesDeletedBatch(ctx context.Context, repoID, scanID int64, paths []string) (int, error) { + if len(paths) == 0 { + return 0, nil + } + const chunkSize = 400 + total := int64(0) + for start := 0; start < len(paths); start += chunkSize { + end := min(start+chunkSize, len(paths)) + chunk := paths[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := ` + UPDATE files + SET is_deleted = 1, parse_state = 'deleted', last_scan_id = ? + WHERE repo_id = ? AND is_deleted = 0 AND path IN (` + placeholders + `) + ` + args := make([]any, 0, len(chunk)+2) + args = append(args, scanID, repoID) + for _, path := range chunk { + args = append(args, path) + } + res, err := s.db.ExecContext(ctx, query, args...) + if err != nil { + return int(total), err + } + affected, err := res.RowsAffected() + if err != nil { + return int(total), err + } + total += affected + } + return int(total), nil +} + type edgeTarget struct { edgeID int64 dstName string @@ -1134,18 +1165,28 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { totalResolved += int(n) } - // Strategy 3: Suffix match (e.g., pkg.Func matches github.com/org/repo/pkg.Func) + // Strategy 3a: Suffix match for slash-qualified symbols (e.g., pkg.Func matches github.com/org/repo/pkg.Func). + // Avoid per-edge LIKE scans by precomputing (tail -> symbol_id) once, then doing an indexed equality update. + n, err := s.resolveEdgesBySlashSuffix(ctx, repoID) + if err != nil { + return totalResolved, err + } + totalResolved += n + + // Strategy 3b: Dot-suffix fallback (for qualified names without a slash separator). + // Kept as a narrower fallback to preserve existing semantics across languages that don't use '/' in qualified names. res, err = s.db.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s WHERE s.repo_id = ? - AND (s.qualified_name LIKE '%.' || edges.dst_name OR s.qualified_name LIKE '%/' || edges.dst_name) + AND s.qualified_name LIKE '%.' || edges.dst_name LIMIT 1 ) WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND instr(edges.dst_name, '.') > 0 AND instr(edges.dst_name, '/') = 0 AND EXISTS ( SELECT 1 FROM symbols s WHERE s.repo_id = ? - AND (s.qualified_name LIKE '%.' || edges.dst_name OR s.qualified_name LIKE '%/' || edges.dst_name) + AND s.qualified_name LIKE '%.' || edges.dst_name ) `, repoID, repoID, repoID) if err != nil { @@ -1180,6 +1221,113 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { return totalResolved, nil } +func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, repoID int64) (int, error) { + rows, err := s.db.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) + if err != nil { + return 0, err + } + type resolvedSuffix struct { + dstName string + dstID int64 + } + // Map: suffix (after last '/') -> minimum symbol id. Using MIN keeps behavior stable when multiple symbols share the same suffix. + minBySuffix := map[string]int64{} + for rows.Next() { + var id int64 + var qname string + if err := rows.Scan(&id, &qname); err != nil { + _ = rows.Close() + return 0, err + } + if qname == "" { + continue + } + slash := strings.LastIndexByte(qname, '/') + if slash < 0 || slash+1 >= len(qname) { + continue + } + suffix := strings.Clone(qname[slash+1:]) + if suffix == "" { + continue + } + if cur, ok := minBySuffix[suffix]; !ok || id < cur { + minBySuffix[suffix] = id + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, err + } + if err := rows.Close(); err != nil { + return 0, err + } + if len(minBySuffix) == 0 { + return 0, nil + } + + resolved := make([]resolvedSuffix, 0, len(minBySuffix)) + for dstName, dstID := range minBySuffix { + resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + _ = tx.Rollback() + return 0, err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { + _ = tx.Rollback() + return 0, err + } + + // Keep well under SQLite's default variable limit (999). + const maxPairsPerInsert = 400 + for start := 0; start < len(resolved); start += maxPairsPerInsert { + end := min(start+maxPairsPerInsert, len(resolved)) + chunk := resolved[start:end] + var b strings.Builder + b.WriteString(`INSERT INTO tmp_symbol_slash_suffix(dst_name, dst_symbol_id) VALUES `) + args := make([]any, 0, len(chunk)*2) + for i, r := range chunk { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?)") + args = append(args, r.dstName, r.dstID) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + _ = tx.Rollback() + return 0, err + } + } + + updateRes, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_slash_suffix t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) + `, repoID) + if err != nil { + _ = tx.Rollback() + return 0, err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { + _ = tx.Rollback() + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err + } + n, err := updateRes.RowsAffected() + if err != nil { + return 0, err + } + return int(n), nil +} + func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths []string) error { if len(paths) == 0 { return nil diff --git a/internal/store/store_bench_test.go b/internal/store/store_bench_test.go index 833fa76..827a7fe 100644 --- a/internal/store/store_bench_test.go +++ b/internal/store/store_bench_test.go @@ -15,6 +15,7 @@ import ( func BenchmarkStoreSearchSymbols(b *testing.B) { ctx := context.Background() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) s, repoID := setupStoreBenchData(b, ctx) b.ResetTimer() @@ -31,8 +32,15 @@ func BenchmarkStoreSearchSymbols(b *testing.B) { func BenchmarkStoreSemanticSearch(b *testing.B) { ctx := context.Background() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) s, repoID := setupStoreBenchData(b, ctx) + // SemanticSearch may be disabled/unavailable depending on build/config (e.g. embeddings not enabled). + // Skip rather than failing the whole benchmark suite. + if items, err := s.SemanticSearch(ctx, repoID, "BenchFn", 20, 0); err != nil || len(items) == 0 { + b.Skip("SemanticSearch unavailable or returned no results") + } + b.ResetTimer() for i := 0; i < b.N; i++ { items, err := s.SemanticSearch(ctx, repoID, "BenchFn", 20, 0) From d81759771adf725ed218c645993671d67fc72ecf Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:32:53 +0200 Subject: [PATCH 05/59] =?UTF-8?q?perf(store):=20add=20Phase=201=20pragmas,?= =?UTF-8?q?=20batch=20MarkFilesSeen=20updates,=20and=20in=E2=80=A6=20(#24)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(store): add Phase 1 pragmas, batch MarkFilesSeen updates, and index active-file language stats * fix(store): avoid partial ResolveEdges counts on rollback; apply auto_vacuum only for new DBs --- .../store/schema/010_files_active_indexes.sql | 8 ++ internal/store/store.go | 97 +++++++++++-------- 2 files changed, 67 insertions(+), 38 deletions(-) create mode 100644 internal/store/schema/010_files_active_indexes.sql diff --git a/internal/store/schema/010_files_active_indexes.sql b/internal/store/schema/010_files_active_indexes.sql new file mode 100644 index 0000000..f3a2462 --- /dev/null +++ b/internal/store/schema/010_files_active_indexes.sql @@ -0,0 +1,8 @@ +-- Speeds up common large-repo queries and scan bookkeeping that filter out deleted files. +-- Partial index keeps bloat low while supporting: +-- - WHERE repo_id = ? AND is_deleted = 0 +-- - GROUP BY language for active files +CREATE INDEX IF NOT EXISTS idx_files_repo_active_lang +ON files(repo_id, language) +WHERE is_deleted = 0; + diff --git a/internal/store/store.go b/internal/store/store.go index 8752b7d..ec688bf 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -137,11 +137,21 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, err } + isNewDB := false + if st, err := os.Stat(path); err != nil { + if errors.Is(err, fs.ErrNotExist) { + isNewDB = true + } else { + return nil, err + } + } else if st.Size() == 0 { + isNewDB = true + } db, err := sql.Open(sqliteDriverName, path) if err != nil { return nil, err } - if err := applyPragmas(db, opts.PerformanceProfile); err != nil { + if err := applyPragmas(db, isNewDB, opts.PerformanceProfile); err != nil { return nil, err } s := &Store{db: db} @@ -151,11 +161,21 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { return s, nil } -func applyPragmas(db *sql.DB, profile string) error { +func applyPragmas(db *sql.DB, isNewDB bool, profile string) error { base := []string{ `PRAGMA journal_mode = WAL;`, `PRAGMA busy_timeout = 5000;`, `PRAGMA foreign_keys = ON;`, + `PRAGMA cache_size = -65536;`, + } + if isNewDB { + // auto_vacuum is only reliably applied for a brand-new DB before any tables are created. + // For existing DBs, switching auto_vacuum requires VACUUM; we avoid doing that implicitly. + base = append(base, `PRAGMA auto_vacuum = INCREMENTAL;`) + + // page_size is only reliably applied for a brand-new DB before any tables are created. + // For existing DBs, changing page_size requires VACUUM; we avoid doing that implicitly. + base = append(base, `PRAGMA page_size = 8192;`) } perf := strings.ToLower(strings.TrimSpace(profile)) switch perf { @@ -560,22 +580,26 @@ func (s *Store) MarkFilesSeenBatch(ctx context.Context, repoID, scanID int64, pa if len(paths) == 0 { return nil } + const chunkSize = 400 tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } - stmt, err := tx.PrepareContext(ctx, ` - UPDATE files - SET last_scan_id = ?, is_deleted = 0 - WHERE repo_id = ? AND path = ? - `) - if err != nil { - _ = tx.Rollback() - return err - } - defer stmt.Close() - for _, path := range paths { - if _, err := stmt.ExecContext(ctx, scanID, repoID, path); err != nil { + for start := 0; start < len(paths); start += chunkSize { + end := min(start+chunkSize, len(paths)) + chunk := paths[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := ` + UPDATE files + SET last_scan_id = ?, is_deleted = 0 + WHERE repo_id = ? AND path IN (` + placeholders + `) + ` + args := make([]any, 0, len(chunk)+2) + args = append(args, scanID, repoID) + for _, path := range chunk { + args = append(args, path) + } + if _, err := tx.ExecContext(ctx, query, args...); err != nil { _ = tx.Rollback() return err } @@ -1129,9 +1153,16 @@ type edgeTarget struct { func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { totalResolved := 0 + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer func() { + _ = tx.Rollback() + }() // Strategy 1: Exact qualified name match - res, err := s.db.ExecContext(ctx, ` + res, err := tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s WHERE s.repo_id = ? AND s.qualified_name = edges.dst_name @@ -1148,7 +1179,7 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { } // Strategy 2: Name match (unqualified) - res, err = s.db.ExecContext(ctx, ` + res, err = tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s WHERE s.repo_id = ? AND s.name = edges.dst_name @@ -1159,7 +1190,7 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { AND EXISTS (SELECT 1 FROM symbols s WHERE s.repo_id = ? AND s.name = edges.dst_name AND s.kind IN ('function', 'method', 'class', 'type', 'struct', 'interface')) `, repoID, repoID, repoID) if err != nil { - return totalResolved, err + return 0, err } if n, _ := res.RowsAffected(); n > 0 { totalResolved += int(n) @@ -1167,15 +1198,15 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { // Strategy 3a: Suffix match for slash-qualified symbols (e.g., pkg.Func matches github.com/org/repo/pkg.Func). // Avoid per-edge LIKE scans by precomputing (tail -> symbol_id) once, then doing an indexed equality update. - n, err := s.resolveEdgesBySlashSuffix(ctx, repoID) + n, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID) if err != nil { - return totalResolved, err + return 0, err } totalResolved += n // Strategy 3b: Dot-suffix fallback (for qualified names without a slash separator). // Kept as a narrower fallback to preserve existing semantics across languages that don't use '/' in qualified names. - res, err = s.db.ExecContext(ctx, ` + res, err = tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s WHERE s.repo_id = ? @@ -1190,14 +1221,14 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { ) `, repoID, repoID, repoID) if err != nil { - return totalResolved, err + return 0, err } if n, _ := res.RowsAffected(); n > 0 { totalResolved += int(n) } // Strategy 4: Method receiver match (e.g., DoSomething matches MyStruct.DoSomething) - res, err = s.db.ExecContext(ctx, ` + res, err = tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s WHERE s.repo_id = ? @@ -1212,17 +1243,20 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { ) `, repoID, repoID, repoID) if err != nil { - return totalResolved, err + return 0, err } if n, _ := res.RowsAffected(); n > 0 { totalResolved += int(n) } + if err := tx.Commit(); err != nil { + return 0, err + } return totalResolved, nil } -func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, repoID int64) (int, error) { - rows, err := s.db.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) +func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { + rows, err := tx.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) if err != nil { return 0, err } @@ -1269,17 +1303,10 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, repoID int64) (in for dstName, dstID := range minBySuffix { resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) } - - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return 0, err - } if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { - _ = tx.Rollback() return 0, err } if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { - _ = tx.Rollback() return 0, err } @@ -1299,7 +1326,6 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, repoID int64) (in args = append(args, r.dstName, r.dstID) } if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { - _ = tx.Rollback() return 0, err } } @@ -1311,14 +1337,9 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, repoID int64) (in AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) `, repoID) if err != nil { - _ = tx.Rollback() return 0, err } if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { - _ = tx.Rollback() - return 0, err - } - if err := tx.Commit(); err != nil { return 0, err } n, err := updateRes.RowsAffected() From 7f86a15debbc054b00cce1aa33747cc4aaced9ef Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:55:08 +0200 Subject: [PATCH 06/59] =?UTF-8?q?perf(index/store):=20batch=20token=20+=20?= =?UTF-8?q?embedding=20writes;=20make=20index=20benches=E2=80=A6=20(#25)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(index/store): batch token + embedding writes; make index benches scale via CODEGRAPH_BENCH_FILES * Update internal/store/store.go * refactor(store): replace sqlite batch-size literals with named constants --- internal/indexer/indexer.go | 74 +++++- internal/indexer/indexer_bench_test.go | 19 +- internal/store/store.go | 335 +++++++++++++++++++++---- internal/store/store_bench_test.go | 89 +++++++ 4 files changed, 464 insertions(+), 53 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index fe28811..26cee49 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -333,13 +333,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err writeReplaceMS += time.Since(started).Milliseconds() if !embedding.IsNoop(i.embedder) { embedStarted := time.Now() - for idx, input := range replaceBatch { - fileID := int64(0) - if idx < len(fileIDs) { - fileID = fileIDs[idx] - } - i.embedFileSymbolsWithFileID(ctx, repo.ID, fileID, input.Parsed) - } + i.embedReplaceBatch(ctx, repo.ID, fileIDs, replaceBatch) embedMS += time.Since(embedStarted).Milliseconds() } replaceBatch = replaceBatch[:0] @@ -867,3 +861,69 @@ func (i *Indexer) embedFileSymbolsWithFileID(ctx context.Context, repoID int64, _ = i.store.UpsertSymbolEmbeddings(ctx, repoID, fileID, "", symbolMap) } } + +func (i *Indexer) embedReplaceBatch(ctx context.Context, repoID int64, fileIDs []int64, inputs []store.ReplaceFileGraphInput) { + type keyRef struct { + fileID int64 + stableKey string + } + + totalSyms := 0 + for idx := range inputs { + if idx >= len(fileIDs) { + break + } + totalSyms += len(inputs[idx].Parsed.Symbols) + } + if totalSyms == 0 { + return + } + + texts := make([]string, 0, totalSyms) + keys := make([]keyRef, 0, totalSyms) + for idx, input := range inputs { + if idx >= len(fileIDs) { + break + } + fileID := fileIDs[idx] + if fileID == 0 || len(input.Parsed.Symbols) == 0 { + continue + } + for _, sym := range input.Parsed.Symbols { + texts = append(texts, embedding.FormatSymbolText(sym.Kind, sym.QualifiedName, sym.Signature, sym.DocSummary)) + keys = append(keys, keyRef{fileID: fileID, stableKey: sym.StableKey}) + } + } + if len(texts) == 0 { + return + } + + const embedChunkSize = 128 + upserts := make([]store.SymbolEmbeddingUpsert, 0, len(texts)) + for start := 0; start < len(texts); start += embedChunkSize { + end := min(start+embedChunkSize, len(texts)) + vectors, err := i.embedder.EmbedBatch(ctx, texts[start:end]) + if err != nil { + return // best-effort: don't fail indexing on embedding errors + } + for j := range vectors { + vec := vectors[j] + if vec == nil { + continue + } + ref := keys[start+j] + if ref.fileID == 0 || ref.stableKey == "" { + continue + } + upserts = append(upserts, store.SymbolEmbeddingUpsert{ + FileID: ref.fileID, + StableKey: ref.stableKey, + Vector: vec, + }) + } + } + if len(upserts) == 0 { + return + } + _ = i.store.UpsertSymbolEmbeddingsBatch(ctx, repoID, "", upserts) +} diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index 15f57fc..b008ed9 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "testing" "github.com/isink17/codegraph/internal/parser" @@ -15,9 +16,16 @@ import ( func BenchmarkIndexerIndex(b *testing.B) { ctx := context.Background() repoRoot := b.TempDir() - createGoFixtureRepo(b, repoRoot, 80) + files := 80 + if v := os.Getenv("CODEGRAPH_BENCH_FILES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + files = n + } + } + createGoFixtureRepo(b, repoRoot, files) dbDir := b.TempDir() b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("fixture_files=%d", files) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -39,9 +47,16 @@ func BenchmarkIndexerIndex(b *testing.B) { func BenchmarkIndexerUpdateOneFile(b *testing.B) { ctx := context.Background() repoRoot := b.TempDir() - createGoFixtureRepo(b, repoRoot, 80) + files := 80 + if v := os.Getenv("CODEGRAPH_BENCH_FILES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + files = n + } + } + createGoFixtureRepo(b, repoRoot, files) dbPath := filepath.Join(b.TempDir(), "bench-update.sqlite") b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("fixture_files=%d", files) s, err := store.Open(dbPath) if err != nil { diff --git a/internal/store/store.go b/internal/store/store.go index ec688bf..2077cbf 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -25,6 +25,24 @@ import ( //go:embed schema/*.sql var migrationFS embed.FS +const ( + // sqliteDefaultMaxVariables is SQLite's commonly configured parameter limit (often 999). + // Keep batch sizes below this to avoid "too many SQL variables" errors. + sqliteDefaultMaxVariables = 999 + + // sqliteInClauseBatchSize is a conservative IN-clause chunk size used for set-based deletes/updates. + // It stays under sqliteDefaultMaxVariables with room for any additional parameters. + sqliteInClauseBatchSize = 900 + + // sqliteTokenValuesBatchRows controls multi-row inserts into token tables where each row uses 3 parameters. + // 300*3=900 variables, staying under sqliteDefaultMaxVariables. + sqliteTokenValuesBatchRows = 300 + + // sqliteEmbeddingValuesBatchRows controls multi-row upserts into symbol_embeddings where each row uses 7 parameters. + // 100*7=700 variables, staying under sqliteDefaultMaxVariables. + sqliteEmbeddingValuesBatchRows = 100 +) + type Store struct { db *sql.DB } @@ -763,13 +781,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 } defer insertSymbolFTSStmt.Close() - insertSymbolTokenStmt, err := tx.PrepareContext(ctx, `INSERT INTO symbol_tokens(symbol_id, token, weight) VALUES(?, ?, ?)`) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertSymbolTokenStmt.Close() - insertReferenceStmt, err := tx.PrepareContext(ctx, ` INSERT INTO references_tbl(repo_id, file_id, symbol_id, ref_kind, name, qualified_name, start_line, start_col, end_line, end_col, context_symbol_id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -797,13 +808,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 } defer insertImportStmt.Close() - insertFileTokenStmt, err := tx.PrepareContext(ctx, `INSERT INTO file_tokens(file_id, token, weight) VALUES(?, ?, ?)`) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertFileTokenStmt.Close() - insertTestLinkStmt, err := tx.PrepareContext(ctx, ` INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score) VALUES(?, ?, ?, ?, ?, ?) @@ -840,11 +844,9 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 input.Parsed, insertSymbolStmt, insertSymbolFTSStmt, - insertSymbolTokenStmt, insertReferenceStmt, insertEdgeStmt, insertImportStmt, - insertFileTokenStmt, insertTestLinkStmt, ); err != nil { _ = tx.Rollback() @@ -863,11 +865,9 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64) err return nil } - const maxVars = 900 - execInChunks := func(sqlPrefix, sqlSuffix string, ids []int64) error { - for start := 0; start < len(ids); start += maxVars { - end := start + maxVars + for start := 0; start < len(ids); start += sqliteInClauseBatchSize { + end := start + sqliteInClauseBatchSize if end > len(ids) { end = len(ids) } @@ -985,14 +985,13 @@ func insertParsedFileGraph( parsed graph.ParsedFile, insertSymbolStmt *sql.Stmt, insertSymbolFTSStmt *sql.Stmt, - insertSymbolTokenStmt *sql.Stmt, insertReferenceStmt *sql.Stmt, insertEdgeStmt *sql.Stmt, insertImportStmt *sql.Stmt, - insertFileTokenStmt *sql.Stmt, insertTestLinkStmt *sql.Stmt, ) error { stableToID := map[string]int64{} + symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) for _, sym := range parsed.Symbols { res, err := insertSymbolStmt.ExecContext(ctx, repoID, fileID, sym.Language, sym.Kind, sym.Name, sym.QualifiedName, sym.ContainerName, sym.Signature, sym.Visibility, sym.Range.StartLine, sym.Range.StartCol, sym.Range.EndLine, sym.Range.EndCol, sym.DocSummary, sym.StableKey) if err != nil { @@ -1007,11 +1006,20 @@ func insertParsedFileGraph( return err } for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { - if _, err := insertSymbolTokenStmt.ExecContext(ctx, symbolID, token, weight); err != nil { - return err + symbolTokenArgs = append(symbolTokenArgs, symbolID, token, weight) + if len(symbolTokenArgs) >= sqliteTokenValuesBatchRows*3 { + if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs); err != nil { + return err + } + symbolTokenArgs = symbolTokenArgs[:0] } } } + if len(symbolTokenArgs) > 0 { + if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs); err != nil { + return err + } + } contextSymbolID := firstFunctionID(stableToID, parsed.Symbols) for _, ref := range parsed.References { var symbolID any @@ -1040,9 +1048,21 @@ func insertParsedFileGraph( return err } } - for token, weight := range parsed.FileTokens { - if _, err := insertFileTokenStmt.ExecContext(ctx, fileID, token, weight); err != nil { - return err + if len(parsed.FileTokens) > 0 { + fileTokenArgs := make([]any, 0, min(len(parsed.FileTokens), sqliteTokenValuesBatchRows)*3) + for token, weight := range parsed.FileTokens { + fileTokenArgs = append(fileTokenArgs, fileID, token, weight) + if len(fileTokenArgs) >= sqliteTokenValuesBatchRows*3 { + if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs); err != nil { + return err + } + fileTokenArgs = fileTokenArgs[:0] + } + } + if len(fileTokenArgs) > 0 { + if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs); err != nil { + return err + } } } @@ -1076,6 +1096,27 @@ func insertParsedFileGraph( return nil } +func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn string, args []any) error { + if len(args) == 0 { + return nil + } + // Each row uses 3 params; batch sizes are controlled by sqliteTokenValuesBatchRows to stay under sqliteDefaultMaxVariables. + if len(args)%3 != 0 { + return fmt.Errorf("invalid token insert args len=%d", len(args)) + } + rows := len(args) / 3 + var b strings.Builder + fmt.Fprintf(&b, "INSERT INTO %s(%s, token, weight) VALUES ", table, idColumn) + for i := 0; i < rows; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?,?)") + } + _, err := tx.ExecContext(ctx, b.String(), args...) + return err +} + func firstFunctionID(stableToID map[string]int64, symbols []graph.Symbol) int64 { for _, sym := range symbols { if sym.Kind == "function" { @@ -2868,32 +2909,238 @@ func (s *Store) UpsertSymbolEmbeddings(ctx context.Context, repoID, fileID int64 return err } now := time.Now().UTC().Format(time.RFC3339) + stableKeys := make([]string, 0, len(symbolMap)) + for k := range symbolMap { + stableKeys = append(stableKeys, k) + } - stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO symbol_embeddings(symbol_id, file_id, repo_id, embedding, dimensions, model_name, updated_at) - SELECT s.id, ?, ?, ?, ?, ?, ? - FROM symbols s - WHERE s.file_id = ? AND s.stable_key = ? - ON CONFLICT(symbol_id) DO UPDATE SET - embedding = excluded.embedding, - dimensions = excluded.dimensions, - model_name = excluded.model_name, - updated_at = excluded.updated_at - `) - if err != nil { - _ = tx.Rollback() - return err + // Resolve symbol ids in chunks to avoid per-row subqueries during embedding writes. + // Keep under sqliteDefaultMaxVariables. + stableToID := make(map[string]int64, len(stableKeys)) + for start := 0; start < len(stableKeys); start += sqliteInClauseBatchSize { + end := min(start+sqliteInClauseBatchSize, len(stableKeys)) + chunk := stableKeys[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := `SELECT stable_key, id FROM symbols WHERE file_id = ? AND stable_key IN (` + placeholders + `)` + args := make([]any, 0, len(chunk)+1) + args = append(args, fileID) + for _, k := range chunk { + args = append(args, k) + } + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + _ = tx.Rollback() + return err + } + for rows.Next() { + var stableKey string + var id int64 + if err := rows.Scan(&stableKey, &id); err != nil { + _ = rows.Close() + _ = tx.Rollback() + return err + } + stableToID[stableKey] = id + } + if err := rows.Err(); err != nil { + _ = rows.Close() + _ = tx.Rollback() + return err + } + if err := rows.Close(); err != nil { + _ = tx.Rollback() + return err + } + } + + const embedCols = 7 + embedArgs := make([]any, 0, sqliteEmbeddingValuesBatchRows*embedCols) + flush := func() error { + if len(embedArgs) == 0 { + return nil + } + rows := len(embedArgs) / embedCols + var b strings.Builder + b.WriteString(`INSERT INTO symbol_embeddings(symbol_id, file_id, repo_id, embedding, dimensions, model_name, updated_at) VALUES `) + for i := 0; i < rows; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?,?,?,?,?,?)") + } + b.WriteString(` + ON CONFLICT(symbol_id) DO UPDATE SET + embedding = excluded.embedding, + dimensions = excluded.dimensions, + model_name = excluded.model_name, + updated_at = excluded.updated_at + `) + if _, err := tx.ExecContext(ctx, b.String(), embedArgs...); err != nil { + return err + } + embedArgs = embedArgs[:0] + return nil } - defer stmt.Close() for stableKey, vec := range symbolMap { + symbolID := stableToID[stableKey] + if symbolID == 0 { + continue + } blob := float32ToBytes(vec) - if _, err := stmt.ExecContext(ctx, fileID, repoID, blob, len(vec), modelName, now, fileID, stableKey); err != nil { - _ = tx.Rollback() + embedArgs = append(embedArgs, symbolID, fileID, repoID, blob, len(vec), modelName, now) + if len(embedArgs) >= sqliteEmbeddingValuesBatchRows*embedCols { + if err := flush(); err != nil { + _ = tx.Rollback() + return err + } + } + } + if err := flush(); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} + +type SymbolEmbeddingUpsert struct { + FileID int64 + StableKey string + Vector []float32 +} + +func (s *Store) UpsertSymbolEmbeddingsBatch(ctx context.Context, repoID int64, modelName string, items []SymbolEmbeddingUpsert) error { + if len(items) == 0 { + return nil + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err != nil { + return err + } + now := time.Now().UTC().Format(time.RFC3339) + type vecRef struct { + stableKey string + vector []float32 + } + byFile := map[int64][]vecRef{} + for _, item := range items { + if item.FileID == 0 || item.StableKey == "" || len(item.Vector) == 0 { + continue + } + byFile[item.FileID] = append(byFile[item.FileID], vecRef{stableKey: item.StableKey, vector: item.Vector}) + } + if len(byFile) == 0 { + _ = tx.Rollback() + return nil + } + + const embedCols = 7 + embedArgs := make([]any, 0, sqliteEmbeddingValuesBatchRows*embedCols) + flush := func() error { + if len(embedArgs) == 0 { + return nil + } + rows := len(embedArgs) / embedCols + var b strings.Builder + b.WriteString(`INSERT INTO symbol_embeddings(symbol_id, file_id, repo_id, embedding, dimensions, model_name, updated_at) VALUES `) + for i := 0; i < rows; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?,?,?,?,?,?)") + } + b.WriteString(` + ON CONFLICT(symbol_id) DO UPDATE SET + embedding = excluded.embedding, + dimensions = excluded.dimensions, + model_name = excluded.model_name, + updated_at = excluded.updated_at + `) + if _, err := tx.ExecContext(ctx, b.String(), embedArgs...); err != nil { return err } + embedArgs = embedArgs[:0] + return nil } + // Resolve symbol ids per file in chunks, then multi-row upsert embeddings. + // Keep well under sqliteDefaultMaxVariables. + for fileID, vecs := range byFile { + if len(vecs) == 0 { + continue + } + stableToVec := make(map[string][]float32, len(vecs)) + stableKeys := make([]string, 0, len(vecs)) + for _, v := range vecs { + if v.stableKey == "" || len(v.vector) == 0 { + continue + } + // If the same stable key appears multiple times, the last one wins; vectors should be identical anyway. + if _, ok := stableToVec[v.stableKey]; !ok { + stableKeys = append(stableKeys, v.stableKey) + } + stableToVec[v.stableKey] = v.vector + } + for start := 0; start < len(stableKeys); start += sqliteInClauseBatchSize { + end := min(start+sqliteInClauseBatchSize, len(stableKeys)) + chunk := stableKeys[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := `SELECT stable_key, id FROM symbols WHERE file_id = ? AND stable_key IN (` + placeholders + `)` + args := make([]any, 0, len(chunk)+1) + args = append(args, fileID) + for _, k := range chunk { + args = append(args, k) + } + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + _ = tx.Rollback() + return err + } + for rows.Next() { + var stableKey string + var symbolID int64 + if err := rows.Scan(&stableKey, &symbolID); err != nil { + _ = rows.Close() + _ = tx.Rollback() + return err + } + vec := stableToVec[stableKey] + if symbolID == 0 || len(vec) == 0 { + continue + } + embedArgs = append(embedArgs, symbolID, fileID, repoID, float32ToBytes(vec), len(vec), modelName, now) + if len(embedArgs) >= sqliteEmbeddingValuesBatchRows*embedCols { + if err := rows.Close(); err != nil { + _ = tx.Rollback() + return err + } + if err := flush(); err != nil { + _ = tx.Rollback() + return err + } + continue + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + _ = tx.Rollback() + return err + } + if err := rows.Close(); err != nil { + _ = tx.Rollback() + return err + } + } + } + if err := flush(); err != nil { + _ = tx.Rollback() + return err + } return tx.Commit() } diff --git a/internal/store/store_bench_test.go b/internal/store/store_bench_test.go index 827a7fe..0d52c5e 100644 --- a/internal/store/store_bench_test.go +++ b/internal/store/store_bench_test.go @@ -5,8 +5,10 @@ import ( "fmt" "os" "path/filepath" + "strconv" "testing" + "github.com/isink17/codegraph/internal/graph" "github.com/isink17/codegraph/internal/indexer" "github.com/isink17/codegraph/internal/parser" goparser "github.com/isink17/codegraph/internal/parser/golang" @@ -53,6 +55,37 @@ func BenchmarkStoreSemanticSearch(b *testing.B) { } } +func BenchmarkStoreReplaceFileGraphWriteHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-write-heavy.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + parsed := heavyParsedFile(80, 3, 2) + path := "bench/heavy.go" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.ReplaceFileGraph(ctx, repo.ID, 1, path, "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil { + b.Fatalf("ReplaceFileGraph() error = %v", err) + } + } +} + func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64) { b.Helper() repoRoot := b.TempDir() @@ -82,3 +115,59 @@ func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64 } return s, repo.ID } + +func sqliteBenchProfile() string { + // Keep benchmarks repeatable without requiring CLI wiring; override with CODEGRAPH_BENCH_SQLITE_PROFILE. + if v := os.Getenv("CODEGRAPH_BENCH_SQLITE_PROFILE"); v != "" { + return v + } + return "balanced" +} + +func heavyParsedFile(symbols, refsPerSymbol, edgesPerSymbol int) graph.ParsedFile { + out := graph.ParsedFile{ + Language: "go", + } + out.Symbols = make([]graph.Symbol, 0, symbols) + out.References = make([]graph.Reference, 0, symbols*refsPerSymbol) + out.Edges = make([]graph.Edge, 0, symbols*edgesPerSymbol) + + for i := 0; i < symbols; i++ { + name := fmt.Sprintf("BenchHeavyFn%d", i) + qname := fmt.Sprintf("bench.%s", name) + stable := fmt.Sprintf("go:%s:%d", qname, i) + out.Symbols = append(out.Symbols, graph.Symbol{ + Language: "go", + Kind: "function", + Name: name, + QualifiedName: qname, + Range: graph.Position{StartLine: i + 1, StartCol: 1, EndLine: i + 1, EndCol: 10}, + StableKey: stable, + }) + + for r := 0; r < refsPerSymbol; r++ { + out.References = append(out.References, graph.Reference{ + Kind: "call", + Name: name, + Range: graph.Position{ + StartLine: i + 1, + StartCol: 1 + r, + EndLine: i + 1, + EndCol: 2 + r, + }, + }) + } + + for e := 0; e < edgesPerSymbol; e++ { + dst := fmt.Sprintf("bench.BenchHeavyFn%d", (i+e+1)%symbols) + out.Edges = append(out.Edges, graph.Edge{ + DstName: dst, + Kind: "calls", + Evidence: "bench", + Line: i + 1, + }) + } + } + + return out +} From a8e9fcc70e5ab5101e0a1b460cd118eaefabf582 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 22:17:44 +0200 Subject: [PATCH 07/59] =?UTF-8?q?perf(store):=20batch=20multi-row=20insert?= =?UTF-8?q?s=20for=20edges,=20references,=20and=20impor=E2=80=A6=20(#26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(store): batch multi-row inserts for edges, references, and imports during indexing * refactor(store): Extract common logic from batch insert helpers --- internal/store/store.go | 173 ++++++++++++++++++++++++++++------------ 1 file changed, 121 insertions(+), 52 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 2077cbf..1a58bc1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -41,6 +41,18 @@ const ( // sqliteEmbeddingValuesBatchRows controls multi-row upserts into symbol_embeddings where each row uses 7 parameters. // 100*7=700 variables, staying under sqliteDefaultMaxVariables. sqliteEmbeddingValuesBatchRows = 100 + + // sqliteReferenceValuesBatchRows controls multi-row inserts into references_tbl where each row uses 11 parameters. + // 90*11=990 variables, staying under sqliteDefaultMaxVariables. + sqliteReferenceValuesBatchRows = 90 + + // sqliteEdgeValuesBatchRows controls multi-row inserts into edges where each row uses 7 parameters. + // 140*7=980 variables, staying under sqliteDefaultMaxVariables. + sqliteEdgeValuesBatchRows = 140 + + // sqliteImportValuesBatchRows controls multi-row inserts into file_imports where each row uses 3 parameters. + // 300*3=900 variables, staying under sqliteDefaultMaxVariables. + sqliteImportValuesBatchRows = 300 ) type Store struct { @@ -781,33 +793,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 } defer insertSymbolFTSStmt.Close() - insertReferenceStmt, err := tx.PrepareContext(ctx, ` - INSERT INTO references_tbl(repo_id, file_id, symbol_id, ref_kind, name, qualified_name, start_line, start_col, end_line, end_col, context_symbol_id) - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertReferenceStmt.Close() - - insertEdgeStmt, err := tx.PrepareContext(ctx, ` - INSERT INTO edges(repo_id, src_symbol_id, dst_name, edge_kind, evidence, file_id, line) - VALUES(?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertEdgeStmt.Close() - - insertImportStmt, err := tx.PrepareContext(ctx, `INSERT INTO file_imports(repo_id, file_id, import_path) VALUES(?, ?, ?)`) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertImportStmt.Close() - insertTestLinkStmt, err := tx.PrepareContext(ctx, ` INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score) VALUES(?, ?, ?, ?, ?, ?) @@ -844,9 +829,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 input.Parsed, insertSymbolStmt, insertSymbolFTSStmt, - insertReferenceStmt, - insertEdgeStmt, - insertImportStmt, insertTestLinkStmt, ); err != nil { _ = tx.Rollback() @@ -985,9 +967,6 @@ func insertParsedFileGraph( parsed graph.ParsedFile, insertSymbolStmt *sql.Stmt, insertSymbolFTSStmt *sql.Stmt, - insertReferenceStmt *sql.Stmt, - insertEdgeStmt *sql.Stmt, - insertImportStmt *sql.Stmt, insertTestLinkStmt *sql.Stmt, ) error { stableToID := map[string]int64{} @@ -1021,31 +1000,82 @@ func insertParsedFileGraph( } } contextSymbolID := firstFunctionID(stableToID, parsed.Symbols) - for _, ref := range parsed.References { - var symbolID any - if ref.SymbolID != nil { - symbolID = *ref.SymbolID - } - var contextID any - if contextSymbolID != 0 { - contextID = contextSymbolID + if len(parsed.References) > 0 { + referenceArgs := make([]any, 0, min(len(parsed.References), sqliteReferenceValuesBatchRows)*11) + for _, ref := range parsed.References { + var symbolID any + if ref.SymbolID != nil { + symbolID = *ref.SymbolID + } + var contextID any + if contextSymbolID != 0 { + contextID = contextSymbolID + } + referenceArgs = append( + referenceArgs, + repoID, + fileID, + symbolID, + ref.Kind, + ref.Name, + ref.QualifiedName, + ref.Range.StartLine, + ref.Range.StartCol, + ref.Range.EndLine, + ref.Range.EndCol, + contextID, + ) + if len(referenceArgs) >= sqliteReferenceValuesBatchRows*11 { + if err := execReferencesInsert(ctx, tx, referenceArgs); err != nil { + return err + } + referenceArgs = referenceArgs[:0] + } } - if _, err := insertReferenceStmt.ExecContext(ctx, repoID, fileID, symbolID, ref.Kind, ref.Name, ref.QualifiedName, ref.Range.StartLine, ref.Range.StartCol, ref.Range.EndLine, ref.Range.EndCol, contextID); err != nil { - return err + if len(referenceArgs) > 0 { + if err := execReferencesInsert(ctx, tx, referenceArgs); err != nil { + return err + } } } - for _, edge := range parsed.Edges { - srcID := chooseSrcSymbolID(stableToID, parsed.Symbols, edge.Line) - if srcID == 0 { - continue + + if len(parsed.Edges) > 0 { + edgeArgs := make([]any, 0, min(len(parsed.Edges), sqliteEdgeValuesBatchRows)*7) + for _, edge := range parsed.Edges { + srcID := chooseSrcSymbolID(stableToID, parsed.Symbols, edge.Line) + if srcID == 0 { + continue + } + edgeArgs = append(edgeArgs, repoID, srcID, edge.DstName, edge.Kind, edge.Evidence, fileID, edge.Line) + if len(edgeArgs) >= sqliteEdgeValuesBatchRows*7 { + if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs); err != nil { + return err + } + edgeArgs = edgeArgs[:0] + } } - if _, err := insertEdgeStmt.ExecContext(ctx, repoID, srcID, edge.DstName, edge.Kind, edge.Evidence, fileID, edge.Line); err != nil { - return err + if len(edgeArgs) > 0 { + if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs); err != nil { + return err + } } } - for _, imp := range parsed.Imports { - if _, err := insertImportStmt.ExecContext(ctx, repoID, fileID, imp); err != nil { - return err + + if len(parsed.Imports) > 0 { + importArgs := make([]any, 0, min(len(parsed.Imports), sqliteImportValuesBatchRows)*3) + for _, imp := range parsed.Imports { + importArgs = append(importArgs, repoID, fileID, imp) + if len(importArgs) >= sqliteImportValuesBatchRows*3 { + if err := execImportsInsert(ctx, tx, importArgs); err != nil { + return err + } + importArgs = importArgs[:0] + } + } + if len(importArgs) > 0 { + if err := execImportsInsert(ctx, tx, importArgs); err != nil { + return err + } } } if len(parsed.FileTokens) > 0 { @@ -1117,6 +1147,45 @@ func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn str return err } +func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, rowsPerBatch int, args []any) error { + if len(args) == 0 { + return nil + } + if len(args)%rowsPerBatch != 0 { + return fmt.Errorf("invalid %s insert args len=%d (expected multiple of %d)", table, len(args), rowsPerBatch) + } + rowCount := len(args) / rowsPerBatch + var b strings.Builder + fmt.Fprintf(&b, "INSERT INTO %s(%s) VALUES ", table, columns) + for i := 0; i < rowCount; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(") + for j := 0; j < rowsPerBatch; j++ { + if j > 0 { + b.WriteString(",") + } + b.WriteString("?") + } + b.WriteString(")") + } + _, err := tx.ExecContext(ctx, b.String(), args...) + return err +} + +func execReferencesInsert(ctx context.Context, tx *sql.Tx, args []any) error { + return execBatchInsert(ctx, tx, "references_tbl", "repo_id, file_id, symbol_id, ref_kind, name, qualified_name, start_line, start_col, end_line, end_col, context_symbol_id", 11, args) +} + +func execUnresolvedEdgesInsert(ctx context.Context, tx *sql.Tx, args []any) error { + return execBatchInsert(ctx, tx, "edges", "repo_id, src_symbol_id, dst_name, edge_kind, evidence, file_id, line", 7, args) +} + +func execImportsInsert(ctx context.Context, tx *sql.Tx, args []any) error { + return execBatchInsert(ctx, tx, "file_imports", "repo_id, file_id, import_path", 3, args) +} + func firstFunctionID(stableToID map[string]int64, symbols []graph.Symbol) int64 { for _, sym := range symbols { if sym.Kind == "function" { From bf988eb4153a8649b96bbc9823d8641bc924cc35 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Wed, 22 Apr 2026 22:35:25 +0200 Subject: [PATCH 08/59] refactor(store): batch test_links inserts and cleanup prepared statements --- internal/store/store.go | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 1a58bc1..0e9aeb4 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -53,6 +53,9 @@ const ( // sqliteImportValuesBatchRows controls multi-row inserts into file_imports where each row uses 3 parameters. // 300*3=900 variables, staying under sqliteDefaultMaxVariables. sqliteImportValuesBatchRows = 300 + // sqliteTestLinkValuesBatchRows controls multi-row inserts into test_links where each row uses 6 parameters. + // 150*6=900 variables, staying under sqliteDefaultMaxVariables. + sqliteTestLinkValuesBatchRows = 150 ) type Store struct { @@ -793,16 +796,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 } defer insertSymbolFTSStmt.Close() - insertTestLinkStmt, err := tx.PrepareContext(ctx, ` - INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score) - VALUES(?, ?, ?, ?, ?, ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertTestLinkStmt.Close() - now := time.Now().UTC().Format(time.RFC3339) fileIDs := make([]int64, 0, len(inputs)) for _, input := range inputs { @@ -829,7 +822,6 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 input.Parsed, insertSymbolStmt, insertSymbolFTSStmt, - insertTestLinkStmt, ); err != nil { _ = tx.Rollback() return nil, err @@ -967,7 +959,6 @@ func insertParsedFileGraph( parsed graph.ParsedFile, insertSymbolStmt *sql.Stmt, insertSymbolFTSStmt *sql.Stmt, - insertTestLinkStmt *sql.Stmt, ) error { stableToID := map[string]int64{} symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) @@ -1110,17 +1101,29 @@ func insertParsedFileGraph( if err != nil { return err } - for _, link := range parsed.TestLinks { - var testSymbolID any - var targetSymbolID any - if id := stableToID[link.TestSymbolKey]; id != 0 { - testSymbolID = id - } - if id, ok := targetStableToID[link.TargetStableKey]; ok { - targetSymbolID = id + if len(parsed.TestLinks) > 0 { + testLinkArgs := make([]any, 0, min(len(parsed.TestLinks), sqliteTestLinkValuesBatchRows)*6) + for _, link := range parsed.TestLinks { + var testSymbolID any + var targetSymbolID any + if id := stableToID[link.TestSymbolKey]; id != 0 { + testSymbolID = id + } + if id, ok := targetStableToID[link.TargetStableKey]; ok { + targetSymbolID = id + } + testLinkArgs = append(testLinkArgs, repoID, fileID, testSymbolID, targetSymbolID, link.Reason, link.Score) + if len(testLinkArgs) >= sqliteTestLinkValuesBatchRows*6 { + if err := execTestLinksInsert(ctx, tx, testLinkArgs); err != nil { + return err + } + testLinkArgs = testLinkArgs[:0] + } } - if _, err := insertTestLinkStmt.ExecContext(ctx, repoID, fileID, testSymbolID, targetSymbolID, link.Reason, link.Score); err != nil { - return err + if len(testLinkArgs) > 0 { + if err := execTestLinksInsert(ctx, tx, testLinkArgs); err != nil { + return err + } } } return nil @@ -1182,6 +1185,10 @@ func execUnresolvedEdgesInsert(ctx context.Context, tx *sql.Tx, args []any) erro return execBatchInsert(ctx, tx, "edges", "repo_id, src_symbol_id, dst_name, edge_kind, evidence, file_id, line", 7, args) } +func execTestLinksInsert(ctx context.Context, tx *sql.Tx, args []any) error { + return execBatchInsert(ctx, tx, "test_links", "repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score", 6, args) +} + func execImportsInsert(ctx context.Context, tx *sql.Tx, args []any) error { return execBatchInsert(ctx, tx, "file_imports", "repo_id, file_id, import_path", 3, args) } From 9ce71a020766038bbfd38d05ea61496647c79b96 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 06:17:30 +0200 Subject: [PATCH 09/59] perf(indexing): add phase timings + write_stats counters; improve benchmark repeatability (#28) * perf(indexing): add phase timings + write_stats counters; improve benchmark repeatability * fix(indexer): use duration-based timing internally for observability --- internal/cli/app.go | 103 ++++++++++++- internal/cli/commands.go | 2 +- internal/indexer/indexer.go | 133 ++++++++++------ internal/indexer/indexer_bench_test.go | 13 +- internal/store/store.go | 206 ++++++++++++++++++------- 5 files changed, 346 insertions(+), 111 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index cabf3cf..db2600e 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -219,6 +219,13 @@ type benchmarkBaseline struct { Command []string `json:"command"` Count int `json:"count"` Benchtime string `json:"benchtime"` + GoVersion string `json:"go_version,omitempty"` + GOOS string `json:"goos,omitempty"` + GOARCH string `json:"goarch,omitempty"` + GOMAXPROCS string `json:"gomaxprocs,omitempty"` + SQLite string `json:"sqlite_driver,omitempty"` + BenchCtx map[string]any `json:"bench_ctx,omitempty"` + Env map[string]string `json:"env,omitempty"` Benchmarks map[string]benchmarkMetric `json:"benchmarks"` } @@ -277,12 +284,50 @@ func computeMetricDelta(current, baseline benchmarkMetric) map[string]any { return out } +func extractBenchmarkContext(output string) map[string]any { + // Keep this intentionally narrow: only pull a few high-signal markers that help + // interpret perf comparisons (fixture sizing, driver selection, etc.). + keys := []string{ + "fixture_files", + "sqlite_driver", + } + out := map[string]any{} + for line := range strings.SplitSeq(output, "\n") { + line = strings.TrimSpace(line) + for _, k := range keys { + needle := k + "=" + idx := strings.Index(line, needle) + if idx == -1 { + continue + } + raw := strings.TrimSpace(line[idx+len(needle):]) + if raw == "" { + continue + } + switch k { + case "fixture_files": + if n, err := strconv.Atoi(raw); err == nil { + out[k] = n + } + default: + out[k] = raw + } + } + } + if len(out) == 0 { + return nil + } + return out +} + func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { fs := flag.NewFlagSet("benchmark", flag.ContinueOnError) fs.SetOutput(io.Discard) count := fs.Int("count", 1, "number of benchmark runs") benchtime := fs.String("benchtime", "100ms", "benchmark time per test") saveBaseline := fs.Bool("save-baseline", true, "save current benchmark result as baseline") + files := fs.Int("files", 0, "fixture file count (sets CODEGRAPH_BENCH_FILES)") + gomaxprocs := fs.Int("gomaxprocs", 0, "GOMAXPROCS for benchmark subprocess (0 = default)") if err := fs.Parse(args); err != nil { return err } @@ -294,6 +339,7 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { "./internal/indexer", "./internal/store", "./internal/mcp", + "-v", "-run", "^$", "-bench", ".", "-benchmem", @@ -301,13 +347,44 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { "-benchtime", *benchtime, } cmd := exec.CommandContext(ctx, "go", cmdArgs...) - cmd.Env = append(os.Environ(), "GOCACHE="+filepath.Join(".", ".gocache")) + env := append([]string(nil), os.Environ()...) + gocachePath, err := filepath.Abs(filepath.Join(".", ".gocache")) + if err != nil { + return err + } + if err := os.MkdirAll(gocachePath, 0o755); err != nil { + return err + } + env = append(env, "GOCACHE="+gocachePath) + benchEnv := map[string]string{} + if *files > 0 { + v := strconv.Itoa(*files) + env = append(env, "CODEGRAPH_BENCH_FILES="+v) + benchEnv["CODEGRAPH_BENCH_FILES"] = v + } else if v := strings.TrimSpace(os.Getenv("CODEGRAPH_BENCH_FILES")); v != "" { + benchEnv["CODEGRAPH_BENCH_FILES"] = v + } + if *gomaxprocs > 0 { + v := strconv.Itoa(*gomaxprocs) + env = append(env, "GOMAXPROCS="+v) + benchEnv["GOMAXPROCS"] = v + } else if v := strings.TrimSpace(os.Getenv("GOMAXPROCS")); v != "" { + benchEnv["GOMAXPROCS"] = v + } + cmd.Env = env out, err := cmd.CombinedOutput() + benchCtx := extractBenchmarkContext(string(out)) result := map[string]any{ - "command": append([]string{"go"}, cmdArgs...), - "count": *count, - "benchtime": *benchtime, - "output": string(out), + "command": append([]string{"go"}, cmdArgs...), + "count": *count, + "benchtime": *benchtime, + "go_version": runtime.Version(), + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + "sqlite_driver": store.SQLiteDriverName(), + "env": benchEnv, + "bench_ctx": benchCtx, + "output": string(out), } if err != nil { result["ok"] = false @@ -339,11 +416,22 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { } if *saveBaseline { if mkdirErr := os.MkdirAll(filepath.Dir(baselinePath), 0o755); mkdirErr == nil { + gomaxLabel := benchEnv["GOMAXPROCS"] + if gomaxLabel == "" { + gomaxLabel = "default" + } payload := benchmarkBaseline{ CreatedAt: time.Now().UTC().Format(time.RFC3339), Command: append([]string{"go"}, cmdArgs...), Count: *count, Benchtime: *benchtime, + GoVersion: runtime.Version(), + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + GOMAXPROCS: gomaxLabel, + SQLite: store.SQLiteDriverName(), + BenchCtx: benchCtx, + Env: benchEnv, Benchmarks: parsed, } if data, marshalErr := json.MarshalIndent(payload, "", " "); marshalErr == nil { @@ -509,6 +597,11 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName "existing_load_ms": summary.ExistingLoadMS, "walk_ms": summary.WalkMS, "process_wall_ms": summary.ProcessWallMS, + "task_ms": summary.TaskMS, + "task_other_ms": summary.TaskOtherMS, + "read_ms": summary.ReadMS, + "hash_ms": summary.HashMS, + "adapter_parse_ms": summary.AdapterParseMS, "write_ms": summary.WriteMS, "write_metadata_ms": summary.WriteMetadataMS, "write_replace_ms": summary.WriteReplaceMS, diff --git a/internal/cli/commands.go b/internal/cli/commands.go index a8071a9..582dd18 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -276,7 +276,7 @@ func newCommandList() []*command { { name: "benchmark", description: "benchmarks", - usageLines: []string{" benchmark [--count N] [--benchtime DURATION] [--save-baseline]"}, + usageLines: []string{" benchmark [--count N] [--benchtime DURATION] [--save-baseline] [--files N] [--gomaxprocs N]"}, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { return runBenchmark(ctx, stdout, args) }, diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 26cee49..124e07e 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -48,16 +48,16 @@ type fileTask struct { } type fileResult struct { - task fileTask - action string - hash string - parsed graph.ParsedFile - err error - parseErr string - processMS int64 - readMS int64 - hashMS int64 - parseMS int64 + task fileTask + action string + hash string + parsed graph.ParsedFile + err error + parseErr string + processDur time.Duration + readDur time.Duration + hashDur time.Duration + parseDur time.Duration } func New(s *store.Store, registry *parser.Registry, embedder embedding.Embedder) *Indexer { @@ -281,9 +281,10 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err replaceBatch := make([]store.ReplaceFileGraphInput, 0, replaceBatchSize) changedPathSet := make(map[string]struct{}, 64) - var writeMetadataMS int64 - var writeReplaceMS int64 - var embedMS int64 + var writeMetadataDur time.Duration + var writeReplaceDur time.Duration + var embedDur time.Duration + var writeStats store.WriteStats flushMarkSeen := func() error { if len(markSeenBatch) == 0 { @@ -293,7 +294,8 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if err := i.store.MarkFilesSeenBatch(ctx, repo.ID, scanID, markSeenBatch); err != nil { return err } - writeMetadataMS += time.Since(started).Milliseconds() + summary.WriteMarkSeenFlushes++ + writeMetadataDur += time.Since(started) markSeenBatch = markSeenBatch[:0] return nil } @@ -305,7 +307,8 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if err := i.store.TouchFilesMetadataBatch(ctx, repo.ID, scanID, touchBatch); err != nil { return err } - writeMetadataMS += time.Since(started).Milliseconds() + summary.WriteTouchFlushes++ + writeMetadataDur += time.Since(started) touchBatch = touchBatch[:0] return nil } @@ -317,7 +320,8 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if err := i.store.MarkFilesParseFailedBatch(ctx, repo.ID, scanID, parseFailedBatch); err != nil { return err } - writeMetadataMS += time.Since(started).Milliseconds() + summary.WriteParseFailedFlushes++ + writeMetadataDur += time.Since(started) parseFailedBatch = parseFailedBatch[:0] return nil } @@ -326,32 +330,38 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return nil } started := time.Now() - fileIDs, err := i.store.ReplaceFileGraphsBatch(ctx, repo.ID, scanID, replaceBatch) + fileIDs, err := i.store.ReplaceFileGraphsBatchWithStats(ctx, repo.ID, scanID, replaceBatch, &writeStats) if err != nil { return err } - writeReplaceMS += time.Since(started).Milliseconds() + summary.WriteReplaceFlushes++ + writeReplaceDur += time.Since(started) if !embedding.IsNoop(i.embedder) { embedStarted := time.Now() i.embedReplaceBatch(ctx, repo.ID, fileIDs, replaceBatch) - embedMS += time.Since(embedStarted).Milliseconds() + embedDur += time.Since(embedStarted) } replaceBatch = replaceBatch[:0] return nil } var runErr error - var parseMS int64 - var readMS int64 - var hashMS int64 - var adapterParseMS int64 - var writeMS int64 + var taskDur time.Duration + var taskOtherDur time.Duration + var readDur time.Duration + var hashDur time.Duration + var adapterParseDur time.Duration + var writeDur time.Duration processWallStart := time.Now() for res := range results { - parseMS += res.processMS - readMS += res.readMS - hashMS += res.hashMS - adapterParseMS += res.parseMS + taskDur += res.processDur + readDur += res.readDur + hashDur += res.hashDur + adapterParseDur += res.parseDur + other := res.processDur - res.readDur - res.hashDur - res.parseDur + if other > 0 { + taskOtherDur += other + } summary.FilesSeen++ coverageLanguage := coverageKey(res.task.language) coverage := summary.LanguageCoverage[coverageLanguage] @@ -380,7 +390,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err coverage.Skipped++ summary.LanguageCoverage[coverageLanguage] = coverage if len(markSeenBatch) < metadataBatchSize { - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) summary.FilesSkipped++ continue } @@ -388,7 +398,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) summary.FilesSkipped++ case "touch": writeStart := time.Now() @@ -403,7 +413,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err coverage.Skipped++ summary.LanguageCoverage[coverageLanguage] = coverage if len(touchBatch) < metadataBatchSize { - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) summary.FilesSkipped++ continue } @@ -411,7 +421,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) summary.FilesSkipped++ case "replace": writeStart := time.Now() @@ -435,7 +445,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err cancel() } } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) case "parse_failed": writeStart := time.Now() parseFailedBatch = append(parseFailedBatch, store.FileMetadataUpdate{ @@ -455,7 +465,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err cancel() } } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) summary.FilesSkipped++ coverage := summary.LanguageCoverage[coverageLanguage] coverage.Skipped++ @@ -470,7 +480,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) } if runErr == nil { writeStart := time.Now() @@ -478,7 +488,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) } if runErr == nil { writeStart := time.Now() @@ -486,7 +496,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) } if runErr == nil { writeStart := time.Now() @@ -494,20 +504,25 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err runErr = err cancel() } - writeMS += time.Since(writeStart).Milliseconds() + writeDur += time.Since(writeStart) } walkErr := <-producerErr summary.WalkMS = time.Since(walkStart).Milliseconds() summary.ProcessWallMS = time.Since(processWallStart).Milliseconds() - summary.ParseMS = parseMS - summary.ReadMS = readMS - summary.HashMS = hashMS - summary.AdapterParseMS = adapterParseMS - summary.WriteMS = writeMS - summary.WriteMetadataMS = writeMetadataMS - summary.WriteReplaceMS = writeReplaceMS - summary.EmbedMS = embedMS + summary.TaskMS = taskDur.Milliseconds() + summary.TaskOtherMS = taskOtherDur.Milliseconds() + summary.ParseMS = taskDur.Milliseconds() + summary.ReadMS = readDur.Milliseconds() + summary.HashMS = hashDur.Milliseconds() + summary.AdapterParseMS = adapterParseDur.Milliseconds() + summary.WriteMS = writeDur.Milliseconds() + summary.WriteMetadataMS = writeMetadataDur.Milliseconds() + summary.WriteReplaceMS = writeReplaceDur.Milliseconds() + summary.EmbedMS = embedDur.Milliseconds() + if writeStats != (store.WriteStats{}) { + summary.WriteStats = &writeStats + } if runErr == nil && walkErr != nil && walkErr != context.Canceled { runErr = walkErr } @@ -557,6 +572,22 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err summary.ResolveMS = time.Since(resolveStart).Milliseconds() } summary.DurationMS = time.Since(started).Milliseconds() + summary.PhaseTimings = []store.ScanPhaseTiming{ + {Phase: "existing_load", MS: summary.ExistingLoadMS}, + {Phase: "walk", MS: summary.WalkMS}, + {Phase: "task", MS: summary.TaskMS}, + {Phase: "task_other", MS: summary.TaskOtherMS}, + {Phase: "read", MS: summary.ReadMS}, + {Phase: "hash", MS: summary.HashMS}, + {Phase: "adapter_parse", MS: summary.AdapterParseMS}, + {Phase: "write", MS: summary.WriteMS}, + {Phase: "write_metadata", MS: summary.WriteMetadataMS}, + {Phase: "write_replace", MS: summary.WriteReplaceMS}, + {Phase: "embed", MS: summary.EmbedMS}, + {Phase: "mark_missing", MS: summary.MarkMissingMS}, + {Phase: "resolve_edges", MS: summary.ResolveMS}, + {Phase: "total", MS: summary.DurationMS}, + } summary.FilesTotal = summary.FilesSeen + summary.FilesDeleted if summary.FilesTotal > 0 { summary.FilesDeletedPct = (float64(summary.FilesDeleted) / float64(summary.FilesTotal)) * 100 @@ -571,7 +602,7 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, result := fileResult{task: task} started := time.Now() defer func() { - result.processMS = time.Since(started).Milliseconds() + result.processDur = time.Since(started) }() hasPrev := prev.Path != "" @@ -602,7 +633,7 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, result.err = err return result } - result.hashMS = time.Since(hashStarted).Milliseconds() + result.hashDur = time.Since(hashStarted) } else { readStarted := time.Now() content, err = os.ReadFile(task.path) @@ -610,10 +641,10 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, result.err = err return result } - result.readMS = time.Since(readStarted).Milliseconds() + result.readDur = time.Since(readStarted) hashStarted := time.Now() hash = hashContent(content) - result.hashMS = time.Since(hashStarted).Milliseconds() + result.hashDur = time.Since(hashStarted) } result.hash = hash if hasPrev && !force && prev.ContentHash == hash { @@ -625,7 +656,7 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, if task.adapter != nil { parseStarted := time.Now() parsed, err = task.adapter.Parse(ctx, task.path, content) - result.parseMS = time.Since(parseStarted).Milliseconds() + result.parseDur = time.Since(parseStarted) if err != nil { if parseErrorPolicy == "best_effort" { result.action = "parse_failed" diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index b008ed9..8449443 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -25,13 +25,14 @@ func BenchmarkIndexerIndex(b *testing.B) { createGoFixtureRepo(b, repoRoot, files) dbDir := b.TempDir() b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", sqliteBenchProfile()) b.Logf("fixture_files=%d", files) b.ResetTimer() for i := 0; i < b.N; i++ { dbPath := filepath.Join(dbDir, "bench-index.sqlite") _ = os.Remove(dbPath) - s, err := store.Open(dbPath) + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()}) if err != nil { b.Fatalf("store.Open() error = %v", err) } @@ -56,9 +57,10 @@ func BenchmarkIndexerUpdateOneFile(b *testing.B) { createGoFixtureRepo(b, repoRoot, files) dbPath := filepath.Join(b.TempDir(), "bench-update.sqlite") b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", sqliteBenchProfile()) b.Logf("fixture_files=%d", files) - s, err := store.Open(dbPath) + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()}) if err != nil { b.Fatalf("store.Open() error = %v", err) } @@ -92,3 +94,10 @@ func createGoFixtureRepo(b *testing.B, repoRoot string, files int) { } } } + +func sqliteBenchProfile() string { + if v := os.Getenv("CODEGRAPH_BENCH_SQLITE_PROFILE"); v != "" { + return v + } + return "balanced" +} diff --git a/internal/store/store.go b/internal/store/store.go index 0e9aeb4..925f34b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -85,32 +85,77 @@ type FileMetadataUpdate struct { } type ScanSummary struct { - RepoID int64 `json:"repo_id"` - ScanID int64 `json:"scan_id"` - FilesSeen int `json:"files_seen"` - FilesIndexed int `json:"files_indexed"` - FilesSkipped int `json:"files_skipped"` - FilesChanged int `json:"files_changed"` - FilesDeleted int `json:"files_deleted"` - FilesTotal int `json:"files_total,omitempty"` - FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` - ParseErrors int `json:"parse_errors,omitempty"` - ParseSamples []string `json:"parse_samples,omitempty"` - LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` - ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` - WalkMS int64 `json:"walk_ms,omitempty"` - ProcessWallMS int64 `json:"process_wall_ms,omitempty"` - ParseMS int64 `json:"parse_ms,omitempty"` - ReadMS int64 `json:"read_ms,omitempty"` - HashMS int64 `json:"hash_ms,omitempty"` - AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` - WriteMS int64 `json:"write_ms,omitempty"` - WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` - WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` - EmbedMS int64 `json:"embed_ms,omitempty"` - MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` - ResolveMS int64 `json:"resolve_ms,omitempty"` - DurationMS int64 `json:"duration_ms"` + RepoID int64 `json:"repo_id"` + ScanID int64 `json:"scan_id"` + FilesSeen int `json:"files_seen"` + FilesIndexed int `json:"files_indexed"` + FilesSkipped int `json:"files_skipped"` + FilesChanged int `json:"files_changed"` + FilesDeleted int `json:"files_deleted"` + FilesTotal int `json:"files_total,omitempty"` + FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` + ParseErrors int `json:"parse_errors,omitempty"` + ParseSamples []string `json:"parse_samples,omitempty"` + LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` + PhaseTimings []ScanPhaseTiming `json:"phase_timings,omitempty"` + ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` + WalkMS int64 `json:"walk_ms,omitempty"` + ProcessWallMS int64 `json:"process_wall_ms,omitempty"` + TaskMS int64 `json:"task_ms,omitempty"` + TaskOtherMS int64 `json:"task_other_ms,omitempty"` + ParseMS int64 `json:"parse_ms,omitempty"` + ReadMS int64 `json:"read_ms,omitempty"` + HashMS int64 `json:"hash_ms,omitempty"` + AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` + WriteMS int64 `json:"write_ms,omitempty"` + WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` + WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` + WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` + WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` + WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` + WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` + WriteStats *WriteStats `json:"write_stats,omitempty"` + EmbedMS int64 `json:"embed_ms,omitempty"` + MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` + ResolveMS int64 `json:"resolve_ms,omitempty"` + DurationMS int64 `json:"duration_ms"` +} + +type ScanPhaseTiming struct { + Phase string `json:"phase"` + MS int64 `json:"ms"` +} + +type WriteStats struct { + TxCount int `json:"tx_count,omitempty"` + + FileUpsertStatements int `json:"file_upsert_statements,omitempty"` + + FileGraphDeleteChunks int `json:"file_graph_delete_chunks,omitempty"` + FileGraphDeleteStatements int `json:"file_graph_delete_statements,omitempty"` + + SymbolInserts int `json:"symbol_inserts,omitempty"` + SymbolFTSInserts int `json:"symbol_fts_inserts,omitempty"` + + SymbolTokenInsertBatches int `json:"symbol_token_insert_batches,omitempty"` + SymbolTokenInsertRows int `json:"symbol_token_insert_rows,omitempty"` + + FileTokenInsertBatches int `json:"file_token_insert_batches,omitempty"` + FileTokenInsertRows int `json:"file_token_insert_rows,omitempty"` + + ReferenceInsertBatches int `json:"reference_insert_batches,omitempty"` + ReferenceInsertRows int `json:"reference_insert_rows,omitempty"` + + EdgeInsertBatches int `json:"edge_insert_batches,omitempty"` + EdgeInsertRows int `json:"edge_insert_rows,omitempty"` + + ImportInsertBatches int `json:"import_insert_batches,omitempty"` + ImportInsertRows int `json:"import_insert_rows,omitempty"` + + TestLinkInsertBatches int `json:"test_link_insert_batches,omitempty"` + TestLinkInsertRows int `json:"test_link_insert_rows,omitempty"` + + TotalExecStatements int `json:"total_exec_statements,omitempty"` } type LanguageCounts struct { @@ -747,9 +792,17 @@ func (s *Store) ReplaceFileGraph(ctx context.Context, repoID, scanID int64, path } func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64, inputs []ReplaceFileGraphInput) ([]int64, error) { + return s.ReplaceFileGraphsBatchWithStats(ctx, repoID, scanID, inputs, nil) +} + +func (s *Store) ReplaceFileGraphsBatchWithStats(ctx context.Context, repoID, scanID int64, inputs []ReplaceFileGraphInput, stats *WriteStats) ([]int64, error) { if len(inputs) == 0 { return nil, nil } + if stats != nil { + stats.TxCount++ + stats.FileUpsertStatements += len(inputs) + } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return nil, err @@ -807,7 +860,7 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 fileIDs = append(fileIDs, fileID) } - if err := deleteFileGraphsBatch(ctx, tx, fileIDs); err != nil { + if err := deleteFileGraphsBatch(ctx, tx, fileIDs, stats); err != nil { _ = tx.Rollback() return nil, err } @@ -822,6 +875,7 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 input.Parsed, insertSymbolStmt, insertSymbolFTSStmt, + stats, ); err != nil { _ = tx.Rollback() return nil, err @@ -834,7 +888,7 @@ func (s *Store) ReplaceFileGraphsBatch(ctx context.Context, repoID, scanID int64 return fileIDs, nil } -func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64) error { +func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, stats *WriteStats) error { if len(fileIDs) == 0 { return nil } @@ -856,6 +910,11 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64) err if _, err := tx.ExecContext(ctx, query, args...); err != nil { return err } + if stats != nil { + stats.FileGraphDeleteChunks++ + stats.FileGraphDeleteStatements++ + stats.TotalExecStatements++ + } } return nil } @@ -959,6 +1018,7 @@ func insertParsedFileGraph( parsed graph.ParsedFile, insertSymbolStmt *sql.Stmt, insertSymbolFTSStmt *sql.Stmt, + stats *WriteStats, ) error { stableToID := map[string]int64{} symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) @@ -967,6 +1027,10 @@ func insertParsedFileGraph( if err != nil { return err } + if stats != nil { + stats.SymbolInserts++ + stats.TotalExecStatements++ + } symbolID, err := res.LastInsertId() if err != nil { return err @@ -975,10 +1039,14 @@ func insertParsedFileGraph( if _, err := insertSymbolFTSStmt.ExecContext(ctx, repoID, symbolID, sym.Name, sym.QualifiedName, sym.Signature, sym.DocSummary); err != nil { return err } + if stats != nil { + stats.SymbolFTSInserts++ + stats.TotalExecStatements++ + } for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { symbolTokenArgs = append(symbolTokenArgs, symbolID, token, weight) if len(symbolTokenArgs) >= sqliteTokenValuesBatchRows*3 { - if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs); err != nil { + if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { return err } symbolTokenArgs = symbolTokenArgs[:0] @@ -986,7 +1054,7 @@ func insertParsedFileGraph( } } if len(symbolTokenArgs) > 0 { - if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs); err != nil { + if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { return err } } @@ -1017,14 +1085,14 @@ func insertParsedFileGraph( contextID, ) if len(referenceArgs) >= sqliteReferenceValuesBatchRows*11 { - if err := execReferencesInsert(ctx, tx, referenceArgs); err != nil { + if err := execReferencesInsert(ctx, tx, referenceArgs, stats); err != nil { return err } referenceArgs = referenceArgs[:0] } } if len(referenceArgs) > 0 { - if err := execReferencesInsert(ctx, tx, referenceArgs); err != nil { + if err := execReferencesInsert(ctx, tx, referenceArgs, stats); err != nil { return err } } @@ -1039,14 +1107,14 @@ func insertParsedFileGraph( } edgeArgs = append(edgeArgs, repoID, srcID, edge.DstName, edge.Kind, edge.Evidence, fileID, edge.Line) if len(edgeArgs) >= sqliteEdgeValuesBatchRows*7 { - if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs); err != nil { + if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs, stats); err != nil { return err } edgeArgs = edgeArgs[:0] } } if len(edgeArgs) > 0 { - if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs); err != nil { + if err := execUnresolvedEdgesInsert(ctx, tx, edgeArgs, stats); err != nil { return err } } @@ -1057,14 +1125,14 @@ func insertParsedFileGraph( for _, imp := range parsed.Imports { importArgs = append(importArgs, repoID, fileID, imp) if len(importArgs) >= sqliteImportValuesBatchRows*3 { - if err := execImportsInsert(ctx, tx, importArgs); err != nil { + if err := execImportsInsert(ctx, tx, importArgs, stats); err != nil { return err } importArgs = importArgs[:0] } } if len(importArgs) > 0 { - if err := execImportsInsert(ctx, tx, importArgs); err != nil { + if err := execImportsInsert(ctx, tx, importArgs, stats); err != nil { return err } } @@ -1074,14 +1142,14 @@ func insertParsedFileGraph( for token, weight := range parsed.FileTokens { fileTokenArgs = append(fileTokenArgs, fileID, token, weight) if len(fileTokenArgs) >= sqliteTokenValuesBatchRows*3 { - if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs); err != nil { + if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs, stats); err != nil { return err } fileTokenArgs = fileTokenArgs[:0] } } if len(fileTokenArgs) > 0 { - if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs); err != nil { + if err := execTokenTriplesInsert(ctx, tx, "file_tokens", "file_id", fileTokenArgs, stats); err != nil { return err } } @@ -1114,14 +1182,14 @@ func insertParsedFileGraph( } testLinkArgs = append(testLinkArgs, repoID, fileID, testSymbolID, targetSymbolID, link.Reason, link.Score) if len(testLinkArgs) >= sqliteTestLinkValuesBatchRows*6 { - if err := execTestLinksInsert(ctx, tx, testLinkArgs); err != nil { + if err := execTestLinksInsert(ctx, tx, testLinkArgs, stats); err != nil { return err } testLinkArgs = testLinkArgs[:0] } } if len(testLinkArgs) > 0 { - if err := execTestLinksInsert(ctx, tx, testLinkArgs); err != nil { + if err := execTestLinksInsert(ctx, tx, testLinkArgs, stats); err != nil { return err } } @@ -1129,7 +1197,7 @@ func insertParsedFileGraph( return nil } -func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn string, args []any) error { +func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn string, args []any, stats *WriteStats) error { if len(args) == 0 { return nil } @@ -1147,10 +1215,24 @@ func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn str b.WriteString("(?,?,?)") } _, err := tx.ExecContext(ctx, b.String(), args...) - return err + if err != nil { + return err + } + if stats != nil { + switch table { + case "symbol_tokens": + stats.SymbolTokenInsertBatches++ + stats.SymbolTokenInsertRows += rows + case "file_tokens": + stats.FileTokenInsertBatches++ + stats.FileTokenInsertRows += rows + } + stats.TotalExecStatements++ + } + return nil } -func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, rowsPerBatch int, args []any) error { +func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, rowsPerBatch int, args []any, stats *WriteStats) error { if len(args) == 0 { return nil } @@ -1174,23 +1256,43 @@ func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, row b.WriteString(")") } _, err := tx.ExecContext(ctx, b.String(), args...) - return err + if err != nil { + return err + } + if stats != nil { + switch table { + case "references_tbl": + stats.ReferenceInsertBatches++ + stats.ReferenceInsertRows += rowCount + case "edges": + stats.EdgeInsertBatches++ + stats.EdgeInsertRows += rowCount + case "test_links": + stats.TestLinkInsertBatches++ + stats.TestLinkInsertRows += rowCount + case "file_imports": + stats.ImportInsertBatches++ + stats.ImportInsertRows += rowCount + } + stats.TotalExecStatements++ + } + return nil } -func execReferencesInsert(ctx context.Context, tx *sql.Tx, args []any) error { - return execBatchInsert(ctx, tx, "references_tbl", "repo_id, file_id, symbol_id, ref_kind, name, qualified_name, start_line, start_col, end_line, end_col, context_symbol_id", 11, args) +func execReferencesInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { + return execBatchInsert(ctx, tx, "references_tbl", "repo_id, file_id, symbol_id, ref_kind, name, qualified_name, start_line, start_col, end_line, end_col, context_symbol_id", 11, args, stats) } -func execUnresolvedEdgesInsert(ctx context.Context, tx *sql.Tx, args []any) error { - return execBatchInsert(ctx, tx, "edges", "repo_id, src_symbol_id, dst_name, edge_kind, evidence, file_id, line", 7, args) +func execUnresolvedEdgesInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { + return execBatchInsert(ctx, tx, "edges", "repo_id, src_symbol_id, dst_name, edge_kind, evidence, file_id, line", 7, args, stats) } -func execTestLinksInsert(ctx context.Context, tx *sql.Tx, args []any) error { - return execBatchInsert(ctx, tx, "test_links", "repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score", 6, args) +func execTestLinksInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { + return execBatchInsert(ctx, tx, "test_links", "repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score", 6, args, stats) } -func execImportsInsert(ctx context.Context, tx *sql.Tx, args []any) error { - return execBatchInsert(ctx, tx, "file_imports", "repo_id, file_id, import_path", 3, args) +func execImportsInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { + return execBatchInsert(ctx, tx, "file_imports", "repo_id, file_id, import_path", 3, args, stats) } func firstFunctionID(stableToID map[string]int64, symbols []graph.Symbol) int64 { From c47dfea5d61a245379080d8aef5597be31bef4b0 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:33:14 +0200 Subject: [PATCH 10/59] store: batch symbols + symbol_fts inserts; reduce delete statement pressure (#29) * store: batch symbols + symbol_fts inserts; reduce delete statement pressure * perf(store): tighten batch arg allocation in hot paths --- internal/store/store.go | 269 ++++++++++++++++++++++++++++++++-------- 1 file changed, 217 insertions(+), 52 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 925f34b..b5fee63 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -56,6 +56,13 @@ const ( // sqliteTestLinkValuesBatchRows controls multi-row inserts into test_links where each row uses 6 parameters. // 150*6=900 variables, staying under sqliteDefaultMaxVariables. sqliteTestLinkValuesBatchRows = 150 + + // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 15 parameters. + // 60*15=900 variables, staying under sqliteDefaultMaxVariables. + sqliteSymbolValuesBatchRows = 60 + // sqliteSymbolFTSValuesBatchRows controls multi-row inserts into symbol_fts where each row uses 6 parameters. + // 150*6=900 variables, staying under sqliteDefaultMaxVariables. + sqliteSymbolFTSValuesBatchRows = 150 ) type Store struct { @@ -131,12 +138,19 @@ type WriteStats struct { FileUpsertStatements int `json:"file_upsert_statements,omitempty"` - FileGraphDeleteChunks int `json:"file_graph_delete_chunks,omitempty"` - FileGraphDeleteStatements int `json:"file_graph_delete_statements,omitempty"` + FileGraphDeleteChunks int `json:"file_graph_delete_chunks,omitempty"` + FileGraphDeleteStatements int `json:"file_graph_delete_statements,omitempty"` + FileGraphDeleteTempIDInsertBatches int `json:"file_graph_delete_temp_id_insert_batches,omitempty"` + FileGraphDeleteTempIDInsertRows int `json:"file_graph_delete_temp_id_insert_rows,omitempty"` SymbolInserts int `json:"symbol_inserts,omitempty"` SymbolFTSInserts int `json:"symbol_fts_inserts,omitempty"` + SymbolInsertBatches int `json:"symbol_insert_batches,omitempty"` + SymbolInsertRows int `json:"symbol_insert_rows,omitempty"` + SymbolFTSInsertBatches int `json:"symbol_fts_insert_batches,omitempty"` + SymbolFTSInsertRows int `json:"symbol_fts_insert_rows,omitempty"` + SymbolTokenInsertBatches int `json:"symbol_token_insert_batches,omitempty"` SymbolTokenInsertRows int `json:"symbol_token_insert_rows,omitempty"` @@ -829,26 +843,6 @@ func (s *Store) ReplaceFileGraphsBatchWithStats(ctx context.Context, repoID, sca } defer upsertFileStmt.Close() - insertSymbolStmt, err := tx.PrepareContext(ctx, ` - INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertSymbolStmt.Close() - - insertSymbolFTSStmt, err := tx.PrepareContext(ctx, ` - INSERT INTO symbol_fts(repo_id, symbol_id, name, qualified_name, signature, doc_summary) - VALUES(?, ?, ?, ?, ?, ?) - `) - if err != nil { - _ = tx.Rollback() - return nil, err - } - defer insertSymbolFTSStmt.Close() - now := time.Now().UTC().Format(time.RFC3339) fileIDs := make([]int64, 0, len(inputs)) for _, input := range inputs { @@ -873,8 +867,6 @@ func (s *Store) ReplaceFileGraphsBatchWithStats(ctx context.Context, repoID, sca repoID, fileID, input.Parsed, - insertSymbolStmt, - insertSymbolFTSStmt, stats, ); err != nil { _ = tx.Rollback() @@ -893,6 +885,88 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, sta return nil } + // For large batches, use a temp table to avoid repeating large IN clauses across each dependent-table delete. + // This reduces statement pressure from ~O(numTables * chunks) down to O(chunks + numTables). + if len(fileIDs) > sqliteInClauseBatchSize { + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_delete_file_ids(id INTEGER PRIMARY KEY)`); err != nil { + return err + } + if stats != nil { + stats.TotalExecStatements++ + } + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_delete_file_ids`); err != nil { + return err + } + if stats != nil { + stats.TotalExecStatements++ + } + + for start := 0; start < len(fileIDs); start += sqliteInClauseBatchSize { + end := start + sqliteInClauseBatchSize + if end > len(fileIDs) { + end = len(fileIDs) + } + chunk := fileIDs[start:end] + placeholders := strings.Repeat("(?),", len(chunk)) + placeholders = strings.TrimSuffix(placeholders, ",") + query := `INSERT INTO tmp_delete_file_ids(id) VALUES ` + placeholders + args := make([]any, len(chunk)) + for i, id := range chunk { + args[i] = id + } + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return err + } + if stats != nil { + stats.FileGraphDeleteTempIDInsertBatches++ + stats.FileGraphDeleteTempIDInsertRows += len(chunk) + stats.TotalExecStatements++ + } + } + + exec := func(query string) error { + if _, err := tx.ExecContext(ctx, query); err != nil { + return err + } + if stats != nil { + stats.FileGraphDeleteStatements++ + stats.TotalExecStatements++ + } + return nil + } + + // Dependent tables that reference symbols must be deleted before deleting symbols. + if err := exec(`DELETE FROM symbol_tokens WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { + return err + } + if err := exec(`DELETE FROM symbol_fts WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { + return err + } + + if err := exec(`DELETE FROM edges WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM references_tbl WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM file_imports WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM file_tokens WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM test_links WHERE test_file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM symbol_embeddings WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + return nil + } + execInChunks := func(sqlPrefix, sqlSuffix string, ids []int64) error { for start := 0; start < len(ids); start += sqliteInClauseBatchSize { end := start + sqliteInClauseBatchSize @@ -903,9 +977,9 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, sta placeholders := strings.Repeat("?,", len(chunk)) placeholders = strings.TrimSuffix(placeholders, ",") query := sqlPrefix + placeholders + sqlSuffix - args := make([]any, 0, len(chunk)) - for _, id := range chunk { - args = append(args, id) + args := make([]any, len(chunk)) + for i, id := range chunk { + args[i] = id } if _, err := tx.ExecContext(ctx, query, args...); err != nil { return err @@ -1016,43 +1090,50 @@ func insertParsedFileGraph( repoID int64, fileID int64, parsed graph.ParsedFile, - insertSymbolStmt *sql.Stmt, - insertSymbolFTSStmt *sql.Stmt, stats *WriteStats, ) error { stableToID := map[string]int64{} symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) - for _, sym := range parsed.Symbols { - res, err := insertSymbolStmt.ExecContext(ctx, repoID, fileID, sym.Language, sym.Kind, sym.Name, sym.QualifiedName, sym.ContainerName, sym.Signature, sym.Visibility, sym.Range.StartLine, sym.Range.StartCol, sym.Range.EndLine, sym.Range.EndCol, sym.DocSummary, sym.StableKey) - if err != nil { - return err - } - if stats != nil { - stats.SymbolInserts++ - stats.TotalExecStatements++ - } - symbolID, err := res.LastInsertId() + symbolFTSArgs := make([]any, 0, min(len(parsed.Symbols), sqliteSymbolFTSValuesBatchRows)*6) + for start := 0; start < len(parsed.Symbols); start += sqliteSymbolValuesBatchRows { + end := start + sqliteSymbolValuesBatchRows + if end > len(parsed.Symbols) { + end = len(parsed.Symbols) + } + batch := parsed.Symbols[start:end] + batchStableToID, err := insertSymbolsBatchReturning(ctx, tx, repoID, fileID, batch, stats) if err != nil { return err } - stableToID[sym.StableKey] = symbolID - if _, err := insertSymbolFTSStmt.ExecContext(ctx, repoID, symbolID, sym.Name, sym.QualifiedName, sym.Signature, sym.DocSummary); err != nil { - return err - } - if stats != nil { - stats.SymbolFTSInserts++ - stats.TotalExecStatements++ - } - for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { - symbolTokenArgs = append(symbolTokenArgs, symbolID, token, weight) - if len(symbolTokenArgs) >= sqliteTokenValuesBatchRows*3 { - if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { + for _, sym := range batch { + symbolID, ok := batchStableToID[sym.StableKey] + if !ok || symbolID == 0 { + return fmt.Errorf("missing inserted id for stable_key=%q", sym.StableKey) + } + stableToID[sym.StableKey] = symbolID + symbolFTSArgs = append(symbolFTSArgs, repoID, symbolID, sym.Name, sym.QualifiedName, sym.Signature, sym.DocSummary) + if len(symbolFTSArgs) >= sqliteSymbolFTSValuesBatchRows*6 { + if err := execSymbolFTSInsert(ctx, tx, symbolFTSArgs, stats); err != nil { return err } - symbolTokenArgs = symbolTokenArgs[:0] + symbolFTSArgs = symbolFTSArgs[:0] + } + for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { + symbolTokenArgs = append(symbolTokenArgs, symbolID, token, weight) + if len(symbolTokenArgs) >= sqliteTokenValuesBatchRows*3 { + if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { + return err + } + symbolTokenArgs = symbolTokenArgs[:0] + } } } } + if len(symbolFTSArgs) > 0 { + if err := execSymbolFTSInsert(ctx, tx, symbolFTSArgs, stats); err != nil { + return err + } + } if len(symbolTokenArgs) > 0 { if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { return err @@ -1197,6 +1278,90 @@ func insertParsedFileGraph( return nil } +func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID int64, symbols []graph.Symbol, stats *WriteStats) (map[string]int64, error) { + if len(symbols) == 0 { + return map[string]int64{}, nil + } + + args := make([]any, len(symbols)*15) + var b strings.Builder + b.WriteString("INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) VALUES ") + argIdx := 0 + for i, sym := range symbols { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + args[argIdx+0] = repoID + args[argIdx+1] = fileID + args[argIdx+2] = sym.Language + args[argIdx+3] = sym.Kind + args[argIdx+4] = sym.Name + args[argIdx+5] = sym.QualifiedName + args[argIdx+6] = sym.ContainerName + args[argIdx+7] = sym.Signature + args[argIdx+8] = sym.Visibility + args[argIdx+9] = sym.Range.StartLine + args[argIdx+10] = sym.Range.StartCol + args[argIdx+11] = sym.Range.EndLine + args[argIdx+12] = sym.Range.EndCol + args[argIdx+13] = sym.DocSummary + args[argIdx+14] = sym.StableKey + argIdx += 15 + } + b.WriteString(" RETURNING id, stable_key") + + rows, err := tx.QueryContext(ctx, b.String(), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make(map[string]int64, len(symbols)) + rowCount := 0 + for rows.Next() { + var id int64 + var stableKey string + if err := rows.Scan(&id, &stableKey); err != nil { + return nil, err + } + out[stableKey] = id + rowCount++ + } + if err := rows.Err(); err != nil { + return nil, err + } + if rowCount != len(symbols) { + return nil, fmt.Errorf("symbol insert returned %d rows (expected %d)", rowCount, len(symbols)) + } + if stats != nil { + stats.SymbolInsertBatches++ + stats.SymbolInsertRows += rowCount + stats.SymbolInserts += rowCount + stats.TotalExecStatements++ + } + return out, nil +} + +func execSymbolFTSInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { + if len(args) == 0 { + return nil + } + if len(args)%6 != 0 { + return fmt.Errorf("invalid symbol_fts insert args len=%d", len(args)) + } + rows := len(args) / 6 + if err := execBatchInsert(ctx, tx, "symbol_fts", "repo_id, symbol_id, name, qualified_name, signature, doc_summary", 6, args, stats); err != nil { + return err + } + if stats != nil { + stats.SymbolFTSInsertBatches++ + stats.SymbolFTSInsertRows += rows + stats.SymbolFTSInserts += rows + } + return nil +} + func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn string, args []any, stats *WriteStats) error { if len(args) == 0 { return nil From d0ffa42d9f62897cef63e1db15e830a62d03e178 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:16:19 +0200 Subject: [PATCH 11/59] indexing: reduce tokenization allocations and add tokenize timing stats (#30) * indexing: reduce tokenization allocations and add tokenize timing stats * indexing: reduce tokenization allocations and add tokenize timing stats --- internal/store/store.go | 19 +++++- internal/texttoken/tokenize.go | 109 ++++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 13 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index b5fee63..1e5e6f5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -151,8 +151,10 @@ type WriteStats struct { SymbolFTSInsertBatches int `json:"symbol_fts_insert_batches,omitempty"` SymbolFTSInsertRows int `json:"symbol_fts_insert_rows,omitempty"` - SymbolTokenInsertBatches int `json:"symbol_token_insert_batches,omitempty"` - SymbolTokenInsertRows int `json:"symbol_token_insert_rows,omitempty"` + SymbolTokenInsertBatches int `json:"symbol_token_insert_batches,omitempty"` + SymbolTokenInsertRows int `json:"symbol_token_insert_rows,omitempty"` + SymbolTokenizeNS int64 `json:"symbol_tokenize_ns,omitempty"` + SymbolTokenizeCalls int `json:"symbol_tokenize_calls,omitempty"` FileTokenInsertBatches int `json:"file_token_insert_batches,omitempty"` FileTokenInsertRows int `json:"file_token_insert_rows,omitempty"` @@ -1094,6 +1096,7 @@ func insertParsedFileGraph( ) error { stableToID := map[string]int64{} symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) + symbolTokenWeights := make(map[string]float64, 64) symbolFTSArgs := make([]any, 0, min(len(parsed.Symbols), sqliteSymbolFTSValuesBatchRows)*6) for start := 0; start < len(parsed.Symbols); start += sqliteSymbolValuesBatchRows { end := start + sqliteSymbolValuesBatchRows @@ -1118,7 +1121,17 @@ func insertParsedFileGraph( } symbolFTSArgs = symbolFTSArgs[:0] } - for token, weight := range texttoken.WeightsString(sym.Name + " " + sym.QualifiedName + " " + sym.Signature + " " + sym.DocSummary) { + clear(symbolTokenWeights) + var tokenStart time.Time + if stats != nil { + tokenStart = time.Now() + stats.SymbolTokenizeCalls++ + } + texttoken.WeightsStringsInto(symbolTokenWeights, sym.Name, sym.QualifiedName, sym.Signature, sym.DocSummary) + if stats != nil { + stats.SymbolTokenizeNS += time.Since(tokenStart).Nanoseconds() + } + for token, weight := range symbolTokenWeights { symbolTokenArgs = append(symbolTokenArgs, symbolID, token, weight) if len(symbolTokenArgs) >= sqliteTokenValuesBatchRows*3 { if err := execTokenTriplesInsert(ctx, tx, "symbol_tokens", "symbol_id", symbolTokenArgs, stats); err != nil { diff --git a/internal/texttoken/tokenize.go b/internal/texttoken/tokenize.go index c5365ef..83f45de 100644 --- a/internal/texttoken/tokenize.go +++ b/internal/texttoken/tokenize.go @@ -1,22 +1,111 @@ package texttoken -import "strings" +// Tokenization rules (must match historical behavior): +// - case-insensitive +// - tokens include a-z, 0-9, and the characters: '_', '-', '.', '/' +// - any other rune/byte is a delimiter +// - tokens shorter than 2 characters are ignored func Weights(content []byte) map[string]float64 { - return WeightsString(string(content)) + out := map[string]float64{} + WeightsBytesInto(out, content) + return out } func WeightsString(text string) map[string]float64 { - text = strings.ToLower(text) - fields := strings.FieldsFunc(text, func(r rune) bool { - return !(r == '_' || r == '-' || r == '.' || r == '/' || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')) - }) out := map[string]float64{} - for _, field := range fields { - if len(field) < 2 { + WeightsStringInto(out, text) + return out +} + +func WeightsStrings(parts ...string) map[string]float64 { + out := map[string]float64{} + WeightsStringsInto(out, parts...) + return out +} + +func WeightsBytesInto(dst map[string]float64, content []byte) { + if len(content) == 0 { + return + } + var buf [128]byte + token := buf[:0] + flush := func() { + if len(token) < 2 { + token = token[:0] + return + } + dst[string(token)] += 1 + token = token[:0] + } + + for _, b := range content { + if isTokenByte(b) { + if b >= 'A' && b <= 'Z' { + b = b + ('a' - 'A') + } + token = append(token, b) continue } - out[field] += 1 + if len(token) > 0 { + flush() + } + } + if len(token) > 0 { + flush() + } +} + +func WeightsStringInto(dst map[string]float64, text string) { + if text == "" { + return + } + var buf [128]byte + token := buf[:0] + flush := func() { + if len(token) < 2 { + token = token[:0] + return + } + dst[string(token)] += 1 + token = token[:0] + } + + for i := 0; i < len(text); i++ { + b := text[i] + if isTokenByte(b) { + if b >= 'A' && b <= 'Z' { + b = b + ('a' - 'A') + } + token = append(token, b) + continue + } + if len(token) > 0 { + flush() + } + } + if len(token) > 0 { + flush() + } +} + +func WeightsStringsInto(dst map[string]float64, parts ...string) { + for _, part := range parts { + WeightsStringInto(dst, part) + } +} + +func isTokenByte(b byte) bool { + switch { + case b == '_' || b == '-' || b == '.' || b == '/': + return true + case b >= 'a' && b <= 'z': + return true + case b >= 'A' && b <= 'Z': + return true + case b >= '0' && b <= '9': + return true + default: + return false } - return out } From 5c09a0fac30c1d1ed3c10bf25d8028aa88f559e7 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:37:38 +0200 Subject: [PATCH 12/59] watch: coalesce dirty queue writes and make DrainDirtyFiles concurrency-safe (#31) * watch: coalesce dirty queue writes and make DrainDirtyFiles concurrency-safe * store: make DrainDirtyFiles atomic (DELETE RETURNING) with safe BEGIN IMMEDIATE fallback --- internal/store/store.go | 69 ++++++++++++++++++++++++++++++++----- internal/watcher/watcher.go | 37 ++++++++++++++++++-- 2 files changed, 95 insertions(+), 11 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 1e5e6f5..6629a0b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2688,22 +2688,75 @@ func (s *Store) QueueDirtyFile(ctx context.Context, repoID int64, path, reason s } func (s *Store) DrainDirtyFiles(ctx context.Context, repoID int64) ([]string, error) { - rows, err := s.db.QueryContext(ctx, `SELECT path FROM dirty_files WHERE repo_id = ? ORDER BY queued_at`, repoID) - if err != nil { + // Prefer an atomic drain. `DELETE ... RETURNING` guarantees we only remove rows + // that are returned to the caller (no SELECT+DELETE race). + rows, err := s.db.QueryContext(ctx, ` + WITH deleted AS ( + DELETE FROM dirty_files + WHERE repo_id = ? + RETURNING path, queued_at + ) + SELECT path + FROM deleted + ORDER BY queued_at + `, repoID) + if err == nil { + defer rows.Close() + var out []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + out = append(out, path) + } + return out, rows.Err() + } + + // Fallback for older SQLite builds/drivers without `RETURNING`. + // We take a write lock up-front so that events queued concurrently cannot be + // inserted until after we delete the drained rows. + conn, connErr := s.db.Conn(ctx) + if connErr != nil { return nil, err } - defer rows.Close() + defer conn.Close() + + if _, lockErr := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); lockErr != nil { + return nil, err + } + committed := false + defer func() { + if committed { + return + } + _, _ = conn.ExecContext(ctx, `ROLLBACK`) + }() + + selRows, selErr := conn.QueryContext(ctx, `SELECT path FROM dirty_files WHERE repo_id = ? ORDER BY queued_at`, repoID) + if selErr != nil { + return nil, selErr + } + defer selRows.Close() + var out []string - for rows.Next() { + for selRows.Next() { var path string - if err := rows.Scan(&path); err != nil { - return nil, err + if scanErr := selRows.Scan(&path); scanErr != nil { + return nil, scanErr } out = append(out, path) } - if _, err := s.db.ExecContext(ctx, `DELETE FROM dirty_files WHERE repo_id = ?`, repoID); err != nil { - return nil, err + if rowsErr := selRows.Err(); rowsErr != nil { + return nil, rowsErr + } + if _, delErr := conn.ExecContext(ctx, `DELETE FROM dirty_files WHERE repo_id = ?`, repoID); delErr != nil { + return nil, delErr + } + if _, commitErr := conn.ExecContext(ctx, `COMMIT`); commitErr != nil { + return nil, commitErr } + committed = true return out, nil } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index cf9392e..2ec4f37 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "sync" "sync/atomic" "time" @@ -23,6 +24,8 @@ type Watcher struct { coalescedSignals atomic.Int64 flushRuns atomic.Int64 flushErrors atomic.Int64 + queueExecs atomic.Int64 + queueSkipped atomic.Int64 queueErrors atomic.Int64 } @@ -31,6 +34,8 @@ type WatchStats struct { CoalescedSignals int64 `json:"coalesced_signals"` FlushRuns int64 `json:"flush_runs"` FlushErrors int64 `json:"flush_errors"` + QueueExecs int64 `json:"queue_execs"` + QueueSkipped int64 `json:"queue_skipped"` QueueErrors int64 `json:"queue_errors"` } @@ -44,6 +49,8 @@ func (w *Watcher) Stats() WatchStats { CoalescedSignals: w.coalescedSignals.Load(), FlushRuns: w.flushRuns.Load(), FlushErrors: w.flushErrors.Load(), + QueueExecs: w.queueExecs.Load(), + QueueSkipped: w.queueSkipped.Load(), QueueErrors: w.queueErrors.Load(), } } @@ -53,6 +60,8 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun w.coalescedSignals.Store(0) w.flushRuns.Store(0) w.flushErrors.Store(0) + w.queueExecs.Store(0) + w.queueSkipped.Store(0) w.queueErrors.Store(0) if debounce <= 0 { debounce = 750 * time.Millisecond @@ -108,6 +117,12 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun flushSignalCh := make(chan struct{}, 1) flushErrCh := make(chan error, 1) + + // Coalesce repeated fsnotify events for the same file between flushes to avoid + // redundant SQLite upserts under bursty save patterns. + var seenMu sync.Mutex + seenSinceFlush := make(map[string]struct{}) + go func() { timer := time.NewTimer(debounce) if !timer.Stop() { @@ -135,6 +150,9 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun continue } pending = false + seenMu.Lock() + clear(seenSinceFlush) + seenMu.Unlock() w.flushRuns.Add(1) if err := flush(); err != nil { w.flushErrors.Add(1) @@ -173,9 +191,22 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun } } } - if err := w.queueDirtyWithRetry(ctx, repoID, rel, event.Op.String()); err != nil { - w.queueErrors.Add(1) - return err + + seenMu.Lock() + _, alreadyQueued := seenSinceFlush[rel] + if !alreadyQueued { + seenSinceFlush[rel] = struct{}{} + } + seenMu.Unlock() + + if !alreadyQueued { + w.queueExecs.Add(1) + if err := w.queueDirtyWithRetry(ctx, repoID, rel, event.Op.String()); err != nil { + w.queueErrors.Add(1) + return err + } + } else { + w.queueSkipped.Add(1) } select { case flushSignalCh <- struct{}{}: From 52a5e7dbcd78abacb2736043c411262c8d745858 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:44:52 +0200 Subject: [PATCH 13/59] indexer: skip mark_seen DB writes for path-scoped updates --- internal/indexer/indexer.go | 12 +++++++++++- internal/store/store.go | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 124e07e..e1c8176 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -126,6 +126,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err for rel := range candidateSet { candidatePaths = append(candidatePaths, rel) } + pathScoped := len(candidateSet) > 0 var existing map[string]store.FileRecord existingLoadStarted := time.Now() @@ -290,6 +291,11 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if len(markSeenBatch) == 0 { return nil } + if pathScoped { + summary.WriteMarkSeenSkipped += len(markSeenBatch) + markSeenBatch = markSeenBatch[:0] + return nil + } started := time.Now() if err := i.store.MarkFilesSeenBatch(ctx, repo.ID, scanID, markSeenBatch); err != nil { return err @@ -390,6 +396,10 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err coverage.Skipped++ summary.LanguageCoverage[coverageLanguage] = coverage if len(markSeenBatch) < metadataBatchSize { + if pathScoped { + summary.WriteMarkSeenSkipped++ + markSeenBatch = markSeenBatch[:0] + } writeDur += time.Since(writeStart) summary.FilesSkipped++ continue @@ -614,7 +624,7 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, } return result } - if hasPrev && !force && prev.SizeBytes == task.info.Size() && prev.MtimeUnixNS == task.info.ModTime().UnixNano() { + if hasPrev && !prev.IsDeleted && !force && prev.SizeBytes == task.info.Size() && prev.MtimeUnixNS == task.info.ModTime().UnixNano() { result.action = "mark_seen" return result } diff --git a/internal/store/store.go b/internal/store/store.go index 6629a0b..b7bf88a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -118,6 +118,7 @@ type ScanSummary struct { WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` + WriteMarkSeenSkipped int `json:"write_mark_seen_skipped,omitempty"` WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` From b0d07fcf8874d0936309ef27eab5b9904d7b9d0a Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:10:59 +0200 Subject: [PATCH 14/59] indexer: resolve edges path-scoped for update runs (avoid repo-wide resolve) (#32) * indexer: resolve edges path-scoped for update runs (avoid repo-wide resolve) * refactor(indexer): deduplicate path-scoped edge resolution flow --- internal/indexer/indexer.go | 7 ++++++- internal/store/schema/011_edges_unresolved_file_idx.sql | 7 +++++++ internal/store/store.go | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 internal/store/schema/011_edges_unresolved_file_idx.sql diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index e1c8176..8744fce 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -563,7 +563,10 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if len(changedPathSet) == 0 { summary.ResolveMS = 0 - } else if len(candidateSet) > 0 { + summary.ResolveMode = "none" + } else if len(candidateSet) > 0 || scanKind == "update" { + // For path-scoped updates (Options.Paths) and incremental update runs, limit edge + // resolution to the changed files. Full index runs still do repo-wide resolution. changedPaths := make([]string, 0, len(changedPathSet)) for path := range changedPathSet { changedPaths = append(changedPaths, path) @@ -572,11 +575,13 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) return summary, err } + summary.ResolveMode = "paths" } else { if _, resolveErr := i.store.ResolveEdges(ctx, repo.ID); resolveErr != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", resolveErr.Error()) return summary, resolveErr } + summary.ResolveMode = "repo" } if len(changedPathSet) > 0 { summary.ResolveMS = time.Since(resolveStart).Milliseconds() diff --git a/internal/store/schema/011_edges_unresolved_file_idx.sql b/internal/store/schema/011_edges_unresolved_file_idx.sql new file mode 100644 index 0000000..542b6ab --- /dev/null +++ b/internal/store/schema/011_edges_unresolved_file_idx.sql @@ -0,0 +1,7 @@ +-- Speeds up ResolveEdgesForPaths by restricting unresolved-edge lookups to a file_id subset. +-- The existing idx_edges_repo_dst_file helps, but a file_id-leading partial index performs +-- better for path-scoped updates where file_id filtering is the primary selector. +CREATE INDEX IF NOT EXISTS idx_edges_repo_file_unresolved +ON edges(repo_id, file_id) +WHERE dst_symbol_id IS NULL; + diff --git a/internal/store/store.go b/internal/store/store.go index b7bf88a..b8e40f6 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -126,6 +126,7 @@ type ScanSummary struct { EmbedMS int64 `json:"embed_ms,omitempty"` MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` ResolveMS int64 `json:"resolve_ms,omitempty"` + ResolveMode string `json:"resolve_mode,omitempty"` DurationMS int64 `json:"duration_ms"` } From b70f91254c719d66a1b9dbae269bfc219c29a12a Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:33:48 +0200 Subject: [PATCH 15/59] indexer: targeted cross-file edge resolution on update (preserve path-scoped win) (#33) * indexer: targeted cross-file edge resolution on update (preserve path-scoped win) * indexer: generalize cross-file resolve for partial runs; simplify ResolveEdgesForNames --- internal/indexer/indexer.go | 26 ++++++ .../indexer/indexer_resolve_crossfile_test.go | 79 +++++++++++++++++++ internal/store/store.go | 73 +++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 internal/indexer/indexer_resolve_crossfile_test.go diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 8744fce..aaf5ac6 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -281,6 +281,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err parseFailedBatch := make([]store.FileMetadataUpdate, 0, metadataBatchSize) replaceBatch := make([]store.ReplaceFileGraphInput, 0, replaceBatchSize) changedPathSet := make(map[string]struct{}, 64) + changedSymbolNameSet := make(map[string]struct{}, 256) var writeMetadataDur time.Duration var writeReplaceDur time.Duration @@ -444,6 +445,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err Parsed: res.parsed, }) changedPathSet[res.task.rel] = struct{}{} + for _, sym := range res.parsed.Symbols { + if sym.Name == "" { + continue + } + changedSymbolNameSet[sym.Name] = struct{}{} + } summary.FilesChanged++ summary.FilesIndexed++ coverage := summary.LanguageCoverage[coverageLanguage] @@ -576,6 +583,25 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, err } summary.ResolveMode = "paths" + + // Correctness: partial runs can introduce symbols that should resolve previously-unresolved + // edges in other files. Do a narrow cross-file pass keyed by the introduced symbol names, + // without falling back to repo-wide resolution. + if len(changedSymbolNameSet) > 0 { + names := make([]string, 0, len(changedSymbolNameSet)) + for name := range changedSymbolNameSet { + names = append(names, name) + } + crossStart := time.Now() + n, err := i.store.ResolveEdgesForNames(ctx, repo.ID, names) + if err != nil { + _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) + return summary, err + } + summary.ResolveCrossFileTargets = n + summary.ResolveCrossFileMS = time.Since(crossStart).Milliseconds() + summary.ResolveMode = "paths+names" + } } else { if _, resolveErr := i.store.ResolveEdges(ctx, repo.ID); resolveErr != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", resolveErr.Error()) diff --git a/internal/indexer/indexer_resolve_crossfile_test.go b/internal/indexer/indexer_resolve_crossfile_test.go new file mode 100644 index 0000000..3722fb9 --- /dev/null +++ b/internal/indexer/indexer_resolve_crossfile_test.go @@ -0,0 +1,79 @@ +package indexer + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/isink17/codegraph/internal/parser" + "github.com/isink17/codegraph/internal/parser/golang" + "github.com/isink17/codegraph/internal/store" +) + +func TestUpdateResolvesUnresolvedEdgesInOtherFilesWhenNewSymbolIntroduced(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + dbPath := filepath.Join(t.TempDir(), "codegraph.db") + + write := func(rel, content string) { + t.Helper() + abs := filepath.Join(repoRoot, rel) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(abs, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + + write("a.go", "package p\n\nfunc A() { Foo() }\n") + write("b.go", "package p\n\nfunc B() { Foo() }\n") + + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + reg := parser.NewRegistry(golang.New()) + idx := New(s, reg, nil) + + if _, err := idx.Index(ctx, Options{RepoRoot: repoRoot, ScanKind: "index"}); err != nil { + t.Fatalf("index: %v", err) + } + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("upsert repo: %v", err) + } + + // Introduce Foo in a.go and update only that path. + write("a.go", "package p\n\nfunc Foo() {}\n\nfunc A() { Foo() }\n") + if _, err := idx.Update(ctx, Options{RepoRoot: repoRoot, ScanKind: "update", Paths: []string{"a.go"}}); err != nil { + t.Fatalf("update: %v", err) + } + + // Verify b.go's edge now resolves. + edges, err := s.ExportEdgesPage(ctx, repo.ID, 10000, 0) + if err != nil { + t.Fatalf("export edges: %v", err) + } + foundB := false + resolvedB := false + for _, e := range edges { + if e.FilePath != "b.go" { + continue + } + foundB = true + if e.DstSymbolID != nil { + resolvedB = true + break + } + } + if !foundB { + t.Fatalf("expected at least one edge from b.go") + } + if !resolvedB { + t.Fatalf("expected unresolved edge in b.go to be resolved after updating only a.go") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index b8e40f6..6ce8718 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -127,6 +127,8 @@ type ScanSummary struct { MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` ResolveMS int64 `json:"resolve_ms,omitempty"` ResolveMode string `json:"resolve_mode,omitempty"` + ResolveCrossFileMS int64 `json:"resolve_cross_file_ms,omitempty"` + ResolveCrossFileTargets int `json:"resolve_cross_file_targets,omitempty"` DurationMS int64 `json:"duration_ms"` } @@ -1822,6 +1824,77 @@ func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths [] return s.resolveEdgeTargets(ctx, repoID, targets) } +// ResolveEdgesForNames attempts to resolve currently-unresolved edges across the +// repo whose dst_name matches (or ends with ".") any of the provided +// names. This is used to keep incremental update runs correct when newly +// introduced symbols should resolve previously-unresolved edges in other files. +// +// It returns the number of candidate edges selected for resolution. +func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names []string) (int, error) { + if len(names) == 0 { + return 0, nil + } + seen := make(map[string]struct{}, len(names)) + unique := make([]string, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + unique = append(unique, name) + } + if len(unique) == 0 { + return 0, nil + } + + // Query shape: do a single unresolved-edge scan, then filter in Go. + // This avoids repeated SQL queries with many OR/L LIKE clauses (which are + // difficult for SQLite to optimize with indexes). + rows, err := s.db.QueryContext(ctx, ` + SELECT id, dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + `, repoID) + if err != nil { + return 0, err + } + defer rows.Close() + + nameSet := make(map[string]struct{}, len(unique)) + for _, name := range unique { + nameSet[name] = struct{}{} + } + + targets := make([]edgeTarget, 0, 64) + for rows.Next() { + var id int64 + var dstName string + if err := rows.Scan(&id, &dstName); err != nil { + return 0, err + } + if _, ok := nameSet[dstName]; ok { + targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + continue + } + if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { + if _, ok := nameSet[dstName[dot+1:]]; ok { + targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + } + } + } + if err := rows.Err(); err != nil { + return 0, err + } + if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { + return 0, err + } + return len(targets), nil +} + func scanEdgeTargets(rows *sql.Rows) ([]edgeTarget, error) { defer rows.Close() var targets []edgeTarget From 91d1d4df0dcae4cd9bbd6bc6d34001351e17dedf Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:45:54 +0200 Subject: [PATCH 16/59] store: narrow ResolveEdgesForNames candidates with indexed exact-match prefilter --- .../012_edges_unresolved_dst_name_idx.sql | 6 ++ internal/store/store.go | 89 +++++++++++++++---- 2 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 internal/store/schema/012_edges_unresolved_dst_name_idx.sql diff --git a/internal/store/schema/012_edges_unresolved_dst_name_idx.sql b/internal/store/schema/012_edges_unresolved_dst_name_idx.sql new file mode 100644 index 0000000..e765b48 --- /dev/null +++ b/internal/store/schema/012_edges_unresolved_dst_name_idx.sql @@ -0,0 +1,6 @@ +-- Helps targeted cross-file edge resolution (ResolveEdgesForNames) quickly find +-- exact dst_name matches without scanning all unresolved edges. +CREATE INDEX IF NOT EXISTS idx_edges_repo_unresolved_dst_name +ON edges(repo_id, dst_name) +WHERE dst_symbol_id IS NULL AND dst_name != ''; + diff --git a/internal/store/store.go b/internal/store/store.go index 6ce8718..251de9e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1851,44 +1851,103 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] return 0, nil } - // Query shape: do a single unresolved-edge scan, then filter in Go. - // This avoids repeated SQL queries with many OR/L LIKE clauses (which are - // difficult for SQLite to optimize with indexes). + nameSet := make(map[string]struct{}, len(unique)) + for _, name := range unique { + nameSet[name] = struct{}{} + } + + // Candidate selection: + // + // 1) Use indexed exact matches for dst_name = (covers the common case + // where unresolved edges reference the simple symbol name directly). + // 2) Only if needed, scan unresolved edges that contain a '.' and filter in Go + // for suffix matches (dst_name ends with "."). + // + // This avoids scanning the full unresolved-edge set on large repos where many + // unresolved edges have simple (non-qualified) dst_name values. + targetByID := make(map[int64]edgeTarget, 64) + + const maxVars = 999 // SQLite default + const chunkSize = 900 + for start := 0; start < len(unique); start += chunkSize { + end := min(start+chunkSize, len(unique)) + chunk := unique[start:end] + + // args: repoID + N names + if len(chunk)+1 > maxVars { + // Defensive: chunkSize should keep us under maxVars. + chunk = chunk[:maxVars-1] + } + + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := `SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name IN (` + placeholders + `)` + args := make([]any, 1+len(chunk)) + args[0] = repoID + for i, name := range chunk { + args[i+1] = name + } + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return 0, err + } + for rows.Next() { + var id int64 + var dstName string + if err := rows.Scan(&id, &dstName); err != nil { + _ = rows.Close() + return 0, err + } + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, err + } + if err := rows.Close(); err != nil { + return 0, err + } + } + + // Suffix matching requires looking at qualified dst_name values. Keep this + // as a single pass over the qualified unresolved set (no repeated LIKE + // queries), but avoid scanning simple dst_name values entirely. rows, err := s.db.QueryContext(ctx, ` SELECT id, dst_name FROM edges - WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '.') > 0 `, repoID) if err != nil { return 0, err } - defer rows.Close() - - nameSet := make(map[string]struct{}, len(unique)) - for _, name := range unique { - nameSet[name] = struct{}{} - } - - targets := make([]edgeTarget, 0, 64) for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { + _ = rows.Close() return 0, err } - if _, ok := nameSet[dstName]; ok { - targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + if _, ok := targetByID[id]; ok { continue } if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { if _, ok := nameSet[dstName[dot+1:]]; ok { - targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} } } } if err := rows.Err(); err != nil { + _ = rows.Close() return 0, err } + if err := rows.Close(); err != nil { + return 0, err + } + + targets := make([]edgeTarget, 0, len(targetByID)) + for _, target := range targetByID { + targets = append(targets, target) + } if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { return 0, err } From 968c113bc3cdc200e76a3c5f1ad5a5f24f7a9563 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:46:14 +0200 Subject: [PATCH 17/59] Revert "store: narrow ResolveEdgesForNames candidates with indexed exact-match prefilter" This reverts commit 91d1d4df0dcae4cd9bbd6bc6d34001351e17dedf. --- .../012_edges_unresolved_dst_name_idx.sql | 6 -- internal/store/store.go | 89 ++++--------------- 2 files changed, 15 insertions(+), 80 deletions(-) delete mode 100644 internal/store/schema/012_edges_unresolved_dst_name_idx.sql diff --git a/internal/store/schema/012_edges_unresolved_dst_name_idx.sql b/internal/store/schema/012_edges_unresolved_dst_name_idx.sql deleted file mode 100644 index e765b48..0000000 --- a/internal/store/schema/012_edges_unresolved_dst_name_idx.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Helps targeted cross-file edge resolution (ResolveEdgesForNames) quickly find --- exact dst_name matches without scanning all unresolved edges. -CREATE INDEX IF NOT EXISTS idx_edges_repo_unresolved_dst_name -ON edges(repo_id, dst_name) -WHERE dst_symbol_id IS NULL AND dst_name != ''; - diff --git a/internal/store/store.go b/internal/store/store.go index 251de9e..6ce8718 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1851,103 +1851,44 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] return 0, nil } - nameSet := make(map[string]struct{}, len(unique)) - for _, name := range unique { - nameSet[name] = struct{}{} - } - - // Candidate selection: - // - // 1) Use indexed exact matches for dst_name = (covers the common case - // where unresolved edges reference the simple symbol name directly). - // 2) Only if needed, scan unresolved edges that contain a '.' and filter in Go - // for suffix matches (dst_name ends with "."). - // - // This avoids scanning the full unresolved-edge set on large repos where many - // unresolved edges have simple (non-qualified) dst_name values. - targetByID := make(map[int64]edgeTarget, 64) - - const maxVars = 999 // SQLite default - const chunkSize = 900 - for start := 0; start < len(unique); start += chunkSize { - end := min(start+chunkSize, len(unique)) - chunk := unique[start:end] - - // args: repoID + N names - if len(chunk)+1 > maxVars { - // Defensive: chunkSize should keep us under maxVars. - chunk = chunk[:maxVars-1] - } - - placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") - query := `SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name IN (` + placeholders + `)` - args := make([]any, 1+len(chunk)) - args[0] = repoID - for i, name := range chunk { - args[i+1] = name - } - - rows, err := s.db.QueryContext(ctx, query, args...) - if err != nil { - return 0, err - } - for rows.Next() { - var id int64 - var dstName string - if err := rows.Scan(&id, &dstName); err != nil { - _ = rows.Close() - return 0, err - } - targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} - } - if err := rows.Err(); err != nil { - _ = rows.Close() - return 0, err - } - if err := rows.Close(); err != nil { - return 0, err - } - } - - // Suffix matching requires looking at qualified dst_name values. Keep this - // as a single pass over the qualified unresolved set (no repeated LIKE - // queries), but avoid scanning simple dst_name values entirely. + // Query shape: do a single unresolved-edge scan, then filter in Go. + // This avoids repeated SQL queries with many OR/L LIKE clauses (which are + // difficult for SQLite to optimize with indexes). rows, err := s.db.QueryContext(ctx, ` SELECT id, dst_name FROM edges - WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '.') > 0 + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' `, repoID) if err != nil { return 0, err } + defer rows.Close() + + nameSet := make(map[string]struct{}, len(unique)) + for _, name := range unique { + nameSet[name] = struct{}{} + } + + targets := make([]edgeTarget, 0, 64) for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { - _ = rows.Close() return 0, err } - if _, ok := targetByID[id]; ok { + if _, ok := nameSet[dstName]; ok { + targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) continue } if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { if _, ok := nameSet[dstName[dot+1:]]; ok { - targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) } } } if err := rows.Err(); err != nil { - _ = rows.Close() return 0, err } - if err := rows.Close(); err != nil { - return 0, err - } - - targets := make([]edgeTarget, 0, len(targetByID)) - for _, target := range targetByID { - targets = append(targets, target) - } if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { return 0, err } From 4f9624b51b3b22bdf92c60c9ccb0a8a2f07e43dc Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 12:06:29 +0200 Subject: [PATCH 18/59] store: narrow ResolveEdgesForNames candidates with indexed exact-match prefilter (#34) * store: narrow ResolveEdgesForNames candidates with indexed exact-match prefilter * store: drop redundant unresolved dst_name index; reuse sqlite batch constants --- internal/store/store.go | 82 +++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 6ce8718..7cd6458 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1851,44 +1851,96 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] return 0, nil } - // Query shape: do a single unresolved-edge scan, then filter in Go. - // This avoids repeated SQL queries with many OR/L LIKE clauses (which are - // difficult for SQLite to optimize with indexes). + nameSet := make(map[string]struct{}, len(unique)) + for _, name := range unique { + nameSet[name] = struct{}{} + } + + // Candidate selection: + // + // 1) Use indexed exact matches for dst_name = (covers the common case + // where unresolved edges reference the simple symbol name directly). + // 2) Only if needed, scan unresolved edges that contain a '.' and filter in Go + // for suffix matches (dst_name ends with "."). + // + // This avoids scanning the full unresolved-edge set on large repos where many + // unresolved edges have simple (non-qualified) dst_name values. + targetByID := make(map[int64]edgeTarget, 64) + + // Keep under sqliteDefaultMaxVariables (repoID + N names). + for start := 0; start < len(unique); start += sqliteInClauseBatchSize { + end := min(start+sqliteInClauseBatchSize, len(unique)) + chunk := unique[start:end] + + placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") + query := `SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name IN (` + placeholders + `)` + args := make([]any, 1+len(chunk)) + args[0] = repoID + for i, name := range chunk { + args[i+1] = name + } + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return 0, err + } + for rows.Next() { + var id int64 + var dstName string + if err := rows.Scan(&id, &dstName); err != nil { + _ = rows.Close() + return 0, err + } + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, err + } + if err := rows.Close(); err != nil { + return 0, err + } + } + + // Suffix matching requires looking at qualified dst_name values. Keep this + // as a single pass over the qualified unresolved set (no repeated LIKE + // queries), but avoid scanning simple dst_name values entirely. rows, err := s.db.QueryContext(ctx, ` SELECT id, dst_name FROM edges - WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '.') > 0 `, repoID) if err != nil { return 0, err } - defer rows.Close() - - nameSet := make(map[string]struct{}, len(unique)) - for _, name := range unique { - nameSet[name] = struct{}{} - } - - targets := make([]edgeTarget, 0, 64) for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { + _ = rows.Close() return 0, err } - if _, ok := nameSet[dstName]; ok { - targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + if _, ok := targetByID[id]; ok { continue } if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { if _, ok := nameSet[dstName[dot+1:]]; ok { - targets = append(targets, edgeTarget{edgeID: id, dstName: dstName}) + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} } } } if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, err + } + if err := rows.Close(); err != nil { return 0, err } + + targets := make([]edgeTarget, 0, len(targetByID)) + for _, target := range targetByID { + targets = append(targets, target) + } if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { return 0, err } From c18a83f85b6f52249a8cc64c241608ad83a39a2d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:01:58 +0200 Subject: [PATCH 19/59] resolve: add cross-file candidate stats + tests for ResolveEdgesForNames --- internal/indexer/indexer.go | 7 +- internal/store/cross_file_resolution_test.go | 60 +++++++ .../store/resolve_edges_for_names_test.go | 152 ++++++++++++++++++ internal/store/store.go | 146 +++++++++++------ 4 files changed, 313 insertions(+), 52 deletions(-) create mode 100644 internal/store/cross_file_resolution_test.go create mode 100644 internal/store/resolve_edges_for_names_test.go diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index aaf5ac6..c026b3d 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -593,12 +593,15 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err names = append(names, name) } crossStart := time.Now() - n, err := i.store.ResolveEdgesForNames(ctx, repo.ID, names) + stats, err := i.store.ResolveEdgesForNamesWithStats(ctx, repo.ID, names) if err != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) return summary, err } - summary.ResolveCrossFileTargets = n + if stats != (store.ResolveEdgesForNamesStats{}) { + summary.ResolveCrossFile = &stats + } + summary.ResolveCrossFileTargets = stats.TargetsSelected summary.ResolveCrossFileMS = time.Since(crossStart).Milliseconds() summary.ResolveMode = "paths+names" } diff --git a/internal/store/cross_file_resolution_test.go b/internal/store/cross_file_resolution_test.go new file mode 100644 index 0000000..c1d2a0c --- /dev/null +++ b/internal/store/cross_file_resolution_test.go @@ -0,0 +1,60 @@ +package store_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/isink17/codegraph/internal/indexer" + "github.com/isink17/codegraph/internal/parser" + goparser "github.com/isink17/codegraph/internal/parser/golang" + "github.com/isink17/codegraph/internal/store" +) + +func TestCrossFileResolution_PartialRunIntroducedSymbolResolvesOtherFile(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + + if err := os.WriteFile(filepath.Join(repoRoot, "b.go"), []byte("package main\nfunc Bar() { Foo() }\n"), 0o644); err != nil { + t.Fatalf("WriteFile(b.go) error = %v", err) + } + + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + + if err := os.WriteFile(filepath.Join(repoRoot, "a.go"), []byte("package main\nfunc Foo() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile(a.go) error = %v", err) + } + summary, err := idx.Update(ctx, indexer.Options{RepoRoot: repoRoot, Paths: []string{"a.go"}}) + if err != nil { + t.Fatalf("Update() error = %v", err) + } + if summary.ResolveMode != "paths+names" { + t.Fatalf("ResolveMode = %q, want %q", summary.ResolveMode, "paths+names") + } + if summary.ResolveCrossFile == nil || summary.ResolveCrossFile.TargetsSelected == 0 { + t.Fatalf("ResolveCrossFile stats missing or empty: %#v", summary.ResolveCrossFile) + } + + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + unresolved, err := s.CountUnresolvedEdgesByDstName(ctx, repo.ID, "Foo") + if err != nil { + t.Fatalf("CountUnresolvedEdgesByDstName() error = %v", err) + } + if unresolved != 0 { + t.Fatalf("unresolved edges for Foo = %d, want 0", unresolved) + } +} diff --git a/internal/store/resolve_edges_for_names_test.go b/internal/store/resolve_edges_for_names_test.go new file mode 100644 index 0000000..46f356a --- /dev/null +++ b/internal/store/resolve_edges_for_names_test.go @@ -0,0 +1,152 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +func TestResolveEdgesForNames_ExactName(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + fileA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + fileB, err := insertTestFile(ctx, s, repo.ID, "b.go") + if err != nil { + t.Fatalf("insertTestFile(b.go) error = %v", err) + } + + dstID, err := insertTestSymbol(ctx, s, repo.ID, fileA, "Foo", "Foo") + if err != nil { + t.Fatalf("insertTestSymbol(dst) error = %v", err) + } + srcID, err := insertTestSymbol(ctx, s, repo.ID, fileB, "Bar", "Bar") + if err != nil { + t.Fatalf("insertTestSymbol(src) error = %v", err) + } + edgeID, err := insertTestEdge(ctx, s, repo.ID, fileB, srcID, "Foo") + if err != nil { + t.Fatalf("insertTestEdge() error = %v", err) + } + + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repo.ID, []string{"Foo"}) + if err != nil { + t.Fatalf("ResolveEdgesForNamesWithStats() error = %v", err) + } + if stats.TargetsSelected != 1 { + t.Fatalf("TargetsSelected = %d, want 1", stats.TargetsSelected) + } + if stats.ExactHits != 1 { + t.Fatalf("ExactHits = %d, want 1", stats.ExactHits) + } + + var gotDst sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT dst_symbol_id FROM edges WHERE id = ?`, edgeID).Scan(&gotDst); err != nil { + t.Fatalf("QueryRow(dst_symbol_id) error = %v", err) + } + if !gotDst.Valid || gotDst.Int64 != dstID { + t.Fatalf("dst_symbol_id = (%v,%d), want (%v,%d)", gotDst.Valid, gotDst.Int64, true, dstID) + } +} + +func TestResolveEdgesForNames_QualifiedSuffix(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + fileA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + fileB, err := insertTestFile(ctx, s, repo.ID, "b.go") + if err != nil { + t.Fatalf("insertTestFile(b.go) error = %v", err) + } + + dstID, err := insertTestSymbol(ctx, s, repo.ID, fileA, "Foo", "pkg.Foo") + if err != nil { + t.Fatalf("insertTestSymbol(dst) error = %v", err) + } + srcID, err := insertTestSymbol(ctx, s, repo.ID, fileB, "Bar", "Bar") + if err != nil { + t.Fatalf("insertTestSymbol(src) error = %v", err) + } + edgeID, err := insertTestEdge(ctx, s, repo.ID, fileB, srcID, "pkg.Foo") + if err != nil { + t.Fatalf("insertTestEdge() error = %v", err) + } + + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repo.ID, []string{"Foo"}) + if err != nil { + t.Fatalf("ResolveEdgesForNamesWithStats() error = %v", err) + } + if stats.TargetsSelected != 1 { + t.Fatalf("TargetsSelected = %d, want 1", stats.TargetsSelected) + } + if stats.SuffixHits != 1 { + t.Fatalf("SuffixHits = %d, want 1", stats.SuffixHits) + } + + var gotDst sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT dst_symbol_id FROM edges WHERE id = ?`, edgeID).Scan(&gotDst); err != nil { + t.Fatalf("QueryRow(dst_symbol_id) error = %v", err) + } + if !gotDst.Valid || gotDst.Int64 != dstID { + t.Fatalf("dst_symbol_id = (%v,%d), want (%v,%d)", gotDst.Valid, gotDst.Int64, true, dstID) + } +} + +func insertTestFile(ctx context.Context, s *Store, repoID int64, path string) (int64, error) { + res, err := s.db.ExecContext(ctx, `INSERT INTO files(repo_id, path, language, indexed_at) VALUES(?, ?, ?, '')`, repoID, path, "go") + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func insertTestSymbol(ctx context.Context, s *Store, repoID, fileID int64, name, qualified string) (int64, error) { + res, err := s.db.ExecContext(ctx, ` + INSERT INTO symbols( + repo_id, file_id, language, kind, name, qualified_name, + start_line, start_col, end_line, end_col, stable_key + ) + VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?) + `, repoID, fileID, "go", "function", name, qualified, qualified) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func insertTestEdge(ctx context.Context, s *Store, repoID, fileID, srcSymbolID int64, dstName string) (int64, error) { + res, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, evidence, file_id, line) + VALUES(?, ?, NULL, ?, 'call', '', ?, 1) + `, repoID, srcSymbolID, dstName, fileID) + if err != nil { + return 0, err + } + return res.LastInsertId() +} diff --git a/internal/store/store.go b/internal/store/store.go index 7cd6458..dabdde1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -92,44 +92,58 @@ type FileMetadataUpdate struct { } type ScanSummary struct { - RepoID int64 `json:"repo_id"` - ScanID int64 `json:"scan_id"` - FilesSeen int `json:"files_seen"` - FilesIndexed int `json:"files_indexed"` - FilesSkipped int `json:"files_skipped"` - FilesChanged int `json:"files_changed"` - FilesDeleted int `json:"files_deleted"` - FilesTotal int `json:"files_total,omitempty"` - FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` - ParseErrors int `json:"parse_errors,omitempty"` - ParseSamples []string `json:"parse_samples,omitempty"` - LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` - PhaseTimings []ScanPhaseTiming `json:"phase_timings,omitempty"` - ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` - WalkMS int64 `json:"walk_ms,omitempty"` - ProcessWallMS int64 `json:"process_wall_ms,omitempty"` - TaskMS int64 `json:"task_ms,omitempty"` - TaskOtherMS int64 `json:"task_other_ms,omitempty"` - ParseMS int64 `json:"parse_ms,omitempty"` - ReadMS int64 `json:"read_ms,omitempty"` - HashMS int64 `json:"hash_ms,omitempty"` - AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` - WriteMS int64 `json:"write_ms,omitempty"` - WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` - WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` - WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` - WriteMarkSeenSkipped int `json:"write_mark_seen_skipped,omitempty"` - WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` - WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` - WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` - WriteStats *WriteStats `json:"write_stats,omitempty"` - EmbedMS int64 `json:"embed_ms,omitempty"` - MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` - ResolveMS int64 `json:"resolve_ms,omitempty"` - ResolveMode string `json:"resolve_mode,omitempty"` - ResolveCrossFileMS int64 `json:"resolve_cross_file_ms,omitempty"` - ResolveCrossFileTargets int `json:"resolve_cross_file_targets,omitempty"` - DurationMS int64 `json:"duration_ms"` + RepoID int64 `json:"repo_id"` + ScanID int64 `json:"scan_id"` + FilesSeen int `json:"files_seen"` + FilesIndexed int `json:"files_indexed"` + FilesSkipped int `json:"files_skipped"` + FilesChanged int `json:"files_changed"` + FilesDeleted int `json:"files_deleted"` + FilesTotal int `json:"files_total,omitempty"` + FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` + ParseErrors int `json:"parse_errors,omitempty"` + ParseSamples []string `json:"parse_samples,omitempty"` + LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` + PhaseTimings []ScanPhaseTiming `json:"phase_timings,omitempty"` + ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` + WalkMS int64 `json:"walk_ms,omitempty"` + ProcessWallMS int64 `json:"process_wall_ms,omitempty"` + TaskMS int64 `json:"task_ms,omitempty"` + TaskOtherMS int64 `json:"task_other_ms,omitempty"` + ParseMS int64 `json:"parse_ms,omitempty"` + ReadMS int64 `json:"read_ms,omitempty"` + HashMS int64 `json:"hash_ms,omitempty"` + AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` + WriteMS int64 `json:"write_ms,omitempty"` + WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` + WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` + WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` + WriteMarkSeenSkipped int `json:"write_mark_seen_skipped,omitempty"` + WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` + WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` + WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` + WriteStats *WriteStats `json:"write_stats,omitempty"` + EmbedMS int64 `json:"embed_ms,omitempty"` + MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` + ResolveMS int64 `json:"resolve_ms,omitempty"` + ResolveMode string `json:"resolve_mode,omitempty"` + ResolveCrossFileMS int64 `json:"resolve_cross_file_ms,omitempty"` + ResolveCrossFileTargets int `json:"resolve_cross_file_targets,omitempty"` + ResolveCrossFile *ResolveEdgesForNamesStats `json:"resolve_cross_file,omitempty"` + DurationMS int64 `json:"duration_ms"` +} + +type ResolveEdgesForNamesStats struct { + NamesInput int `json:"names_input,omitempty"` + NamesUnique int `json:"names_unique,omitempty"` + ExactQueryBatches int `json:"exact_query_batches,omitempty"` + ExactHits int `json:"exact_hits,omitempty"` + QualifiedScanned int `json:"qualified_scanned,omitempty"` + SuffixHits int `json:"suffix_hits,omitempty"` + TargetsSelected int `json:"targets_selected,omitempty"` + ExactSelectMS int64 `json:"exact_select_ms,omitempty"` + SuffixSelectMS int64 `json:"suffix_select_ms,omitempty"` + ResolveTargetsMS int64 `json:"resolve_targets_ms,omitempty"` } type ScanPhaseTiming struct { @@ -1831,9 +1845,19 @@ func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths [] // // It returns the number of candidate edges selected for resolution. func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names []string) (int, error) { + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repoID, names) + if err != nil { + return 0, err + } + return stats.TargetsSelected, nil +} + +func (s *Store) ResolveEdgesForNamesWithStats(ctx context.Context, repoID int64, names []string) (ResolveEdgesForNamesStats, error) { + var stats ResolveEdgesForNamesStats if len(names) == 0 { - return 0, nil + return stats, nil } + stats.NamesInput = len(names) seen := make(map[string]struct{}, len(names)) unique := make([]string, 0, len(names)) for _, name := range names { @@ -1848,8 +1872,9 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] unique = append(unique, name) } if len(unique) == 0 { - return 0, nil + return stats, nil } + stats.NamesUnique = len(unique) nameSet := make(map[string]struct{}, len(unique)) for _, name := range unique { @@ -1867,6 +1892,7 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] // unresolved edges have simple (non-qualified) dst_name values. targetByID := make(map[int64]edgeTarget, 64) + exactStarted := time.Now() // Keep under sqliteDefaultMaxVariables (repoID + N names). for start := 0; start < len(unique); start += sqliteInClauseBatchSize { end := min(start+sqliteInClauseBatchSize, len(unique)) @@ -1882,69 +1908,79 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { - return 0, err + return stats, err } for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { _ = rows.Close() - return 0, err + return stats, err } targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.ExactHits++ } if err := rows.Err(); err != nil { _ = rows.Close() - return 0, err + return stats, err } if err := rows.Close(); err != nil { - return 0, err + return stats, err } + stats.ExactQueryBatches++ } + stats.ExactSelectMS = time.Since(exactStarted).Milliseconds() // Suffix matching requires looking at qualified dst_name values. Keep this // as a single pass over the qualified unresolved set (no repeated LIKE // queries), but avoid scanning simple dst_name values entirely. + suffixStarted := time.Now() rows, err := s.db.QueryContext(ctx, ` SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '.') > 0 `, repoID) if err != nil { - return 0, err + return stats, err } for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { _ = rows.Close() - return 0, err + return stats, err } + stats.QualifiedScanned++ if _, ok := targetByID[id]; ok { continue } if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { if _, ok := nameSet[dstName[dot+1:]]; ok { targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.SuffixHits++ } } } if err := rows.Err(); err != nil { _ = rows.Close() - return 0, err + return stats, err } if err := rows.Close(); err != nil { - return 0, err + return stats, err } + stats.SuffixSelectMS = time.Since(suffixStarted).Milliseconds() targets := make([]edgeTarget, 0, len(targetByID)) for _, target := range targetByID { targets = append(targets, target) } + stats.TargetsSelected = len(targets) + resolveStarted := time.Now() if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { - return 0, err + return stats, err } - return len(targets), nil + stats.ResolveTargetsMS = time.Since(resolveStarted).Milliseconds() + return stats, nil } func scanEdgeTargets(rows *sql.Rows) ([]edgeTarget, error) { @@ -1963,6 +1999,16 @@ func scanEdgeTargets(rows *sql.Rows) ([]edgeTarget, error) { return targets, nil } +func (s *Store) CountUnresolvedEdgesByDstName(ctx context.Context, repoID int64, dstName string) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name = ? + `, repoID, dstName).Scan(&n) + return n, err +} + func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets []edgeTarget) error { if len(targets) == 0 { return nil From 0fe669471f47840895776e0f097d6e4c8531ff9e Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:41:20 +0200 Subject: [PATCH 20/59] watch: ignore chmod-only + directory-create events; add repeated-workstats (#36) * watch: ignore chmod-only + directory-create events; add repeated-work stats * fix(watcher): count only successful update runs in watcher stats --- internal/watcher/watcher.go | 84 +++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 2ec4f37..1cbf7de 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -20,23 +20,47 @@ type Watcher struct { store *store.Store indexer *indexer.Indexer + eventsSeen atomic.Int64 + eventsIgnored atomic.Int64 + eventsIgnoredDir atomic.Int64 + eventsIgnoredChmod atomic.Int64 + flushSignals atomic.Int64 coalescedSignals atomic.Int64 flushRuns atomic.Int64 flushErrors atomic.Int64 - queueExecs atomic.Int64 - queueSkipped atomic.Int64 - queueErrors atomic.Int64 + flushNoop atomic.Int64 + + queueExecs atomic.Int64 + queueSkipped atomic.Int64 + queueErrors atomic.Int64 + + drainRuns atomic.Int64 + drainPaths atomic.Int64 + updateRuns atomic.Int64 + updatePaths atomic.Int64 } type WatchStats struct { + EventsSeen int64 `json:"events_seen"` + EventsIgnored int64 `json:"events_ignored"` + EventsIgnoredDir int64 `json:"events_ignored_dir"` + EventsIgnoredChmod int64 `json:"events_ignored_chmod"` + FlushSignals int64 `json:"flush_signals"` CoalescedSignals int64 `json:"coalesced_signals"` FlushRuns int64 `json:"flush_runs"` FlushErrors int64 `json:"flush_errors"` - QueueExecs int64 `json:"queue_execs"` - QueueSkipped int64 `json:"queue_skipped"` - QueueErrors int64 `json:"queue_errors"` + FlushNoop int64 `json:"flush_noop"` + + DrainRuns int64 `json:"drain_runs"` + DrainPaths int64 `json:"drain_paths"` + UpdateRuns int64 `json:"update_runs"` + UpdatePaths int64 `json:"update_paths"` + + QueueExecs int64 `json:"queue_execs"` + QueueSkipped int64 `json:"queue_skipped"` + QueueErrors int64 `json:"queue_errors"` } func New(s *store.Store, idx *indexer.Indexer) *Watcher { @@ -45,24 +69,45 @@ func New(s *store.Store, idx *indexer.Indexer) *Watcher { func (w *Watcher) Stats() WatchStats { return WatchStats{ + EventsSeen: w.eventsSeen.Load(), + EventsIgnored: w.eventsIgnored.Load(), + EventsIgnoredDir: w.eventsIgnoredDir.Load(), + EventsIgnoredChmod: w.eventsIgnoredChmod.Load(), + FlushSignals: w.flushSignals.Load(), CoalescedSignals: w.coalescedSignals.Load(), FlushRuns: w.flushRuns.Load(), FlushErrors: w.flushErrors.Load(), - QueueExecs: w.queueExecs.Load(), - QueueSkipped: w.queueSkipped.Load(), - QueueErrors: w.queueErrors.Load(), + FlushNoop: w.flushNoop.Load(), + + DrainRuns: w.drainRuns.Load(), + DrainPaths: w.drainPaths.Load(), + UpdateRuns: w.updateRuns.Load(), + UpdatePaths: w.updatePaths.Load(), + + QueueExecs: w.queueExecs.Load(), + QueueSkipped: w.queueSkipped.Load(), + QueueErrors: w.queueErrors.Load(), } } func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, debounce time.Duration) error { + w.eventsSeen.Store(0) + w.eventsIgnored.Store(0) + w.eventsIgnoredDir.Store(0) + w.eventsIgnoredChmod.Store(0) w.flushSignals.Store(0) w.coalescedSignals.Store(0) w.flushRuns.Store(0) w.flushErrors.Store(0) + w.flushNoop.Store(0) w.queueExecs.Store(0) w.queueSkipped.Store(0) w.queueErrors.Store(0) + w.drainRuns.Store(0) + w.drainPaths.Store(0) + w.updateRuns.Store(0) + w.updatePaths.Store(0) if debounce <= 0 { debounce = 750 * time.Millisecond } @@ -100,18 +145,25 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun } flush := func() error { + w.drainRuns.Add(1) paths, err := w.store.DrainDirtyFiles(ctx, repoID) if err != nil { return err } if len(paths) == 0 { + w.flushNoop.Add(1) return nil } + w.drainPaths.Add(int64(len(paths))) _, err = w.indexer.Update(ctx, indexer.Options{ RepoRoot: repoRoot, Paths: paths, ScanKind: "watch", }) + if err == nil { + w.updateRuns.Add(1) + w.updatePaths.Add(int64(len(paths))) + } return err } @@ -176,12 +228,22 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun if !ok { return nil } + w.eventsSeen.Add(1) + + // Avoid churn from chmod-only events (common on some editors). + if event.Op == fsnotify.Chmod { + w.eventsIgnored.Add(1) + w.eventsIgnoredChmod.Add(1) + continue + } rel, err := filepath.Rel(repoRoot, event.Name) if err != nil { + w.eventsIgnored.Add(1) continue } rel = filepath.Clean(rel) if shouldIgnorePath(rel) { + w.eventsIgnored.Add(1) continue } if event.Op&fsnotify.Create == fsnotify.Create { @@ -189,6 +251,10 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun if err := addWatchTree(event.Name); err != nil { return err } + // Directory creates are not indexable file updates. + w.eventsIgnored.Add(1) + w.eventsIgnoredDir.Add(1) + continue } } From e02d5879096e6993525393e83cf6cd92b71c5331 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:19:40 +0200 Subject: [PATCH 21/59] maintenance: add FTS optimize to clean and expand doctor DB diagnostics (#37) * maintenance: add FTS optimize to clean and expand doctor DB diagnostics * doctor/clean: reuse store pragma inspection and dedupe repo-root parsing --- internal/cli/app.go | 130 ++++++++++++++++++++++++++------- internal/cli/commands.go | 7 +- internal/doctor/doctor.go | 54 +++++++++++++- internal/doctor/doctor_test.go | 2 +- internal/store/store.go | 72 ++++++++++++++++++ 5 files changed, 233 insertions(+), 32 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index db2600e..d0eeb4c 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -103,14 +103,55 @@ func hasHelpFlag(args []string) bool { return false } -func runDoctor(stdout io.Writer, args []string) error { +func parseOptionalRepoRootArg(fs *flag.FlagSet, args []string, repoRootFlag *string, defaultRepoRoot string) (string, error) { + repoRootArg := "" + parseArgs := args + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + repoRootArg = args[0] + parseArgs = args[1:] + } + if err := fs.Parse(parseArgs); err != nil { + return "", err + } + repoRoot := strings.TrimSpace(*repoRootFlag) + if repoRoot == "" { + repoRoot = strings.TrimSpace(repoRootArg) + } + if repoRoot == "" { + repoRoot = defaultRepoRoot + } + return repoRoot, nil +} + +func runDoctor(ctx context.Context, cfg config.Config, stdout io.Writer, args []string) error { fs := flag.NewFlagSet("doctor", flag.ContinueOnError) fs.SetOutput(io.Discard) fix := fs.Bool("fix", false, "apply non-destructive fixes") - if err := fs.Parse(args); err != nil { + repoRootFlag := fs.String("repo-root", "", "repository root to inspect (optional)") + + defaultRepoRoot := "" + if config.IsRepoDBDir(cfg.DBDir) { + defaultRepoRoot = "." + } + repoRoot, err := parseOptionalRepoRootArg(fs, args, repoRootFlag, defaultRepoRoot) + if err != nil { return err } - report, err := doctor.RunWithFix(*fix) + + dbPath := "" + if repoRoot != "" { + canonical, err := store.CanonicalRepoPath(repoRoot) + if err != nil { + return err + } + p, err := dbPathForRepo(cfg, repoRoot, canonical) + if err != nil { + return err + } + dbPath = p + } + + report, err := doctor.RunWithFix(*fix, dbPath) if err != nil { return err } @@ -1021,28 +1062,34 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s fs := flag.NewFlagSet("clean", flag.ContinueOnError) fs.SetOutput(io.Discard) vacuum := fs.Bool("vacuum", false, "run VACUUM on databases") + ftsOptimize := fs.Bool("fts-optimize", false, "run FTS optimize on databases (symbol_fts)") repoRootFlag := fs.String("repo-root", "", "repository root to clean") - if err := fs.Parse(args); err != nil { - return err - } - repoRoot := strings.TrimSpace(*repoRootFlag) - if repoRoot == "" && fs.NArg() > 0 { - repoRoot = fs.Arg(0) + + defaultRepoRoot := "" + if config.IsRepoDBDir(cfg.DBDir) { + defaultRepoRoot = "." } - if repoRoot == "" && config.IsRepoDBDir(cfg.DBDir) { - repoRoot = "." + repoRoot, err := parseOptionalRepoRootArg(fs, args, repoRootFlag, defaultRepoRoot) + if err != nil { + return err } type dbResult struct { Path string `json:"path"` Action string `json:"action"` + SizeBefore int64 `json:"size_before_bytes"` + SizeAfter int64 `json:"size_after_bytes"` ReclaimedBytes int64 `json:"reclaimed_bytes"` + Vacuumed bool `json:"vacuumed,omitempty"` + FTSOptimized bool `json:"fts_optimized,omitempty"` + FTSOptimizeMS int64 `json:"fts_optimize_ms,omitempty"` CanonicalRepo string `json:"canonical_repo,omitempty"` Error string `json:"error,omitempty"` } report := map[string]any{ - "vacuum": *vacuum, - "dbs": []dbResult{}, + "vacuum": *vacuum, + "fts_optimize": *ftsOptimize, + "dbs": []dbResult{}, } results := make([]dbResult, 0) var reclaimed int64 @@ -1058,24 +1105,37 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s } res := dbResult{Path: dbPath, CanonicalRepo: canonical} before := fileSize(dbPath) + res.SizeBefore = before s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: cfg.DBPerformanceProfile}) if err != nil { return err } defer s.Close() + if *ftsOptimize { + dur, err := s.OptimizeFTS(ctx) + if err != nil { + return err + } + res.FTSOptimized = true + res.FTSOptimizeMS = dur.Milliseconds() + } if *vacuum { if err := s.Vacuum(ctx); err != nil { return err } - after := fileSize(dbPath) - if before > after { - res.ReclaimedBytes = before - after - reclaimed += res.ReclaimedBytes - } + res.Vacuumed = true res.Action = "vacuumed" + } else if *ftsOptimize { + res.Action = "fts_optimized" } else { res.Action = "inspected" } + after := fileSize(dbPath) + res.SizeAfter = after + if before > after { + res.ReclaimedBytes = before - after + reclaimed += res.ReclaimedBytes + } results = append(results, res) report["dbs"] = results report["reclaimed_bytes"] = reclaimed @@ -1095,7 +1155,7 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s } dbPath := filepath.Join(cfg.DBDir, entry.Name()) sizeBefore := fileSize(dbPath) - res := dbResult{Path: dbPath} + res := dbResult{Path: dbPath, SizeBefore: sizeBefore} s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: cfg.DBPerformanceProfile}) if err != nil { res.Action = "skipped" @@ -1138,6 +1198,24 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s results = append(results, res) continue } + if !*vacuum { + res.Action = "kept" + } + if *ftsOptimize { + dur, err := s.OptimizeFTS(ctx) + if err != nil { + _ = s.Close() + res.Action = "skipped" + res.Error = err.Error() + results = append(results, res) + continue + } + res.FTSOptimized = true + res.FTSOptimizeMS = dur.Milliseconds() + if !*vacuum && res.Action == "kept" { + res.Action = "fts_optimized" + } + } if *vacuum { if err := s.Vacuum(ctx); err != nil { _ = s.Close() @@ -1146,16 +1224,16 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s results = append(results, res) continue } - after := fileSize(dbPath) - if sizeBefore > after { - res.ReclaimedBytes = sizeBefore - after - reclaimed += res.ReclaimedBytes - } + res.Vacuumed = true res.Action = "vacuumed" - } else { - res.Action = "kept" } _ = s.Close() + sizeAfter := fileSize(dbPath) + res.SizeAfter = sizeAfter + if sizeBefore > sizeAfter { + res.ReclaimedBytes = sizeBefore - sizeAfter + reclaimed += res.ReclaimedBytes + } results = append(results, res) } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 582dd18..7f6861e 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -256,10 +256,11 @@ func newCommandList() []*command { description: "run diagnostics", usageLines: []string{ " doctor", + " add --repo-root PATH to inspect a repo DB", " add --fix for non-destructive autofixes", }, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { - return runDoctor(stdout, args) + return runDoctor(ctx, cfg, stdout, args) }, }, { @@ -324,13 +325,15 @@ func newCommandList() []*command { { name: "clean", description: "clean index data", - usageLines: []string{" clean [repo-path] [--vacuum]"}, + usageLines: []string{" clean [repo-path] [--vacuum] [--fts-optimize]"}, flags: []commandFlag{ {name: "--vacuum", description: "VACUUM the database after cleanup"}, + {name: "--fts-optimize", description: "run FTS optimize (symbol_fts)"}, }, examples: []string{ "codegraph clean .", "codegraph clean . --vacuum", + "codegraph clean . --fts-optimize", }, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { return runClean(ctx, cfg, stdout, args) diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 287eca4..cf4fa46 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -1,17 +1,21 @@ package doctor import ( + "context" + "database/sql" "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" + "strings" "github.com/isink17/codegraph/internal/appname" "github.com/isink17/codegraph/internal/config" "github.com/isink17/codegraph/internal/gotool" "github.com/isink17/codegraph/internal/platform" + "github.com/isink17/codegraph/internal/store" ) type Report struct { @@ -22,15 +26,23 @@ type Report struct { CacheDir string `json:"cache_dir"` CodegraphOnPath bool `json:"codegraph_on_path"` CodegraphPath string `json:"codegraph_path,omitempty"` + SQLiteDriver string `json:"sqlite_driver"` + DB *DBInfo `json:"db,omitempty"` AppliedFixes []string `json:"applied_fixes,omitempty"` Recommendations []string `json:"recommendations,omitempty"` } +type DBInfo struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + Pragmas store.DBPragmas `json:"pragmas"` +} + func Run() (Report, error) { - return RunWithFix(false) + return RunWithFix(false, "") } -func RunWithFix(fix bool) (Report, error) { +func RunWithFix(fix bool, dbPath string) (Report, error) { paths, err := platform.DefaultPaths() if err != nil { return Report{}, err @@ -44,7 +56,9 @@ func RunWithFix(fix bool) (Report, error) { binaryPath, lookErr := exec.LookPath("codegraph") onPath := lookErr == nil if lookErr != nil && !errors.Is(lookErr, exec.ErrNotFound) { - return Report{}, lookErr + // Be conservative: doctor should still run even if the PATH lookup found something unusable. + onPath = false + binaryPath = "" } appliedFixes := []string{} if fix { @@ -81,6 +95,16 @@ func RunWithFix(fix bool) (Report, error) { recommendations = append(recommendations, "verify after reopening shell: "+gotool.VerifyCommandHint(appname.BinaryName)) } + var dbInfo *DBInfo + if strings.TrimSpace(dbPath) != "" { + info, err := inspectDB(context.Background(), dbPath) + if err != nil { + recommendations = append(recommendations, "repo DB inspect failed: "+err.Error()) + } else { + dbInfo = info + } + } + return Report{ GOOS: runtime.GOOS, ConfigPath: filepath.Clean(configPath), @@ -89,6 +113,8 @@ func RunWithFix(fix bool) (Report, error) { CacheDir: paths.CacheDir, CodegraphOnPath: onPath, CodegraphPath: binaryPath, + SQLiteDriver: store.SQLiteDriverName(), + DB: dbInfo, AppliedFixes: appliedFixes, Recommendations: recommendations, }, nil @@ -101,3 +127,25 @@ func firstPathHint() string { } return hints[0] } + +func inspectDB(ctx context.Context, dbPath string) (*DBInfo, error) { + st, err := os.Stat(dbPath) + if err != nil { + return nil, err + } + db, err := sql.Open(store.SQLiteDriverName(), dbPath) + if err != nil { + return nil, err + } + defer db.Close() + pragmas, err := store.QueryDBPragmas(ctx, db) + if err != nil { + return nil, err + } + + return &DBInfo{ + Path: filepath.Clean(dbPath), + SizeBytes: st.Size(), + Pragmas: pragmas, + }, nil +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 2f43890..f05298b 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -58,7 +58,7 @@ func TestRunReportsCodegraphMissingFromPath(t *testing.T) { func TestRunWithFixCreatesConfig(t *testing.T) { home := filepath.Join(t.TempDir(), "codegraph-home") t.Setenv("CODEGRAPH_HOME", home) - report, err := RunWithFix(true) + report, err := RunWithFix(true, "") if err != nil { t.Fatalf("RunWithFix(true) error = %v", err) } diff --git a/internal/store/store.go b/internal/store/store.go index dabdde1..3a3d764 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -503,6 +503,78 @@ func (s *Store) Vacuum(ctx context.Context) error { return err } +func (s *Store) OptimizeFTS(ctx context.Context) (time.Duration, error) { + start := time.Now() + _, err := s.db.ExecContext(ctx, `INSERT INTO symbol_fts(symbol_fts) VALUES('optimize')`) + return time.Since(start), err +} + +type DBPragmas struct { + SQLiteVersion string `json:"sqlite_version"` + JournalMode string `json:"journal_mode"` + Synchronous string `json:"synchronous"` + TempStore string `json:"temp_store"` + AutoVacuum int64 `json:"auto_vacuum"` + PageSize int64 `json:"page_size"` + BusyTimeoutMS int64 `json:"busy_timeout_ms"` + ForeignKeys bool `json:"foreign_keys"` + WalAutocheckpoint int64 `json:"wal_autocheckpoint"` + UserVersion int64 `json:"user_version"` + SymbolFTSPresent bool `json:"symbol_fts_present"` +} + +func (s *Store) DBPragmas(ctx context.Context) (DBPragmas, error) { + return QueryDBPragmas(ctx, s.db) +} + +func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { + var out DBPragmas + + if err := db.QueryRowContext(ctx, `SELECT sqlite_version()`).Scan(&out.SQLiteVersion); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA journal_mode`).Scan(&out.JournalMode); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA synchronous`).Scan(&out.Synchronous); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA temp_store`).Scan(&out.TempStore); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&out.AutoVacuum); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA page_size`).Scan(&out.PageSize); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA busy_timeout`).Scan(&out.BusyTimeoutMS); err != nil { + return DBPragmas{}, err + } + var foreignKeys int64 + if err := db.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil { + return DBPragmas{}, err + } + out.ForeignKeys = foreignKeys != 0 + if err := db.QueryRowContext(ctx, `PRAGMA wal_autocheckpoint`).Scan(&out.WalAutocheckpoint); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&out.UserVersion); err != nil { + return DBPragmas{}, err + } + + var symbolFTSName string + err := db.QueryRowContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND name='symbol_fts'`).Scan(&symbolFTSName) + if err == nil && symbolFTSName == "symbol_fts" { + out.SymbolFTSPresent = true + } else if errors.Is(err, sql.ErrNoRows) { + out.SymbolFTSPresent = false + } else if err != nil { + return DBPragmas{}, err + } + return out, nil +} + func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]FileRecord, error) { rows, err := s.db.QueryContext(ctx, ` SELECT id, path, language, size_bytes, mtime_unix_ns, content_sha256, is_deleted From 1c16147e24cb3beffd8e68a724337746a7804fb5 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:27:04 +0200 Subject: [PATCH 22/59] cli: disable HTML escaping in JSON/JSONL output for schema stability (#38) --- internal/cli/app.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/cli/app.go b/internal/cli/app.go index d0eeb4c..ad94542 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1323,12 +1323,14 @@ type graphRepo struct { func writeJSON(w io.Writer, v any) error { enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) enc.SetIndent("", " ") return enc.Encode(v) } func writeJSONL(w io.Writer, v any) error { enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) return enc.Encode(v) } From b4718efd3162c79010ff56f21d5cd1652d9514c0 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:37:49 +0200 Subject: [PATCH 23/59] clean/doctor: add explicit ANALYZE, WAL checkpoint, incremental vacuum; add doctor --deep integrity checks (#39) * clean/doctor: add explicit ANALYZE, WAL checkpoint, incremental vacuum; add doctor --deep integrity checks * clean: dedupe maintenance action execution; clarify incremental_vacuum auto_vacuum mode errors --- internal/cli/app.go | 128 +++++++++++++++++++++++++++++++++----- internal/cli/commands.go | 8 ++- internal/doctor/doctor.go | 121 ++++++++++++++++++++++++++++++----- internal/store/store.go | 69 ++++++++++++++++++++ 4 files changed, 297 insertions(+), 29 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index ad94542..8cb1a36 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -127,6 +127,7 @@ func runDoctor(ctx context.Context, cfg config.Config, stdout io.Writer, args [] fs := flag.NewFlagSet("doctor", flag.ContinueOnError) fs.SetOutput(io.Discard) fix := fs.Bool("fix", false, "apply non-destructive fixes") + deep := fs.Bool("deep", false, "run deeper (potentially slow) diagnostics, including integrity_check") repoRootFlag := fs.String("repo-root", "", "repository root to inspect (optional)") defaultRepoRoot := "" @@ -151,7 +152,7 @@ func runDoctor(ctx context.Context, cfg config.Config, stdout io.Writer, args [] dbPath = p } - report, err := doctor.RunWithFix(*fix, dbPath) + report, err := doctor.RunWithOptions(doctor.Options{Fix: *fix, DBPath: dbPath, Deep: *deep}) if err != nil { return err } @@ -1063,6 +1064,9 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s fs.SetOutput(io.Discard) vacuum := fs.Bool("vacuum", false, "run VACUUM on databases") ftsOptimize := fs.Bool("fts-optimize", false, "run FTS optimize on databases (symbol_fts)") + analyze := fs.Bool("analyze", false, "run ANALYZE on databases") + walCheckpointTruncate := fs.Bool("wal-checkpoint-truncate", false, "run PRAGMA wal_checkpoint(TRUNCATE) on databases") + incrementalVacuum := fs.Bool("incremental-vacuum", false, "run PRAGMA incremental_vacuum (requires auto_vacuum=INCREMENTAL)") repoRootFlag := fs.String("repo-root", "", "repository root to clean") defaultRepoRoot := "" @@ -1075,25 +1079,64 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s } type dbResult struct { - Path string `json:"path"` - Action string `json:"action"` - SizeBefore int64 `json:"size_before_bytes"` - SizeAfter int64 `json:"size_after_bytes"` - ReclaimedBytes int64 `json:"reclaimed_bytes"` - Vacuumed bool `json:"vacuumed,omitempty"` - FTSOptimized bool `json:"fts_optimized,omitempty"` - FTSOptimizeMS int64 `json:"fts_optimize_ms,omitempty"` - CanonicalRepo string `json:"canonical_repo,omitempty"` - Error string `json:"error,omitempty"` + Path string `json:"path"` + Action string `json:"action"` + SizeBefore int64 `json:"size_before_bytes"` + SizeAfter int64 `json:"size_after_bytes"` + ReclaimedBytes int64 `json:"reclaimed_bytes"` + Vacuumed bool `json:"vacuumed,omitempty"` + FTSOptimized bool `json:"fts_optimized,omitempty"` + FTSOptimizeMS int64 `json:"fts_optimize_ms,omitempty"` + Analyzed bool `json:"analyzed,omitempty"` + AnalyzeMS int64 `json:"analyze_ms,omitempty"` + WALCheckpoint *store.WalCheckpointResult `json:"wal_checkpoint,omitempty"` + IncVacuumed bool `json:"incremental_vacuumed,omitempty"` + IncVacuumMS int64 `json:"incremental_vacuum_ms,omitempty"` + IncVacuumBeforeFreelist int64 `json:"incremental_vacuum_before_freelist_pages,omitempty"` + IncVacuumAfterFreelist int64 `json:"incremental_vacuum_after_freelist_pages,omitempty"` + CanonicalRepo string `json:"canonical_repo,omitempty"` + Error string `json:"error,omitempty"` } report := map[string]any{ - "vacuum": *vacuum, - "fts_optimize": *ftsOptimize, - "dbs": []dbResult{}, + "vacuum": *vacuum, + "fts_optimize": *ftsOptimize, + "analyze": *analyze, + "wal_checkpoint_truncate": *walCheckpointTruncate, + "incremental_vacuum": *incrementalVacuum, + "dbs": []dbResult{}, } results := make([]dbResult, 0) var reclaimed int64 + runAnalyze := func(ctx context.Context, s *store.Store, res *dbResult) error { + dur, err := s.Analyze(ctx) + if err != nil { + return err + } + res.Analyzed = true + res.AnalyzeMS = dur.Milliseconds() + return nil + } + runWalCheckpointTruncate := func(ctx context.Context, s *store.Store, res *dbResult) error { + walRes, err := s.WalCheckpointTruncate(ctx) + if err != nil { + return err + } + res.WALCheckpoint = &walRes + return nil + } + runIncrementalVacuumAll := func(ctx context.Context, s *store.Store, res *dbResult) error { + beforePages, afterPages, dur, err := s.IncrementalVacuumAll(ctx) + if err != nil { + return err + } + res.IncVacuumed = true + res.IncVacuumMS = dur.Milliseconds() + res.IncVacuumBeforeFreelist = beforePages + res.IncVacuumAfterFreelist = afterPages + return nil + } + if repoRoot != "" { canonical, err := store.CanonicalRepoPath(repoRoot) if err != nil { @@ -1111,6 +1154,11 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s return err } defer s.Close() + if *analyze { + if err := runAnalyze(ctx, s, &res); err != nil { + return err + } + } if *ftsOptimize { dur, err := s.OptimizeFTS(ctx) if err != nil { @@ -1119,12 +1167,28 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s res.FTSOptimized = true res.FTSOptimizeMS = dur.Milliseconds() } + if *walCheckpointTruncate { + if err := runWalCheckpointTruncate(ctx, s, &res); err != nil { + return err + } + } + if *incrementalVacuum { + if err := runIncrementalVacuumAll(ctx, s, &res); err != nil { + return err + } + } if *vacuum { if err := s.Vacuum(ctx); err != nil { return err } res.Vacuumed = true res.Action = "vacuumed" + } else if *incrementalVacuum { + res.Action = "incremental_vacuumed" + } else if *walCheckpointTruncate { + res.Action = "wal_checkpointed" + } else if *analyze { + res.Action = "analyzed" } else if *ftsOptimize { res.Action = "fts_optimized" } else { @@ -1201,6 +1265,18 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s if !*vacuum { res.Action = "kept" } + if *analyze { + if err := runAnalyze(ctx, s, &res); err != nil { + _ = s.Close() + res.Action = "skipped" + res.Error = err.Error() + results = append(results, res) + continue + } + if !*vacuum && !*ftsOptimize && res.Action == "kept" { + res.Action = "analyzed" + } + } if *ftsOptimize { dur, err := s.OptimizeFTS(ctx) if err != nil { @@ -1216,6 +1292,30 @@ func runClean(ctx context.Context, cfg config.Config, stdout io.Writer, args []s res.Action = "fts_optimized" } } + if *walCheckpointTruncate { + if err := runWalCheckpointTruncate(ctx, s, &res); err != nil { + _ = s.Close() + res.Action = "skipped" + res.Error = err.Error() + results = append(results, res) + continue + } + if !*vacuum && !*ftsOptimize && !*analyze && res.Action == "kept" { + res.Action = "wal_checkpointed" + } + } + if *incrementalVacuum { + if err := runIncrementalVacuumAll(ctx, s, &res); err != nil { + _ = s.Close() + res.Action = "skipped" + res.Error = err.Error() + results = append(results, res) + continue + } + if !*vacuum && !*ftsOptimize && !*analyze && !*walCheckpointTruncate && res.Action == "kept" { + res.Action = "incremental_vacuumed" + } + } if *vacuum { if err := s.Vacuum(ctx); err != nil { _ = s.Close() diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 7f6861e..53b0f60 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -258,6 +258,7 @@ func newCommandList() []*command { " doctor", " add --repo-root PATH to inspect a repo DB", " add --fix for non-destructive autofixes", + " add --deep for slower DB checks (integrity_check)", }, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { return runDoctor(ctx, cfg, stdout, args) @@ -325,15 +326,20 @@ func newCommandList() []*command { { name: "clean", description: "clean index data", - usageLines: []string{" clean [repo-path] [--vacuum] [--fts-optimize]"}, + usageLines: []string{" clean [repo-path] [--vacuum] [--fts-optimize] [--analyze] [--wal-checkpoint-truncate] [--incremental-vacuum]"}, flags: []commandFlag{ {name: "--vacuum", description: "VACUUM the database after cleanup"}, {name: "--fts-optimize", description: "run FTS optimize (symbol_fts)"}, + {name: "--analyze", description: "run ANALYZE"}, + {name: "--wal-checkpoint-truncate", description: "run PRAGMA wal_checkpoint(TRUNCATE)"}, + {name: "--incremental-vacuum", description: "run PRAGMA incremental_vacuum (requires auto_vacuum=INCREMENTAL)"}, }, examples: []string{ "codegraph clean .", "codegraph clean . --vacuum", "codegraph clean . --fts-optimize", + "codegraph clean . --analyze", + "codegraph clean . --wal-checkpoint-truncate", }, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { return runClean(ctx, cfg, stdout, args) diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index cf4fa46..9155f94 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -19,17 +19,18 @@ import ( ) type Report struct { - GOOS string `json:"goos"` - ConfigPath string `json:"config_path"` - ConfigExists bool `json:"config_exists"` - DataDir string `json:"data_dir"` - CacheDir string `json:"cache_dir"` - CodegraphOnPath bool `json:"codegraph_on_path"` - CodegraphPath string `json:"codegraph_path,omitempty"` - SQLiteDriver string `json:"sqlite_driver"` - DB *DBInfo `json:"db,omitempty"` - AppliedFixes []string `json:"applied_fixes,omitempty"` - Recommendations []string `json:"recommendations,omitempty"` + GOOS string `json:"goos"` + ConfigPath string `json:"config_path"` + ConfigExists bool `json:"config_exists"` + DataDir string `json:"data_dir"` + CacheDir string `json:"cache_dir"` + CodegraphOnPath bool `json:"codegraph_on_path"` + CodegraphPath string `json:"codegraph_path,omitempty"` + SQLiteDriver string `json:"sqlite_driver"` + DB *DBInfo `json:"db,omitempty"` + Deep *DeepInfo `json:"deep,omitempty"` + AppliedFixes []string `json:"applied_fixes,omitempty"` + Recommendations []string `json:"recommendations,omitempty"` } type DBInfo struct { @@ -38,11 +39,32 @@ type DBInfo struct { Pragmas store.DBPragmas `json:"pragmas"` } +type DeepInfo struct { + DB *DBDeepInfo `json:"db,omitempty"` +} + +type DBDeepInfo struct { + IntegrityOK bool `json:"integrity_ok"` + IntegrityMessages []string `json:"integrity_messages,omitempty"` + IntegrityTruncated bool `json:"integrity_truncated,omitempty"` + ForeignKeyIssues int64 `json:"foreign_key_issues,omitempty"` +} + func Run() (Report, error) { return RunWithFix(false, "") } func RunWithFix(fix bool, dbPath string) (Report, error) { + return RunWithOptions(Options{Fix: fix, DBPath: dbPath}) +} + +type Options struct { + Fix bool + DBPath string + Deep bool +} + +func RunWithOptions(opts Options) (Report, error) { paths, err := platform.DefaultPaths() if err != nil { return Report{}, err @@ -61,7 +83,7 @@ func RunWithFix(fix bool, dbPath string) (Report, error) { binaryPath = "" } appliedFixes := []string{} - if fix { + if opts.Fix { defaultCfg, err := config.Default() if err != nil { return Report{}, err @@ -96,13 +118,22 @@ func RunWithFix(fix bool, dbPath string) (Report, error) { } var dbInfo *DBInfo - if strings.TrimSpace(dbPath) != "" { - info, err := inspectDB(context.Background(), dbPath) + var deepInfo *DeepInfo + if strings.TrimSpace(opts.DBPath) != "" { + info, err := inspectDB(context.Background(), opts.DBPath) if err != nil { recommendations = append(recommendations, "repo DB inspect failed: "+err.Error()) } else { dbInfo = info } + if opts.Deep { + di, err := inspectDeepDB(context.Background(), opts.DBPath) + if err != nil { + recommendations = append(recommendations, "repo DB deep inspect failed: "+err.Error()) + } else { + deepInfo = &DeepInfo{DB: di} + } + } } return Report{ @@ -115,6 +146,7 @@ func RunWithFix(fix bool, dbPath string) (Report, error) { CodegraphPath: binaryPath, SQLiteDriver: store.SQLiteDriverName(), DB: dbInfo, + Deep: deepInfo, AppliedFixes: appliedFixes, Recommendations: recommendations, }, nil @@ -149,3 +181,64 @@ func inspectDB(ctx context.Context, dbPath string) (*DBInfo, error) { Pragmas: pragmas, }, nil } + +func inspectDeepDB(ctx context.Context, dbPath string) (*DBDeepInfo, error) { + db, err := sql.Open(store.SQLiteDriverName(), dbPath) + if err != nil { + return nil, err + } + defer db.Close() + + // integrity_check can return many rows; keep this bounded and human-usable. + const maxIntegrityMessages = 20 + integrityMessages := make([]string, 0) + truncated := false + rows, err := db.QueryContext(ctx, `PRAGMA integrity_check`) + if err != nil { + return nil, err + } + for rows.Next() { + var msg string + if err := rows.Scan(&msg); err != nil { + rows.Close() + return nil, err + } + if len(integrityMessages) < maxIntegrityMessages { + integrityMessages = append(integrityMessages, msg) + } else { + truncated = true + } + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + + integrityOK := len(integrityMessages) == 1 && integrityMessages[0] == "ok" + if integrityOK { + integrityMessages = nil + truncated = false + } + + var foreignKeyIssues int64 + fkRows, err := db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return nil, err + } + for fkRows.Next() { + foreignKeyIssues++ + } + if err := fkRows.Err(); err != nil { + fkRows.Close() + return nil, err + } + fkRows.Close() + + return &DBDeepInfo{ + IntegrityOK: integrityOK, + IntegrityMessages: integrityMessages, + IntegrityTruncated: truncated, + ForeignKeyIssues: foreignKeyIssues, + }, nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 3a3d764..a4c6ff9 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -509,6 +509,67 @@ func (s *Store) OptimizeFTS(ctx context.Context) (time.Duration, error) { return time.Since(start), err } +type WalCheckpointResult struct { + Busy int64 `json:"busy"` + LogFrames int64 `json:"log_frames"` + CkptFrames int64 `json:"checkpointed_frames"` + Mode string `json:"mode"` + DurationMS int64 `json:"duration_ms"` +} + +func (s *Store) Analyze(ctx context.Context) (time.Duration, error) { + start := time.Now() + _, err := s.db.ExecContext(ctx, `ANALYZE`) + return time.Since(start), err +} + +func (s *Store) WalCheckpointTruncate(ctx context.Context) (WalCheckpointResult, error) { + start := time.Now() + var busy, logFrames, ckptFrames int64 + if err := s.db.QueryRowContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`).Scan(&busy, &logFrames, &ckptFrames); err != nil { + return WalCheckpointResult{}, err + } + return WalCheckpointResult{ + Busy: busy, + LogFrames: logFrames, + CkptFrames: ckptFrames, + Mode: "TRUNCATE", + DurationMS: time.Since(start).Milliseconds(), + }, nil +} + +func (s *Store) IncrementalVacuumAll(ctx context.Context) (beforeFreelist, afterFreelist int64, dur time.Duration, err error) { + before, err := s.DBPragmas(ctx) + if err != nil { + return 0, 0, 0, err + } + // SQLite auto_vacuum modes: + // 0 = NONE + // 1 = FULL + // 2 = INCREMENTAL (required for PRAGMA incremental_vacuum to reclaim pages) + switch before.AutoVacuum { + case 2: + // ok + case 0: + return 0, 0, 0, fmt.Errorf("incremental vacuum requires PRAGMA auto_vacuum=INCREMENTAL (2); database is auto_vacuum=NONE (0)") + case 1: + return 0, 0, 0, fmt.Errorf("incremental vacuum requires PRAGMA auto_vacuum=INCREMENTAL (2); database is auto_vacuum=FULL (1)") + default: + return 0, 0, 0, fmt.Errorf("incremental vacuum requires PRAGMA auto_vacuum=INCREMENTAL (2); got auto_vacuum=%d", before.AutoVacuum) + } + + start := time.Now() + // PRAGMA incremental_vacuum without an argument attempts to remove all pages from the freelist. + if _, err := s.db.ExecContext(ctx, `PRAGMA incremental_vacuum`); err != nil { + return 0, 0, 0, err + } + after, err := s.DBPragmas(ctx) + if err != nil { + return 0, 0, 0, err + } + return before.FreelistCount, after.FreelistCount, time.Since(start), nil +} + type DBPragmas struct { SQLiteVersion string `json:"sqlite_version"` JournalMode string `json:"journal_mode"` @@ -516,6 +577,8 @@ type DBPragmas struct { TempStore string `json:"temp_store"` AutoVacuum int64 `json:"auto_vacuum"` PageSize int64 `json:"page_size"` + PageCount int64 `json:"page_count"` + FreelistCount int64 `json:"freelist_count"` BusyTimeoutMS int64 `json:"busy_timeout_ms"` ForeignKeys bool `json:"foreign_keys"` WalAutocheckpoint int64 `json:"wal_autocheckpoint"` @@ -548,6 +611,12 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { if err := db.QueryRowContext(ctx, `PRAGMA page_size`).Scan(&out.PageSize); err != nil { return DBPragmas{}, err } + if err := db.QueryRowContext(ctx, `PRAGMA page_count`).Scan(&out.PageCount); err != nil { + return DBPragmas{}, err + } + if err := db.QueryRowContext(ctx, `PRAGMA freelist_count`).Scan(&out.FreelistCount); err != nil { + return DBPragmas{}, err + } if err := db.QueryRowContext(ctx, `PRAGMA busy_timeout`).Scan(&out.BusyTimeoutMS); err != nil { return DBPragmas{}, err } From 91034bb3e42983e240688d279217d3443f2f5c17 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:04:30 +0200 Subject: [PATCH 24/59] cli/index: stabilize --jsonl scan payloads (add scan_kind, include parse_ms) (#40) * cli/index: stabilize --jsonl scan payloads (add scan_kind, include parse_ms) * indexer: fix parse_ms semantics and drop redundant adapter_parse_ms from scan summaries --- internal/cli/app.go | 10 ++++++---- internal/indexer/indexer.go | 6 +++--- internal/store/store.go | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index 8cb1a36..6e78e38 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -626,10 +626,12 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName return err } if jsonl { + scanKind := map[bool]string{true: "update", false: "index"}[update] if err := writeJSONL(stdout, map[string]any{ - "type": "scan_summary", - "command": map[bool]string{true: "update", false: "index"}[update], - "data": summary, + "type": "scan_summary", + "command": scanKind, + "scan_kind": scanKind, + "data": summary, }); err != nil { return err } @@ -641,9 +643,9 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName "process_wall_ms": summary.ProcessWallMS, "task_ms": summary.TaskMS, "task_other_ms": summary.TaskOtherMS, + "parse_ms": summary.ParseMS, "read_ms": summary.ReadMS, "hash_ms": summary.HashMS, - "adapter_parse_ms": summary.AdapterParseMS, "write_ms": summary.WriteMS, "write_metadata_ms": summary.WriteMetadataMS, "write_replace_ms": summary.WriteReplaceMS, diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index c026b3d..677eccd 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -529,10 +529,10 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err summary.ProcessWallMS = time.Since(processWallStart).Milliseconds() summary.TaskMS = taskDur.Milliseconds() summary.TaskOtherMS = taskOtherDur.Milliseconds() - summary.ParseMS = taskDur.Milliseconds() + // ParseMS is the time spent parsing file content (adapter parse), excluding read/hash/DB writes. + summary.ParseMS = adapterParseDur.Milliseconds() summary.ReadMS = readDur.Milliseconds() summary.HashMS = hashDur.Milliseconds() - summary.AdapterParseMS = adapterParseDur.Milliseconds() summary.WriteMS = writeDur.Milliseconds() summary.WriteMetadataMS = writeMetadataDur.Milliseconds() summary.WriteReplaceMS = writeReplaceDur.Milliseconds() @@ -623,7 +623,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err {Phase: "task_other", MS: summary.TaskOtherMS}, {Phase: "read", MS: summary.ReadMS}, {Phase: "hash", MS: summary.HashMS}, - {Phase: "adapter_parse", MS: summary.AdapterParseMS}, + {Phase: "parse", MS: summary.ParseMS}, {Phase: "write", MS: summary.WriteMS}, {Phase: "write_metadata", MS: summary.WriteMetadataMS}, {Phase: "write_replace", MS: summary.WriteReplaceMS}, diff --git a/internal/store/store.go b/internal/store/store.go index a4c6ff9..25eee6e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -113,7 +113,6 @@ type ScanSummary struct { ParseMS int64 `json:"parse_ms,omitempty"` ReadMS int64 `json:"read_ms,omitempty"` HashMS int64 `json:"hash_ms,omitempty"` - AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` WriteMS int64 `json:"write_ms,omitempty"` WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` From 8ac3fa7e658ec2a4f39e7068e2e41053dbde6d88 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:14:54 +0200 Subject: [PATCH 25/59] cli/index: stabilize --jsonl scan event envelopes with repo/scan correlation fields (#41) * cli/index: stabilize --jsonl scan event envelopes with repo/scan correlation fields * cli/index: dedupe JSONL scan event envelopes and handle scan_phases write errors --- internal/cli/app.go | 67 +++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index 6e78e38..c0f4b1f 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -627,38 +627,45 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName } if jsonl { scanKind := map[bool]string{true: "update", false: "index"}[update] - if err := writeJSONL(stdout, map[string]any{ - "type": "scan_summary", - "command": scanKind, - "scan_kind": scanKind, - "data": summary, - }); err != nil { + envelope := func(eventType string) map[string]any { + return map[string]any{ + "type": eventType, + "repo_root": repo.RootPath, + "repo_id": summary.RepoID, + "scan_id": summary.ScanID, + } + } + ev := envelope("scan_summary") + ev["command"] = scanKind + ev["scan_kind"] = scanKind + ev["data"] = summary + if err := writeJSONL(stdout, ev); err != nil { return err } - _ = writeJSONL(stdout, map[string]any{ - "type": "scan_phases", - "data": map[string]any{ - "existing_load_ms": summary.ExistingLoadMS, - "walk_ms": summary.WalkMS, - "process_wall_ms": summary.ProcessWallMS, - "task_ms": summary.TaskMS, - "task_other_ms": summary.TaskOtherMS, - "parse_ms": summary.ParseMS, - "read_ms": summary.ReadMS, - "hash_ms": summary.HashMS, - "write_ms": summary.WriteMS, - "write_metadata_ms": summary.WriteMetadataMS, - "write_replace_ms": summary.WriteReplaceMS, - "embed_ms": summary.EmbedMS, - "mark_missing_ms": summary.MarkMissingMS, - "resolve_ms": summary.ResolveMS, - "duration_ms": summary.DurationMS, - }, - }) - return writeJSONL(stdout, map[string]any{ - "type": "scan_stats", - "data": stats, - }) + ev = envelope("scan_phases") + ev["data"] = map[string]any{ + "existing_load_ms": summary.ExistingLoadMS, + "walk_ms": summary.WalkMS, + "process_wall_ms": summary.ProcessWallMS, + "task_ms": summary.TaskMS, + "task_other_ms": summary.TaskOtherMS, + "parse_ms": summary.ParseMS, + "read_ms": summary.ReadMS, + "hash_ms": summary.HashMS, + "write_ms": summary.WriteMS, + "write_metadata_ms": summary.WriteMetadataMS, + "write_replace_ms": summary.WriteReplaceMS, + "embed_ms": summary.EmbedMS, + "mark_missing_ms": summary.MarkMissingMS, + "resolve_ms": summary.ResolveMS, + "duration_ms": summary.DurationMS, + } + if err := writeJSONL(stdout, ev); err != nil { + return err + } + ev = envelope("scan_stats") + ev["data"] = stats + return writeJSONL(stdout, ev) } return writeJSON(stdout, map[string]any{"summary": summary, "stats": stats}) } From 715d2f87420abfa724e149bbc7b2fba45872b10d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:44:54 +0200 Subject: [PATCH 26/59] Q1 (#42) * cli/index: stabilize --jsonl scan event envelopes with repo/scan correlation fields * cli/index: dedupe JSONL scan event envelopes and handle scan_phases write errors * doctor: always emit applied_fixes and recommendations arrays in JSON output --- internal/cli/app.go | 26 ++++++++++++++++++++++++-- internal/doctor/doctor.go | 4 ++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index c0f4b1f..adaf3b7 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -635,10 +635,28 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName "scan_id": summary.ScanID, } } + stripEnvelopeKeys := func(v any, keys ...string) (map[string]any, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + return nil, err + } + for _, k := range keys { + delete(out, k) + } + return out, nil + } ev := envelope("scan_summary") ev["command"] = scanKind ev["scan_kind"] = scanKind - ev["data"] = summary + summaryData, err := stripEnvelopeKeys(summary, "repo_id", "scan_id") + if err != nil { + return err + } + ev["data"] = summaryData if err := writeJSONL(stdout, ev); err != nil { return err } @@ -664,7 +682,11 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName return err } ev = envelope("scan_stats") - ev["data"] = stats + statsData, err := stripEnvelopeKeys(stats, "repo_root", "repo_id") + if err != nil { + return err + } + ev["data"] = statsData return writeJSONL(stdout, ev) } return writeJSON(stdout, map[string]any{"summary": summary, "stats": stats}) diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 9155f94..bcc0507 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -29,8 +29,8 @@ type Report struct { SQLiteDriver string `json:"sqlite_driver"` DB *DBInfo `json:"db,omitempty"` Deep *DeepInfo `json:"deep,omitempty"` - AppliedFixes []string `json:"applied_fixes,omitempty"` - Recommendations []string `json:"recommendations,omitempty"` + AppliedFixes []string `json:"applied_fixes"` + Recommendations []string `json:"recommendations"` } type DBInfo struct { From 8fd60ffbd6de152de4df2879e9c60823e635117c Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:04:14 +0200 Subject: [PATCH 27/59] json: stabilize watch/doctor machine-readable output (watch event envelopes, doctor arrays always present) (#43) --- internal/cli/app.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index adaf3b7..b17add1 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -933,13 +933,17 @@ func runWatch(ctx context.Context, cfg config.Config, stdout io.Writer, args []s case <-ticker.C: current := w.Stats() writeEvent(map[string]any{ - "type": "watch_heartbeat", - "stats": current, + "type": "watch_heartbeat", + "repo_root": repo.RootPath, + "repo_id": repoID, + "stats": current, }) flushDelta := current.FlushRuns - prev.FlushRuns if flushDelta > 0 || current.FlushErrors > prev.FlushErrors || current.QueueErrors > prev.QueueErrors { writeEvent(map[string]any{ "type": "watch_flush_summary", + "repo_root": repo.RootPath, + "repo_id": repoID, "stats": current, "delta_flush_runs": flushDelta, "delta_flush_errors": current.FlushErrors - prev.FlushErrors, @@ -956,8 +960,10 @@ func runWatch(ctx context.Context, cfg config.Config, stdout io.Writer, args []s reporterDone.Wait() if *jsonl { event := map[string]any{ - "type": "watch_stopped", - "stats": w.Stats(), + "type": "watch_stopped", + "repo_root": repo.RootPath, + "repo_id": repoID, + "stats": w.Stats(), } if err != nil { event["error"] = err.Error() From f841e305df02b4951bdc1ca490fcc5603daf20fe Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:22:59 +0200 Subject: [PATCH 28/59] =?UTF-8?q?benchmark:=20add=20--sqlite-profile=20and?= =?UTF-8?q?=20capture=20sqlite=5Fprofile/host=20conte=E2=80=A6=20(#44)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * benchmark: add --sqlite-profile and capture sqlite_profile/host context in JSON output * benchmark: persist num_cpu/cwd in baselines and list benchmark flags in help --- internal/cli/app.go | 15 +++++++++++++++ internal/cli/app_test.go | 31 +++++++++++++++++++++++++++++++ internal/cli/commands.go | 10 +++++++++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index b17add1..9d183e5 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -265,6 +265,8 @@ type benchmarkBaseline struct { GOOS string `json:"goos,omitempty"` GOARCH string `json:"goarch,omitempty"` GOMAXPROCS string `json:"gomaxprocs,omitempty"` + NumCPU int `json:"num_cpu"` + CWD string `json:"cwd"` SQLite string `json:"sqlite_driver,omitempty"` BenchCtx map[string]any `json:"bench_ctx,omitempty"` Env map[string]string `json:"env,omitempty"` @@ -332,6 +334,7 @@ func extractBenchmarkContext(output string) map[string]any { keys := []string{ "fixture_files", "sqlite_driver", + "sqlite_profile", } out := map[string]any{} for line := range strings.SplitSeq(output, "\n") { @@ -369,6 +372,7 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { benchtime := fs.String("benchtime", "100ms", "benchmark time per test") saveBaseline := fs.Bool("save-baseline", true, "save current benchmark result as baseline") files := fs.Int("files", 0, "fixture file count (sets CODEGRAPH_BENCH_FILES)") + sqliteProfile := fs.String("sqlite-profile", "", "SQLite performance profile for benchmarks (sets CODEGRAPH_BENCH_SQLITE_PROFILE)") gomaxprocs := fs.Int("gomaxprocs", 0, "GOMAXPROCS for benchmark subprocess (0 = default)") if err := fs.Parse(args); err != nil { return err @@ -406,6 +410,12 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { } else if v := strings.TrimSpace(os.Getenv("CODEGRAPH_BENCH_FILES")); v != "" { benchEnv["CODEGRAPH_BENCH_FILES"] = v } + if v := strings.TrimSpace(*sqliteProfile); v != "" { + env = append(env, "CODEGRAPH_BENCH_SQLITE_PROFILE="+v) + benchEnv["CODEGRAPH_BENCH_SQLITE_PROFILE"] = v + } else if v := strings.TrimSpace(os.Getenv("CODEGRAPH_BENCH_SQLITE_PROFILE")); v != "" { + benchEnv["CODEGRAPH_BENCH_SQLITE_PROFILE"] = v + } if *gomaxprocs > 0 { v := strconv.Itoa(*gomaxprocs) env = append(env, "GOMAXPROCS="+v) @@ -416,6 +426,7 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { cmd.Env = env out, err := cmd.CombinedOutput() benchCtx := extractBenchmarkContext(string(out)) + cwd, _ := os.Getwd() result := map[string]any{ "command": append([]string{"go"}, cmdArgs...), "count": *count, @@ -423,6 +434,8 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { "go_version": runtime.Version(), "goos": runtime.GOOS, "goarch": runtime.GOARCH, + "num_cpu": runtime.NumCPU(), + "cwd": cwd, "sqlite_driver": store.SQLiteDriverName(), "env": benchEnv, "bench_ctx": benchCtx, @@ -471,6 +484,8 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, GOMAXPROCS: gomaxLabel, + NumCPU: runtime.NumCPU(), + CWD: cwd, SQLite: store.SQLiteDriverName(), BenchCtx: benchCtx, Env: benchEnv, diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index c2ea294..668320e 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -115,6 +115,37 @@ func main() {} } } +func TestBenchmarkBaselineIncludesContextFields(t *testing.T) { + payload := benchmarkBaseline{ + CreatedAt: "2026-01-02T03:04:05Z", + Command: []string{"go", "test"}, + Count: 1, + Benchtime: "10ms", + GoVersion: "go1.x", + GOOS: "windows", + GOARCH: "amd64", + GOMAXPROCS: "default", + NumCPU: 12, + CWD: "C:\\repo", + SQLite: "sqlite", + Benchmarks: map[string]benchmarkMetric{"BenchmarkX": {NsPerOp: 1}}, + } + data, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Marshal(payload) error = %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal(payload) error = %v", err) + } + if _, ok := decoded["num_cpu"]; !ok { + t.Fatalf("payload missing num_cpu: %s", string(data)) + } + if _, ok := decoded["cwd"]; !ok { + t.Fatalf("payload missing cwd: %s", string(data)) + } +} + func TestRunIndexWithRepoDBDirSkipsRepoDatabaseFiles(t *testing.T) { home := filepath.Join(t.TempDir(), "codegraph-home") t.Setenv("CODEGRAPH_HOME", home) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 53b0f60..aa4d24c 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -278,7 +278,15 @@ func newCommandList() []*command { { name: "benchmark", description: "benchmarks", - usageLines: []string{" benchmark [--count N] [--benchtime DURATION] [--save-baseline] [--files N] [--gomaxprocs N]"}, + usageLines: []string{" benchmark [--count N] [--benchtime DURATION] [--save-baseline] [--files N] [--sqlite-profile NAME] [--gomaxprocs N]"}, + flags: []commandFlag{ + {name: "--count", description: "number of benchmark runs"}, + {name: "--benchtime", description: "benchmark time per test"}, + {name: "--save-baseline", description: "save current benchmark result as baseline"}, + {name: "--files", description: "fixture file count (sets CODEGRAPH_BENCH_FILES)"}, + {name: "--sqlite-profile", description: "SQLite performance profile (sets CODEGRAPH_BENCH_SQLITE_PROFILE)"}, + {name: "--gomaxprocs", description: "GOMAXPROCS for benchmark subprocess (0 = default)"}, + }, run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { return runBenchmark(ctx, stdout, args) }, From d90bd817f45e4a88360d54937aaa3eacc87f5764 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 16:50:51 +0200 Subject: [PATCH 29/59] cli: add index_smoke runner with compact jsonl + median baseline for indexing perf diffs (#45) * cli: add index_smoke runner with compact jsonl + median baseline for indexing perf diffs * cli: surface index_smoke baseline/db/config errors; add .codegraphignore --- .codegraphignore | 9 + .gitignore | 1 + internal/cli/app.go | 377 +++++++++++++++++++++++++++++++++++++++ internal/cli/app_test.go | 43 +++++ internal/cli/commands.go | 25 +++ 5 files changed, 455 insertions(+) create mode 100644 .codegraphignore diff --git a/.codegraphignore b/.codegraphignore new file mode 100644 index 0000000..6d16249 --- /dev/null +++ b/.codegraphignore @@ -0,0 +1,9 @@ +# Repo-local ignores for codegraph indexing. +# This file uses .gitignore-like patterns and is loaded by codegraph in addition +# to config defaults. + +codegraph +codegraph.exe +codegraph.sqlite* +.gotelemetry/** +.claude/** diff --git a/.gitignore b/.gitignore index 22f2942..513ebee 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ codegraph.exe codegraph.sqlite codegraph.sqlite-shm codegraph.sqlite-wal +OPTIMIZATIONS.md diff --git a/internal/cli/app.go b/internal/cli/app.go index 9d183e5..2d351b3 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strconv" "strings" "sync" @@ -707,6 +708,382 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName return writeJSON(stdout, map[string]any{"summary": summary, "stats": stats}) } +type indexSmokeRun struct { + Run int `json:"run"` + ScanKind string `json:"scan_kind"` + Force bool `json:"force"` + Summary map[string]any `json:"summary"` + Stats map[string]any `json:"stats,omitempty"` + RepoRoot string `json:"repo_root"` + Context map[string]any `json:"context"` +} + +type indexSmokeBaseline struct { + CreatedAt string `json:"created_at"` + Command []string `json:"command"` + Context map[string]any `json:"context"` + Median map[string]any `json:"median"` +} + +func runIndexSmoke(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName string, args []string) error { + fs := flag.NewFlagSet(cmdName, flag.ContinueOnError) + fs.SetOutput(io.Discard) + mode := fs.String("mode", "index", "scan mode: index|update") + runs := fs.Int("runs", 3, "number of measured runs") + warmup := fs.Int("warmup", 0, "number of warmup runs (not reported)") + force := fs.Bool("force", false, "re-index files even if unchanged") + profile := fs.String("profile", "", "SQLite performance profile: balanced|durable|fast (default from config)") + gomaxprocs := fs.Int("gomaxprocs", 0, "set GOMAXPROCS for this process (0 = leave unchanged)") + baselinePath := fs.String("baseline", "", "read/write baseline JSON (optional)") + saveBaseline := fs.Bool("save-baseline", true, "write baseline JSON") + prettyJSON := fs.Bool("json", false, "pretty JSON output instead of jsonl") + repoRootArg := "" + parseArgs := args + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + repoRootArg = args[0] + parseArgs = args[1:] + } + if err := fs.Parse(parseArgs); err != nil { + return err + } + repoRoot := strings.TrimSpace(repoRootArg) + if repoRoot == "" && fs.NArg() > 0 { + repoRoot = strings.TrimSpace(fs.Arg(0)) + } + if repoRoot == "" { + return fmt.Errorf("usage: %s %s ", appname.BinaryName, cmdName) + } + if *runs < 1 { + return fmt.Errorf("--runs must be >= 1") + } + if *warmup < 0 { + return fmt.Errorf("--warmup must be >= 0") + } + if *gomaxprocs > 0 { + runtime.GOMAXPROCS(*gomaxprocs) + } + scanKind := strings.ToLower(strings.TrimSpace(*mode)) + if scanKind == "" { + scanKind = "index" + } + switch scanKind { + case "index", "update": + default: + return fmt.Errorf("--mode must be one of: index, update") + } + effectiveProfile := strings.TrimSpace(*profile) + if effectiveProfile == "" { + effectiveProfile = cfg.DBPerformanceProfile + } + + repoCanonical, err := store.CanonicalRepoPath(repoRoot) + if err != nil { + return err + } + repoAbs := repoCanonical + + var base *indexSmokeBaseline + if p := strings.TrimSpace(*baselinePath); p != "" { + data, err := os.ReadFile(p) + if errors.Is(err, os.ErrNotExist) { + // Baseline is optional. + } else if err != nil { + return fmt.Errorf("read baseline %s: %w", p, err) + } else { + var decoded indexSmokeBaseline + if err := json.Unmarshal(data, &decoded); err != nil { + return fmt.Errorf("decode baseline %s: %w", p, err) + } + base = &decoded + } + } + + ctxFields := map[string]any{ + "go_version": runtime.Version(), + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + "gomaxprocs": runtime.GOMAXPROCS(0), + "num_cpu": runtime.NumCPU(), + "sqlite": store.SQLiteDriverName(), + "db_profile": effectiveProfile, + } + + compactSummary := func(summary store.ScanSummary) map[string]any { + out := map[string]any{ + "files_seen": summary.FilesSeen, + "files_indexed": summary.FilesIndexed, + "files_changed": summary.FilesChanged, + "files_deleted": summary.FilesDeleted, + "process_wall_ms": summary.ProcessWallMS, + "task_ms": summary.TaskMS, + "task_other_ms": summary.TaskOtherMS, + "write_ms": summary.WriteMS, + "resolve_ms": summary.ResolveMS, + "duration_ms": summary.DurationMS, + "resolve_mode": summary.ResolveMode, + "write_flush_total": summary.WriteMarkSeenFlushes + + summary.WriteTouchFlushes + + summary.WriteParseFailedFlushes + + summary.WriteReplaceFlushes, + } + if summary.WriteStats != nil { + out["write_tx_count"] = summary.WriteStats.TxCount + out["total_exec_statements"] = summary.WriteStats.TotalExecStatements + } + return out + } + + writeJSONLEvent := func(eventType string, data any) error { + if *prettyJSON { + return writeJSON(stdout, data) + } + return writeJSONL(stdout, map[string]any{ + "type": eventType, + "repo_root": repoAbs, + "data": data, + }) + } + + makeApp := func(dbPath string) (*App, graphRepo, int64, error) { + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: effectiveProfile}) + if err != nil { + return nil, graphRepo{}, 0, err + } + registry := newDefaultRegistry() + repoCfg, err := config.LoadRepo(repoCanonical) + if err != nil { + _ = s.Close() + return nil, graphRepo{}, 0, fmt.Errorf("load repo config %s: %w", config.RepoConfigPath(repoCanonical), err) + } + embedder := newEmbedder(repoCfg.Embedding) + idx := indexer.New(s, registry, embedder) + repo, err := s.UpsertRepo(ctx, repoCanonical) + if err != nil { + _ = s.Close() + return nil, graphRepo{}, 0, err + } + return &App{Store: s, Indexer: idx, Query: query.New(s, embedder)}, graphRepo{ID: repo.ID, RootPath: repo.RootPath}, repo.ID, nil + } + + tmpDir, err := os.MkdirTemp("", "codegraph-index-smoke-*") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + dbPath := filepath.Join(tmpDir, "smoke.sqlite") + resetSmokeDB := func() error { + if err := os.RemoveAll(dbPath); err != nil { + return fmt.Errorf("remove smoke db %s: %w", dbPath, err) + } + return nil + } + + median := func(vals []int64) int64 { + if len(vals) == 0 { + return 0 + } + cp := append([]int64(nil), vals...) + sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] }) + mid := len(cp) / 2 + if len(cp)%2 == 1 { + return cp[mid] + } + return (cp[mid-1] + cp[mid]) / 2 + } + medianInt := func(vals []int) int { + if len(vals) == 0 { + return 0 + } + cp := append([]int(nil), vals...) + sort.Ints(cp) + mid := len(cp) / 2 + if len(cp)%2 == 1 { + return cp[mid] + } + return (cp[mid-1] + cp[mid]) / 2 + } + + type sample struct { + wallMS int64 + taskMS int64 + otherMS int64 + writeMS int64 + resMS int64 + exec int + tx int + flush int + } + var samples []sample + + stripStats := func(v any) (map[string]any, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + return nil, err + } + delete(out, "repo_root") + delete(out, "repo_id") + return out, nil + } + + runOne := func(update bool) (store.ScanSummary, map[string]any, error) { + app, repo, repoID, err := makeApp(dbPath) + if err != nil { + return store.ScanSummary{}, nil, err + } + defer app.Close() + opts := indexer.Options{ + RepoRoot: repo.RootPath, + Force: *force, + ScanKind: map[bool]string{true: "update", false: "index"}[update], + } + var summary store.ScanSummary + if update { + summary, err = app.Indexer.Update(ctx, opts) + } else { + summary, err = app.Indexer.Index(ctx, opts) + } + if err != nil { + return store.ScanSummary{}, nil, err + } + stats, err := app.Query.Stats(ctx, repoID) + if err != nil { + return store.ScanSummary{}, nil, err + } + statsData, err := stripStats(stats) + if err != nil { + return store.ScanSummary{}, nil, err + } + return summary, statsData, nil + } + + // Ensure update-mode timings are "update" (db already exists). + if scanKind == "update" { + if err := resetSmokeDB(); err != nil { + return err + } + if _, _, err := runOne(false); err != nil { + return err + } + } + for i := 0; i < *warmup; i++ { + if scanKind == "index" { + if err := resetSmokeDB(); err != nil { + return err + } + } + if _, _, err := runOne(scanKind == "update"); err != nil { + return err + } + } + for i := 0; i < *runs; i++ { + if scanKind == "index" { + if err := resetSmokeDB(); err != nil { + return err + } + } + summary, statsData, err := runOne(scanKind == "update") + if err != nil { + return err + } + ev := indexSmokeRun{ + Run: i + 1, + ScanKind: scanKind, + Force: *force, + Summary: compactSummary(summary), + Stats: statsData, + RepoRoot: repoAbs, + Context: ctxFields, + } + if err := writeJSONLEvent("index_smoke_run", ev); err != nil { + return err + } + var exec, tx int + if summary.WriteStats != nil { + exec = summary.WriteStats.TotalExecStatements + tx = summary.WriteStats.TxCount + } + samples = append(samples, sample{ + wallMS: summary.ProcessWallMS, + taskMS: summary.TaskMS, + otherMS: summary.TaskOtherMS, + writeMS: summary.WriteMS, + resMS: summary.ResolveMS, + exec: exec, + tx: tx, + flush: summary.WriteMarkSeenFlushes + + summary.WriteTouchFlushes + + summary.WriteParseFailedFlushes + + summary.WriteReplaceFlushes, + }) + } + + var walls, tasks, others, writes, resolves []int64 + var execs, txs, flushes []int + for _, s := range samples { + walls = append(walls, s.wallMS) + tasks = append(tasks, s.taskMS) + others = append(others, s.otherMS) + writes = append(writes, s.writeMS) + resolves = append(resolves, s.resMS) + execs = append(execs, s.exec) + txs = append(txs, s.tx) + flushes = append(flushes, s.flush) + } + medianFields := map[string]any{ + "process_wall_ms": median(walls), + "task_ms": median(tasks), + "task_other_ms": median(others), + "write_ms": median(writes), + "resolve_ms": median(resolves), + "total_exec_statements": medianInt(execs), + "write_tx_count": medianInt(txs), + "write_flush_total": medianInt(flushes), + } + final := map[string]any{ + "ok": true, + "context": ctxFields, + "median": medianFields, + } + if base != nil && base.Median != nil { + delta := map[string]any{} + for _, k := range []string{"process_wall_ms", "task_ms", "task_other_ms", "write_ms", "resolve_ms", "total_exec_statements"} { + cur, ok1 := medianFields[k] + baseV, ok2 := base.Median[k] + if ok1 && ok2 { + delta[k] = map[string]any{"current": cur, "baseline": baseV} + } + } + if len(delta) > 0 { + final["delta_vs_baseline"] = delta + final["baseline_created_at"] = base.CreatedAt + } + } + if p := strings.TrimSpace(*baselinePath); p != "" && *saveBaseline { + payload := indexSmokeBaseline{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Command: append([]string{appname.BinaryName, cmdName}, args...), + Context: ctxFields, + Median: medianFields, + } + data, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return fmt.Errorf("encode baseline %s: %w", p, err) + } + dir := filepath.Dir(p) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir baseline dir %s: %w", dir, err) + } + if err := os.WriteFile(p, append(data, '\n'), 0o644); err != nil { + return fmt.Errorf("write baseline %s: %w", p, err) + } + } + return writeJSONLEvent("index_smoke_summary", final) +} + func runStats(ctx context.Context, cfg config.Config, stdout io.Writer, args []string) error { fs := flag.NewFlagSet("stats", flag.ContinueOnError) fs.SetOutput(io.Discard) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 668320e..aaf82ac 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -115,6 +115,49 @@ func main() {} } } +func TestRunIndexSmokeJSONL(t *testing.T) { + home := filepath.Join(t.TempDir(), "codegraph-home") + t.Setenv("CODEGRAPH_HOME", home) + repoRoot := filepath.Join(t.TempDir(), "repo") + if err := os.MkdirAll(repoRoot, 0o755); err != nil { + t.Fatalf("MkdirAll(repoRoot) error = %v", err) + } + if err := os.WriteFile(filepath.Join(repoRoot, "main.go"), []byte(`package main +func main() {} +`), 0o644); err != nil { + t.Fatalf("WriteFile(main.go) error = %v", err) + } + prev := startupVersionCheck + startupVersionCheck = func(context.Context, io.Writer) {} + t.Cleanup(func() { + startupVersionCheck = prev + }) + + var out bytes.Buffer + var errOut bytes.Buffer + if err := Run(context.Background(), []string{"index_smoke", repoRoot, "--runs", "1"}, &out, &errOut); err != nil { + t.Fatalf("Run(index_smoke) error = %v", err) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) < 2 { + t.Fatalf("expected at least 2 jsonl lines, got %d: %q", len(lines), out.String()) + } + var first map[string]any + if err := json.Unmarshal([]byte(lines[0]), &first); err != nil { + t.Fatalf("first line unmarshal error = %v", err) + } + if first["type"] != "index_smoke_run" { + t.Fatalf("first event type = %v, want index_smoke_run", first["type"]) + } + var last map[string]any + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &last); err != nil { + t.Fatalf("last line unmarshal error = %v", err) + } + if last["type"] != "index_smoke_summary" { + t.Fatalf("last event type = %v, want index_smoke_summary", last["type"]) + } +} + func TestBenchmarkBaselineIncludesContextFields(t *testing.T) { payload := benchmarkBaseline{ CreatedAt: "2026-01-02T03:04:05Z", diff --git a/internal/cli/commands.go b/internal/cli/commands.go index aa4d24c..6440054 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -147,6 +147,31 @@ func newCommandList() []*command { return runIndex(ctx, cfg, stdout, invokedName, args, true) }, }, + { + name: "index_smoke", + aliases: []string{"smoke_index", "smoke-index"}, + description: "run repeatable index/update smoke timing", + usageLines: []string{" index_smoke "}, + flags: []commandFlag{ + {name: "--mode", description: "scan mode: index|update (default index)"}, + {name: "--runs", description: "number of measured runs"}, + {name: "--warmup", description: "number of warmup runs (not reported)"}, + {name: "--force", description: "re-index files even if unchanged"}, + {name: "--profile", description: "SQLite performance profile: balanced|durable|fast (default from config)"}, + {name: "--gomaxprocs", description: "set GOMAXPROCS for this process (0 = leave unchanged)"}, + {name: "--baseline", description: "read baseline JSON (optional)"}, + {name: "--save-baseline", description: "write baseline JSON (default true when --baseline is set)"}, + {name: "--json", description: "pretty JSON output instead of jsonl"}, + }, + examples: []string{ + "codegraph index_smoke .", + "codegraph index_smoke . --mode update --force", + "codegraph index_smoke . --runs 5 --baseline .codegraph/smoke-baseline.json", + }, + run: func(ctx context.Context, cfg config.Config, stdout, stderr io.Writer, invokedName string, args []string) error { + return runIndexSmoke(ctx, cfg, stdout, invokedName, args) + }, + }, { name: "stats", description: "show index stats", From 7922ed7de9dcc0e30d338b9fa3c80261991b391d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:46:52 +0200 Subject: [PATCH 30/59] store/indexer: purge deleted-file graph rows and nullify cross-file symbol references (#46) * store/indexer: purge deleted-file graph rows and nullify cross-file symbol references * store: reuse temp id tables in deleted-file purge --- internal/indexer/indexer.go | 10 +- internal/indexer/indexer_test.go | 17 ++ internal/store/store.go | 334 ++++++++++++++++++++++++------- 3 files changed, 285 insertions(+), 76 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 677eccd..d8626c6 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -558,7 +558,6 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err summary.MarkMissingMS = time.Since(missingStarted).Milliseconds() summary.FilesDeleted = deleted } - resolveStart := time.Now() if len(missingCandidatePaths) > 0 { deleted, err := i.store.MarkFilesDeletedBatch(ctx, repo.ID, scanID, missingCandidatePaths) if err != nil { @@ -568,6 +567,15 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err summary.FilesDeleted += deleted } + if summary.FilesDeleted > 0 { + if _, err := i.store.PurgeDeletedFileGraphsForScan(ctx, repo.ID, scanID); err != nil { + _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) + return summary, err + } + } + + resolveStart := time.Now() + if len(changedPathSet) == 0 { summary.ResolveMS = 0 summary.ResolveMode = "none" diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 7be5442..35cfcea 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -103,6 +103,23 @@ func main() { if deletedSummary.FilesDeleted != 1 { t.Fatalf("FilesDeleted = %d, want 1", deletedSummary.FilesDeleted) } + + stats, err = s.Stats(ctx, repo.ID) + if err != nil { + t.Fatalf("Stats(after delete) error = %v", err) + } + if stats.Files != 0 { + t.Fatalf("stats.Files after delete = %d, want 0", stats.Files) + } + if stats.Symbols != 0 { + t.Fatalf("stats.Symbols after delete = %d, want 0", stats.Symbols) + } + if stats.References != 0 { + t.Fatalf("stats.References after delete = %d, want 0", stats.References) + } + if stats.Edges != 0 { + t.Fatalf("stats.Edges after delete = %d, want 0", stats.Edges) + } } func TestIndexSkipsDotAndGeneratedDirectories(t *testing.T) { diff --git a/internal/store/store.go b/internal/store/store.go index 25eee6e..8d250e1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1048,83 +1048,10 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, sta // For large batches, use a temp table to avoid repeating large IN clauses across each dependent-table delete. // This reduces statement pressure from ~O(numTables * chunks) down to O(chunks + numTables). if len(fileIDs) > sqliteInClauseBatchSize { - if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_delete_file_ids(id INTEGER PRIMARY KEY)`); err != nil { + if err := prepareTmpDeleteFileIDs(ctx, tx, fileIDs, stats); err != nil { return err } - if stats != nil { - stats.TotalExecStatements++ - } - if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_delete_file_ids`); err != nil { - return err - } - if stats != nil { - stats.TotalExecStatements++ - } - - for start := 0; start < len(fileIDs); start += sqliteInClauseBatchSize { - end := start + sqliteInClauseBatchSize - if end > len(fileIDs) { - end = len(fileIDs) - } - chunk := fileIDs[start:end] - placeholders := strings.Repeat("(?),", len(chunk)) - placeholders = strings.TrimSuffix(placeholders, ",") - query := `INSERT INTO tmp_delete_file_ids(id) VALUES ` + placeholders - args := make([]any, len(chunk)) - for i, id := range chunk { - args[i] = id - } - if _, err := tx.ExecContext(ctx, query, args...); err != nil { - return err - } - if stats != nil { - stats.FileGraphDeleteTempIDInsertBatches++ - stats.FileGraphDeleteTempIDInsertRows += len(chunk) - stats.TotalExecStatements++ - } - } - - exec := func(query string) error { - if _, err := tx.ExecContext(ctx, query); err != nil { - return err - } - if stats != nil { - stats.FileGraphDeleteStatements++ - stats.TotalExecStatements++ - } - return nil - } - - // Dependent tables that reference symbols must be deleted before deleting symbols. - if err := exec(`DELETE FROM symbol_tokens WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { - return err - } - if err := exec(`DELETE FROM symbol_fts WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { - return err - } - - if err := exec(`DELETE FROM edges WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM references_tbl WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM file_imports WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM file_tokens WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM test_links WHERE test_file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM symbol_embeddings WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - if err := exec(`DELETE FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { - return err - } - return nil + return deleteFileGraphsBatchFromTemp(ctx, tx, stats) } execInChunks := func(sqlPrefix, sqlSuffix string, ids []int64) error { @@ -1191,6 +1118,90 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, sta return execInChunks(`DELETE FROM symbols WHERE file_id IN (`, `)`, fileIDs) } +func prepareTmpDeleteFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []int64, stats *WriteStats) error { + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_delete_file_ids(id INTEGER PRIMARY KEY)`); err != nil { + return err + } + if stats != nil { + stats.TotalExecStatements++ + } + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_delete_file_ids`); err != nil { + return err + } + if stats != nil { + stats.TotalExecStatements++ + } + + for start := 0; start < len(fileIDs); start += sqliteInClauseBatchSize { + end := start + sqliteInClauseBatchSize + if end > len(fileIDs) { + end = len(fileIDs) + } + chunk := fileIDs[start:end] + placeholders := strings.Repeat("(?),", len(chunk)) + placeholders = strings.TrimSuffix(placeholders, ",") + query := `INSERT INTO tmp_delete_file_ids(id) VALUES ` + placeholders + args := make([]any, len(chunk)) + for i, id := range chunk { + args[i] = id + } + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return err + } + if stats != nil { + stats.FileGraphDeleteTempIDInsertBatches++ + stats.FileGraphDeleteTempIDInsertRows += len(chunk) + stats.TotalExecStatements++ + } + } + + return nil +} + +func deleteFileGraphsBatchFromTemp(ctx context.Context, tx *sql.Tx, stats *WriteStats) error { + exec := func(query string) error { + if _, err := tx.ExecContext(ctx, query); err != nil { + return err + } + if stats != nil { + stats.FileGraphDeleteStatements++ + stats.TotalExecStatements++ + } + return nil + } + + // Dependent tables that reference symbols must be deleted before deleting symbols. + if err := exec(`DELETE FROM symbol_tokens WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { + return err + } + if err := exec(`DELETE FROM symbol_fts WHERE symbol_id IN (SELECT id FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids))`); err != nil { + return err + } + + if err := exec(`DELETE FROM edges WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM references_tbl WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM file_imports WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM file_tokens WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM test_links WHERE test_file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM symbol_embeddings WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + if err := exec(`DELETE FROM symbols WHERE file_id IN (SELECT id FROM tmp_delete_file_ids)`); err != nil { + return err + } + return nil +} + func deleteFileGraph(ctx context.Context, tx *sql.Tx, fileID int64) error { deleteSymbolTokensStmt, err := tx.PrepareContext(ctx, ` DELETE FROM symbol_tokens @@ -1701,6 +1712,179 @@ func (s *Store) MarkFilesDeletedBatch(ctx context.Context, repoID, scanID int64, return int(total), nil } +// PurgeDeletedFileGraphsForScan removes dependent graph rows for files that were +// marked deleted during the given scan. This keeps stats/search/export from +// surfacing stale symbols/edges/references after file deletions while allowing +// future restores to re-index cleanly. +// +// Note: this also nulls out cross-file references to symbols defined in deleted +// files (for example edges.dst_symbol_id), so that future resolve passes can +// re-resolve them if the file returns. +func (s *Store) PurgeDeletedFileGraphsForScan(ctx context.Context, repoID, scanID int64) (int, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id + FROM files + WHERE repo_id = ? AND is_deleted = 1 AND last_scan_id = ? + `, repoID, scanID) + if err != nil { + return 0, err + } + var fileIDs []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return 0, err + } + fileIDs = append(fileIDs, id) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, err + } + if err := rows.Close(); err != nil { + return 0, err + } + if len(fileIDs) == 0 { + return 0, nil + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + committed := false + defer func() { + if committed { + return + } + _ = tx.Rollback() + }() + + if len(fileIDs) > sqliteInClauseBatchSize { + if err := prepareTmpDeleteFileIDs(ctx, tx, fileIDs, nil); err != nil { + return 0, err + } + if err := nullifyDeletedSymbolReferencesFromTemp(ctx, tx, repoID); err != nil { + return 0, err + } + if err := deleteFileGraphsBatchFromTemp(ctx, tx, nil); err != nil { + return 0, err + } + } else { + if err := nullifyDeletedSymbolReferences(ctx, tx, repoID, fileIDs); err != nil { + return 0, err + } + if err := deleteFileGraphsBatch(ctx, tx, fileIDs, nil); err != nil { + return 0, err + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + committed = true + return len(fileIDs), nil +} + +func nullifyDeletedSymbolReferences(ctx context.Context, tx *sql.Tx, repoID int64, fileIDs []int64) error { + if len(fileIDs) == 0 { + return nil + } + + if len(fileIDs) > sqliteInClauseBatchSize { + if err := prepareTmpDeleteFileIDs(ctx, tx, fileIDs, nil); err != nil { + return err + } + return nullifyDeletedSymbolReferencesFromTemp(ctx, tx, repoID) + } + + placeholders := strings.Repeat("?,", len(fileIDs)) + placeholders = strings.TrimSuffix(placeholders, ",") + args := make([]any, 0, len(fileIDs)+1) + args = append(args, repoID) + for _, id := range fileIDs { + args = append(args, id) + } + symbolIDs := `SELECT id FROM symbols WHERE file_id IN (` + placeholders + `)` + + if _, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = NULL + WHERE repo_id = ? AND dst_symbol_id IN (`+symbolIDs+`) + `, args...); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE test_links + SET target_symbol_id = NULL + WHERE repo_id = ? AND target_symbol_id IN (`+symbolIDs+`) + `, args...); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE references_tbl + SET symbol_id = NULL + WHERE repo_id = ? AND symbol_id IN (`+symbolIDs+`) + `, args...); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE references_tbl + SET context_symbol_id = NULL + WHERE repo_id = ? AND context_symbol_id IN (`+symbolIDs+`) + `, args...); err != nil { + return err + } + return nil +} + +func nullifyDeletedSymbolReferencesFromTemp(ctx context.Context, tx *sql.Tx, repoID int64) error { + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_delete_symbol_ids(id INTEGER PRIMARY KEY)`); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_delete_symbol_ids`); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_delete_symbol_ids(id) + SELECT id + FROM symbols + WHERE file_id IN (SELECT id FROM tmp_delete_file_ids) + `); err != nil { + return err + } + + if _, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = NULL + WHERE repo_id = ? AND dst_symbol_id IN (SELECT id FROM tmp_delete_symbol_ids) + `, repoID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE test_links + SET target_symbol_id = NULL + WHERE repo_id = ? AND target_symbol_id IN (SELECT id FROM tmp_delete_symbol_ids) + `, repoID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE references_tbl + SET symbol_id = NULL + WHERE repo_id = ? AND symbol_id IN (SELECT id FROM tmp_delete_symbol_ids) + `, repoID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` + UPDATE references_tbl + SET context_symbol_id = NULL + WHERE repo_id = ? AND context_symbol_id IN (SELECT id FROM tmp_delete_symbol_ids) + `, repoID); err != nil { + return err + } + return nil +} + type edgeTarget struct { edgeID int64 dstName string From 5a1aeda210122be4e216416eda96265475ad5a6e Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:21:34 +0200 Subject: [PATCH 31/59] watch: apply repo include/exclude and harden dirty_files draining (#47) * watch: apply repo include/exclude and harden dirty_files draining * watch: batch re-queue dirty_files on update failure --- internal/indexer/indexer.go | 8 +++++ internal/store/store.go | 50 ++++++++++++++++++++++++++++ internal/store/store_test.go | 63 ++++++++++++++++++++++++++++++++++++ internal/watcher/watcher.go | 55 +++++++++++++++++++------------ 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index d8626c6..468ad99 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -749,6 +749,14 @@ func ShouldSkipDir(rel string, excludes []string) bool { return shouldSkipDir(rel, excludes) } +func ShouldSkipFile(rel string, includes, excludes []string) bool { + return shouldSkipFile(rel, includes, excludes) +} + +func ShouldIgnorePath(rel string, excludes []string) bool { + return shouldIgnorePath(rel, excludes) +} + func shouldSkipDir(rel string, excludes []string) bool { rel = filepath.ToSlash(rel) base := filepath.Base(rel) diff --git a/internal/store/store.go b/internal/store/store.go index 8d250e1..40f1ea5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3184,6 +3184,56 @@ func (s *Store) QueueDirtyFile(ctx context.Context, repoID int64, path, reason s return err } +func (s *Store) QueueDirtyFiles(ctx context.Context, repoID int64, paths []string, reason string) error { + if len(paths) == 0 { + return nil + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + committed := false + defer func() { + if committed { + return + } + _ = tx.Rollback() + }() + + stmt, err := tx.PrepareContext(ctx, ` + INSERT INTO dirty_files(repo_id, path, reason, queued_at) + VALUES(?, ?, ?, ?) + ON CONFLICT(repo_id, path) DO UPDATE SET reason=excluded.reason, queued_at=excluded.queued_at + `) + if err != nil { + return err + } + defer stmt.Close() + + for _, path := range paths { + if _, err := stmt.ExecContext(ctx, repoID, path, reason, time.Now().UTC().Format(time.RFC3339)); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + committed = true + return nil +} + +func (s *Store) HasDirtyFiles(ctx context.Context, repoID int64) (bool, error) { + var exists int + err := s.db.QueryRowContext(ctx, `SELECT 1 FROM dirty_files WHERE repo_id = ? LIMIT 1`, repoID).Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + func (s *Store) DrainDirtyFiles(ctx context.Context, repoID int64) ([]string, error) { // Prefer an atomic drain. `DELETE ... RETURNING` guarantees we only remove rows // that are returned to the caller (no SELECT+DELETE race). diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 35b2791..4310377 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -12,6 +12,69 @@ import ( "github.com/isink17/codegraph/internal/store" ) +func TestDirtyFilesQueueAndDrain(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + if ok, err := s.HasDirtyFiles(ctx, repo.ID); err != nil { + t.Fatalf("HasDirtyFiles() error = %v", err) + } else if ok { + t.Fatalf("expected no dirty files at start") + } + + if err := s.QueueDirtyFile(ctx, repo.ID, "a.go", "test"); err != nil { + t.Fatalf("QueueDirtyFile(a.go) error = %v", err) + } + if err := s.QueueDirtyFile(ctx, repo.ID, "b.go", "test"); err != nil { + t.Fatalf("QueueDirtyFile(b.go) error = %v", err) + } + if err := s.QueueDirtyFile(ctx, repo.ID, "a.go", "test2"); err != nil { + t.Fatalf("QueueDirtyFile(a.go update) error = %v", err) + } + + if ok, err := s.HasDirtyFiles(ctx, repo.ID); err != nil { + t.Fatalf("HasDirtyFiles() error = %v", err) + } else if !ok { + t.Fatalf("expected dirty files after queueing") + } + + paths, err := s.DrainDirtyFiles(ctx, repo.ID) + if err != nil { + t.Fatalf("DrainDirtyFiles() error = %v", err) + } + if len(paths) != 2 { + t.Fatalf("expected 2 paths, got %d (%v)", len(paths), paths) + } + + if ok, err := s.HasDirtyFiles(ctx, repo.ID); err != nil { + t.Fatalf("HasDirtyFiles() error = %v", err) + } else if ok { + t.Fatalf("expected no dirty files after drain") + } + + if err := s.QueueDirtyFiles(ctx, repo.ID, []string{"c.go", "d.go"}, "batch"); err != nil { + t.Fatalf("QueueDirtyFiles() error = %v", err) + } + + paths, err = s.DrainDirtyFiles(ctx, repo.ID) + if err != nil { + t.Fatalf("DrainDirtyFiles() (2) error = %v", err) + } + if len(paths) != 2 { + t.Fatalf("expected 2 paths from batch, got %d (%v)", len(paths), paths) + } +} + func TestListScansIncludesLanguageCoverage(t *testing.T) { ctx := context.Background() repoRoot := t.TempDir() diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 1cbf7de..fa739f1 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -12,6 +12,7 @@ import ( "github.com/fsnotify/fsnotify" + "github.com/isink17/codegraph/internal/config" "github.com/isink17/codegraph/internal/indexer" "github.com/isink17/codegraph/internal/store" ) @@ -112,6 +113,13 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun debounce = 750 * time.Millisecond } + repoCfg, err := config.LoadRepo(repoRoot) + if err != nil { + return err + } + includes := repoCfg.Include + excludes := repoCfg.Exclude + fsw, err := fsnotify.NewWatcher() if err != nil { return err @@ -129,7 +137,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun return relErr } rel = filepath.Clean(rel) - if d.IsDir() && indexer.ShouldSkipDir(rel, nil) { + if d.IsDir() && indexer.ShouldSkipDir(rel, excludes) { return filepath.SkipDir } } @@ -160,11 +168,29 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun Paths: paths, ScanKind: "watch", }) - if err == nil { - w.updateRuns.Add(1) - w.updatePaths.Add(int64(len(paths))) + if err != nil { + // `DrainDirtyFiles` is destructive; re-queue the drained paths on failure so + // work isn't silently dropped. + if requeueErr := w.store.QueueDirtyFiles(ctx, repoID, paths, "watch_retry"); requeueErr != nil { + return fmt.Errorf("update failed: %w (retry re-queue failed: %v)", err, requeueErr) + } + return err } + w.updateRuns.Add(1) + w.updatePaths.Add(int64(len(paths))) + return err + } + + if hasDirty, err := w.store.HasDirtyFiles(ctx, repoID); err != nil { return err + } else if hasDirty { + // Ensure any queued work from previous runs is processed even if no new + // fsnotify events occur. + w.flushRuns.Add(1) + if err := flush(); err != nil { + w.flushErrors.Add(1) + return err + } } flushSignalCh := make(chan struct{}, 1) @@ -242,7 +268,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun continue } rel = filepath.Clean(rel) - if shouldIgnorePath(rel) { + if indexer.ShouldIgnorePath(rel, excludes) { w.eventsIgnored.Add(1) continue } @@ -257,6 +283,10 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun continue } } + if indexer.ShouldSkipFile(rel, includes, excludes) { + w.eventsIgnored.Add(1) + continue + } seenMu.Lock() _, alreadyQueued := seenSinceFlush[rel] @@ -307,18 +337,3 @@ func (w *Watcher) queueDirtyWithRetry(ctx context.Context, repoID int64, path, r } return fmt.Errorf("queue dirty file %s: %w", path, lastErr) } - -func shouldIgnorePath(rel string) bool { - current := rel - for current != "." && current != "" { - if indexer.ShouldSkipDir(current, nil) { - return true - } - next := filepath.Dir(current) - if next == current { - break - } - current = next - } - return false -} From 888ce49080aa8d07237f38940e2de02958f9fc1d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:48:35 +0200 Subject: [PATCH 32/59] =?UTF-8?q?store:=20speed=20up=20edge=20resolver=20(?= =?UTF-8?q?dot-tail2=20map,=20dotted-edge=20index,=20reso=E2=80=A6=20(#48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * store: speed up edge resolver (dot-tail2 map, dotted-edge index, resolver symbol indexes) * store: make resolver temp-table cleanup safe + avoid eager suffix clones --- internal/config/config.go | 1 - .../012_edges_unresolved_dotname_idx.sql | 6 + .../schema/013_symbol_resolution_indexes.sql | 13 + internal/store/store.go | 238 ++++++++++++------ 4 files changed, 185 insertions(+), 73 deletions(-) create mode 100644 internal/store/schema/012_edges_unresolved_dotname_idx.sql create mode 100644 internal/store/schema/013_symbol_resolution_indexes.sql diff --git a/internal/config/config.go b/internal/config/config.go index ca037ea..aeb79e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,7 +20,6 @@ const repoDBExcludePattern = "codegraph.sqlite*" var HardcodedSkips = []string{".git", "node_modules", ".next", ".nuxt", ".svelte-kit", ".turbo", ".pnpm-store", ".yarn", ".parcel-cache"} var DefaultExcludes = []string{".codegraph/**", ".codegraph-home/**", ".codegraph-home2/**", ".gocache/**", ".gomodcache/**", ".tmp/**", "vendor/**", "dist/**", "build/**", "coverage/**", "out/**", ".cache/**", repoDBExcludePattern} - type Config struct { DefaultLogLevel string `json:"default_log_level"` DefaultExcludes []string `json:"default_excludes"` diff --git a/internal/store/schema/012_edges_unresolved_dotname_idx.sql b/internal/store/schema/012_edges_unresolved_dotname_idx.sql new file mode 100644 index 0000000..79fa006 --- /dev/null +++ b/internal/store/schema/012_edges_unresolved_dotname_idx.sql @@ -0,0 +1,6 @@ +-- Speeds up ResolveEdgesForNames' qualified dst_name scan by shrinking the +-- unresolved-edge working set to dotted (qualified-ish) names. +CREATE INDEX IF NOT EXISTS idx_edges_repo_unresolved_dotname +ON edges(repo_id, dst_name) +WHERE dst_symbol_id IS NULL AND instr(dst_name, '.') > 0; + diff --git a/internal/store/schema/013_symbol_resolution_indexes.sql b/internal/store/schema/013_symbol_resolution_indexes.sql new file mode 100644 index 0000000..8b454fe --- /dev/null +++ b/internal/store/schema/013_symbol_resolution_indexes.sql @@ -0,0 +1,13 @@ +-- Speeds up ResolveEdges strategies that filter by additional symbol predicates. +-- These are intentionally partial to keep the btree small on large repos. + +-- Strategy 2: name match (unqualified) limited to callable/type-like kinds. +CREATE INDEX IF NOT EXISTS idx_symbols_repo_name_resolve_kind +ON symbols(repo_id, name) +WHERE kind IN ('function', 'method', 'class', 'type', 'struct', 'interface'); + +-- Strategy 4: method receiver match (container_name != ''). +CREATE INDEX IF NOT EXISTS idx_symbols_repo_name_has_container +ON symbols(repo_id, name) +WHERE container_name != ''; + diff --git a/internal/store/store.go b/internal/store/store.go index 40f1ea5..b2e4645 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1936,15 +1936,16 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { } // Strategy 3a: Suffix match for slash-qualified symbols (e.g., pkg.Func matches github.com/org/repo/pkg.Func). - // Avoid per-edge LIKE scans by precomputing (tail -> symbol_id) once, then doing an indexed equality update. + // Strategy 3b: Two-segment dot-tail match (e.g., pkg.Func matches x.y.pkg.Func and also x.pkg.Func matches x.y.x.pkg.Func). + // Avoid per-edge LIKE scans by precomputing suffix maps once, then doing indexed equality updates. n, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID) if err != nil { return 0, err } totalResolved += n - // Strategy 3b: Dot-suffix fallback (for qualified names without a slash separator). - // Kept as a narrower fallback to preserve existing semantics across languages that don't use '/' in qualified names. + // Strategy 3c: Dot-suffix fallback (for qualified names without a slash separator). + // Keep as a narrower fallback (multi-dot dst_name only) to preserve existing semantics without paying LIKE cost on common pkg.Func cases. res, err = tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = ( SELECT s.id FROM symbols s @@ -1954,6 +1955,7 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { ) WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL AND instr(edges.dst_name, '.') > 0 AND instr(edges.dst_name, '/') = 0 + AND instr(substr(edges.dst_name, instr(edges.dst_name, '.') + 1), '.') > 0 AND EXISTS ( SELECT 1 FROM symbols s WHERE s.repo_id = ? AND s.qualified_name LIKE '%.' || edges.dst_name @@ -2005,6 +2007,8 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI } // Map: suffix (after last '/') -> minimum symbol id. Using MIN keeps behavior stable when multiple symbols share the same suffix. minBySuffix := map[string]int64{} + // Map: dot tail (last 2 segments after last '/') -> minimum symbol id. + minByDotTail2 := map[string]int64{} for rows.Next() { var id int64 var qname string @@ -2015,16 +2019,35 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI if qname == "" { continue } - slash := strings.LastIndexByte(qname, '/') - if slash < 0 || slash+1 >= len(qname) { - continue - } - suffix := strings.Clone(qname[slash+1:]) - if suffix == "" { - continue + + afterSlash := qname + if slash := strings.LastIndexByte(qname, '/'); slash >= 0 && slash+1 < len(qname) { + afterSlash = qname[slash+1:] + if afterSlash != "" { + if cur, ok := minBySuffix[afterSlash]; !ok { + minBySuffix[strings.Clone(afterSlash)] = id + } else if id < cur { + // Update value only; existing map key remains the cloned string. + minBySuffix[afterSlash] = id + } + } } - if cur, ok := minBySuffix[suffix]; !ok || id < cur { - minBySuffix[suffix] = id + + lastDot := strings.LastIndexByte(afterSlash, '.') + if lastDot >= 0 && lastDot+1 < len(afterSlash) { + start := 0 + if prevDot := strings.LastIndexByte(afterSlash[:lastDot], '.'); prevDot >= 0 { + start = prevDot + 1 + } + tail2 := afterSlash[start:] + if tail2 != "" { + if cur, ok := minByDotTail2[tail2]; !ok { + minByDotTail2[strings.Clone(tail2)] = id + } else if id < cur { + // Update value only; existing map key remains the cloned string. + minByDotTail2[tail2] = id + } + } } } if err := rows.Err(); err != nil { @@ -2034,58 +2057,129 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI if err := rows.Close(); err != nil { return 0, err } - if len(minBySuffix) == 0 { - return 0, nil - } + totalResolved := 0 - resolved := make([]resolvedSuffix, 0, len(minBySuffix)) - for dstName, dstID := range minBySuffix { - resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) - } - if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { - return 0, err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { - return 0, err - } + if len(minBySuffix) > 0 { + resolved := make([]resolvedSuffix, 0, len(minBySuffix)) + for dstName, dstID := range minBySuffix { + resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) + } + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + return 0, err + } + droppedSlashSuffix := false + defer func() { + if droppedSlashSuffix { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_symbol_slash_suffix`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { + return 0, err + } - // Keep well under SQLite's default variable limit (999). - const maxPairsPerInsert = 400 - for start := 0; start < len(resolved); start += maxPairsPerInsert { - end := min(start+maxPairsPerInsert, len(resolved)) - chunk := resolved[start:end] - var b strings.Builder - b.WriteString(`INSERT INTO tmp_symbol_slash_suffix(dst_name, dst_symbol_id) VALUES `) - args := make([]any, 0, len(chunk)*2) - for i, r := range chunk { - if i > 0 { - b.WriteString(",") + // Keep well under SQLite's default variable limit (999). + const maxPairsPerInsert = 400 + for start := 0; start < len(resolved); start += maxPairsPerInsert { + end := min(start+maxPairsPerInsert, len(resolved)) + chunk := resolved[start:end] + var b strings.Builder + b.WriteString(`INSERT INTO tmp_symbol_slash_suffix(dst_name, dst_symbol_id) VALUES `) + args := make([]any, 0, len(chunk)*2) + for i, r := range chunk { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?)") + args = append(args, r.dstName, r.dstID) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + return 0, err } - b.WriteString("(?,?)") - args = append(args, r.dstName, r.dstID) } - if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + + updateRes, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_slash_suffix t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) + `, repoID) + if err != nil { return 0, err } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { + return 0, err + } + droppedSlashSuffix = true + n, err := updateRes.RowsAffected() + if err != nil { + return 0, err + } + totalResolved += int(n) } - updateRes, err := tx.ExecContext(ctx, ` - UPDATE edges - SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_slash_suffix t WHERE t.dst_name = edges.dst_name) - WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) - `, repoID) - if err != nil { - return 0, err - } - if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { - return 0, err - } - n, err := updateRes.RowsAffected() - if err != nil { - return 0, err + if len(minByDotTail2) > 0 { + resolved := make([]resolvedSuffix, 0, len(minByDotTail2)) + for dstName, dstID := range minByDotTail2 { + resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) + } + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_dot_tail2(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + return 0, err + } + droppedDotTail2 := false + defer func() { + if droppedDotTail2 { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_symbol_dot_tail2`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_dot_tail2`); err != nil { + return 0, err + } + + // Keep well under SQLite's default variable limit (999). + const maxPairsPerInsert = 400 + for start := 0; start < len(resolved); start += maxPairsPerInsert { + end := min(start+maxPairsPerInsert, len(resolved)) + chunk := resolved[start:end] + var b strings.Builder + b.WriteString(`INSERT INTO tmp_symbol_dot_tail2(dst_name, dst_symbol_id) VALUES `) + args := make([]any, 0, len(chunk)*2) + for i, r := range chunk { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?,?)") + args = append(args, r.dstName, r.dstID) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + return 0, err + } + } + + updateRes, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_dot_tail2 t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND instr(edges.dst_name, '.') > 0 AND instr(edges.dst_name, '/') = 0 + AND instr(substr(edges.dst_name, instr(edges.dst_name, '.') + 1), '.') = 0 + AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_dot_tail2) + `, repoID) + if err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_dot_tail2`); err != nil { + return 0, err + } + droppedDotTail2 = true + n, err := updateRes.RowsAffected() + if err != nil { + return 0, err + } + totalResolved += int(n) } - return int(n), nil + + return totalResolved, nil } func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths []string) error { @@ -2352,13 +2446,23 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] return err } + dotTail := func(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + if dot := strings.LastIndexByte(name, '.'); dot >= 0 && dot+1 < len(name) { + return strings.TrimSpace(name[dot+1:]) + } + return name + } + shortSet := map[string]struct{}{} for _, target := range targets { if _, ok := byQualified[target.dstName]; ok { continue } - parts := strings.Split(target.dstName, ".") - short := strings.TrimSpace(parts[len(parts)-1]) + short := dotTail(target.dstName) if short != "" { shortSet[short] = struct{}{} } @@ -2384,8 +2488,7 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] for _, target := range targets { dstID, ok := byQualified[target.dstName] if !ok { - parts := strings.Split(target.dstName, ".") - short := strings.TrimSpace(parts[len(parts)-1]) + short := dotTail(target.dstName) dstID, ok = byShort[short] } if !ok || dstID == 0 { @@ -2500,10 +2603,11 @@ func (s *Store) resolveUniqueSymbolsByNames(ctx context.Context, repoID int64, n chunk := names[start:end] placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") query := ` - SELECT name, id + SELECT name, MIN(id) FROM symbols WHERE repo_id = ? AND name IN (` + placeholders + `) - ORDER BY id ASC + GROUP BY name + HAVING COUNT(1) = 1 ` args := make([]any, 0, len(chunk)+1) args = append(args, repoID) @@ -2514,8 +2618,6 @@ func (s *Store) resolveUniqueSymbolsByNames(ctx context.Context, repoID int64, n if err != nil { return nil, err } - seenCount := map[string]int{} - firstID := map[string]int64{} for rows.Next() { var name string var id int64 @@ -2523,19 +2625,11 @@ func (s *Store) resolveUniqueSymbolsByNames(ctx context.Context, repoID int64, n _ = rows.Close() return nil, err } - seenCount[name]++ - if seenCount[name] == 1 { - firstID[name] = id - } + out[name] = id } if err := rows.Close(); err != nil { return nil, err } - for name, count := range seenCount { - if count == 1 { - out[name] = firstID[name] - } - } } return out, nil } From 916f412e9486a4dac77b40463698909f0fba8a8d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:08:46 +0200 Subject: [PATCH 33/59] =?UTF-8?q?cli/config:=20default=20repo=20artifacts?= =?UTF-8?q?=20under=20.codegraph=20(db,=20bench=20gocach=E2=80=A6=20(#49)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cli/config: default repo artifacts under .codegraph (db, bench gocache) w/ legacy db fallback * cli/config: harden repo DB path stat handling + centralize .codegraph dir constant --- .codegraphignore | 2 ++ CHANGELOG.md | 2 +- README.md | 2 ++ examples/README.md | 9 +++++---- internal/cli/app.go | 19 +++++++++++++++++-- internal/cli/app_test.go | 2 +- internal/config/config.go | 9 ++++++--- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.codegraphignore b/.codegraphignore index 6d16249..9e0d428 100644 --- a/.codegraphignore +++ b/.codegraphignore @@ -2,6 +2,8 @@ # This file uses .gitignore-like patterns and is loaded by codegraph in addition # to config defaults. +.codegraph/** + codegraph codegraph.exe codegraph.sqlite* diff --git a/CHANGELOG.md b/CHANGELOG.md index acbfb56..7f2b1ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,7 @@ Improved Node.js repo indexing stability by hard-skipping common generated/tooli ### Fixed -- Switched the default database location to per-repository `codegraph.sqlite` and excluded `codegraph.sqlite*` from indexing so repo-local DB mode works on repeated `index` and `serve` runs. +- Switched the default repo-local database location to `.codegraph/codegraph.sqlite` (while continuing to recognize legacy `codegraph.sqlite` at the repo root) and excluded both from indexing so repeated `index`/`serve` runs stay repo-local without accidental self-indexing. - Treated the previous global `db_dir` default as a legacy value so existing installs fall forward to repo-local DB behavior without manual config edits. - Updated Codex MCP setup guidance and examples to use `config.toml` with `startup_timeout_sec = 60`. diff --git a/README.md b/README.md index ec87b9f..30cbe55 100644 --- a/README.md +++ b/README.md @@ -359,6 +359,8 @@ dist/ *.generated.go ``` +Note: `.codegraph/` is always skipped by the indexer (repo-local config/artifacts). + --- ## Architecture diff --git a/examples/README.md b/examples/README.md index 1bf3663..d2c7219 100644 --- a/examples/README.md +++ b/examples/README.md @@ -42,13 +42,14 @@ If `codegraph` is not on `PATH`, replace `command` and `args` with a form that l By default, `codegraph` stores its database in the served repository as: ```text -codegraph.sqlite +.codegraph/codegraph.sqlite ``` If you do not want to commit the local database, ignore: ```text -codegraph.sqlite -codegraph.sqlite-shm -codegraph.sqlite-wal +.codegraph/ + +# Legacy location (still recognized if present): +codegraph.sqlite* ``` diff --git a/internal/cli/app.go b/internal/cli/app.go index 2d351b3..8a9e4b0 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -395,7 +395,7 @@ func runBenchmark(ctx context.Context, stdout io.Writer, args []string) error { } cmd := exec.CommandContext(ctx, "go", cmdArgs...) env := append([]string(nil), os.Environ()...) - gocachePath, err := filepath.Abs(filepath.Join(".", ".gocache")) + gocachePath, err := filepath.Abs(filepath.Join(config.RepoArtifactsDir, "gocache")) if err != nil { return err } @@ -1840,7 +1840,22 @@ func dbPathForRepo(cfg config.Config, repoRoot, canonical string) (string, error if err != nil { return "", err } - return filepath.Join(absRoot, repoDBFileName), nil + newPath := filepath.Join(absRoot, config.RepoArtifactsDir, repoDBFileName) + legacyPath := filepath.Join(absRoot, repoDBFileName) + if _, err := os.Stat(newPath); err == nil { + return newPath, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat %s: %w", newPath, err) + } + if _, err := os.Stat(legacyPath); err == nil { + return legacyPath, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat %s: %w", legacyPath, err) + } + if err := os.MkdirAll(filepath.Dir(newPath), 0o755); err != nil { + return "", err + } + return newPath, nil } return filepath.Join(cfg.DBDir, store.DBFileNameForRepo(canonical)), nil } diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index aaf82ac..328331e 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -223,7 +223,7 @@ func TestRunIndexWithRepoDBDirSkipsRepoDatabaseFiles(t *testing.T) { t.Fatalf("second Run(index) error = %v", err) } - dbPath := filepath.Join(repoRoot, "codegraph.sqlite") + dbPath := filepath.Join(repoRoot, ".codegraph", "codegraph.sqlite") if _, err := os.Stat(dbPath); err != nil { t.Fatalf("expected repo database at %q: %v", dbPath, err) } diff --git a/internal/config/config.go b/internal/config/config.go index aeb79e1..2cc3fcc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,11 +14,14 @@ import ( const fileName = "config.json" const ignoreFileName = ".codegraphignore" +const RepoArtifactsDir = ".codegraph" const RepoDBDir = "repo" const repoDBExcludePattern = "codegraph.sqlite*" -var HardcodedSkips = []string{".git", "node_modules", ".next", ".nuxt", ".svelte-kit", ".turbo", ".pnpm-store", ".yarn", ".parcel-cache"} -var DefaultExcludes = []string{".codegraph/**", ".codegraph-home/**", ".codegraph-home2/**", ".gocache/**", ".gomodcache/**", ".tmp/**", "vendor/**", "dist/**", "build/**", "coverage/**", "out/**", ".cache/**", repoDBExcludePattern} +const repoArtifactsExcludePattern = RepoArtifactsDir + "/**" + +var HardcodedSkips = []string{".git", RepoArtifactsDir, "node_modules", ".next", ".nuxt", ".svelte-kit", ".turbo", ".pnpm-store", ".yarn", ".parcel-cache"} +var DefaultExcludes = []string{repoArtifactsExcludePattern, ".codegraph-home/**", ".codegraph-home2/**", ".gocache/**", ".gomodcache/**", ".tmp/**", "vendor/**", "dist/**", "build/**", "coverage/**", "out/**", ".cache/**", repoDBExcludePattern} type Config struct { DefaultLogLevel string `json:"default_log_level"` @@ -81,7 +84,7 @@ func ConfigPath() (string, error) { } func RepoConfigPath(repoRoot string) string { - return filepath.Join(repoRoot, ".codegraph", "config.json") + return filepath.Join(repoRoot, RepoArtifactsDir, "config.json") } func Load() (Config, error) { From b0bc423a1659200dc58a9754ad4d97f9c456e4de Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:42:11 +0200 Subject: [PATCH 34/59] watch/store: watch config parity + crash-safe dirty queue (claim/delete) (#50) * watch/store: watch config parity + crash-safe dirty queue (claim/delete) * store: make DeleteClaimedDirtyFiles atomic + reuse sqliteInClauseBatchSize --- internal/store/store.go | 225 ++++++++++++++++++++++++++++------- internal/store/store_test.go | 23 +++- internal/watcher/watcher.go | 107 ++++++++++++++--- 3 files changed, 295 insertions(+), 60 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index b2e4645..5e4f17f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3274,7 +3274,7 @@ func (s *Store) QueueDirtyFile(ctx context.Context, repoID int64, path, reason s INSERT INTO dirty_files(repo_id, path, reason, queued_at) VALUES(?, ?, ?, ?) ON CONFLICT(repo_id, path) DO UPDATE SET reason=excluded.reason, queued_at=excluded.queued_at - `, repoID, path, reason, time.Now().UTC().Format(time.RFC3339)) + `, repoID, path, reason, time.Now().UTC().Format(time.RFC3339Nano)) return err } @@ -3305,7 +3305,7 @@ func (s *Store) QueueDirtyFiles(ctx context.Context, repoID int64, paths []strin defer stmt.Close() for _, path := range paths { - if _, err := stmt.ExecContext(ctx, repoID, path, reason, time.Now().UTC().Format(time.RFC3339)); err != nil { + if _, err := stmt.ExecContext(ctx, repoID, path, reason, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { return err } } @@ -3328,42 +3328,130 @@ func (s *Store) HasDirtyFiles(ctx context.Context, repoID int64) (bool, error) { return true, nil } -func (s *Store) DrainDirtyFiles(ctx context.Context, repoID int64) ([]string, error) { - // Prefer an atomic drain. `DELETE ... RETURNING` guarantees we only remove rows - // that are returned to the caller (no SELECT+DELETE race). - rows, err := s.db.QueryContext(ctx, ` - WITH deleted AS ( +// ClaimDirtyFiles snapshots all currently queued dirty paths for a repo and +// marks them as "inflight" by rewriting queued_at/reason in a single write +// transaction. This avoids the crash window inherent in destructive draining +// (delete-then-process) while still allowing successful callers to delete only +// the rows they claimed. +func (s *Store) ClaimDirtyFiles(ctx context.Context, repoID int64, claimAt, claimReason string) ([]string, error) { + if claimAt == "" { + return nil, fmt.Errorf("claimAt must be non-empty") + } + conn, err := s.db.Conn(ctx) + if err != nil { + return nil, err + } + defer conn.Close() + + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return nil, err + } + committed := false + defer func() { + if committed { + return + } + _, _ = conn.ExecContext(ctx, `ROLLBACK`) + }() + + rows, err := conn.QueryContext(ctx, `SELECT path FROM dirty_files WHERE repo_id = ? ORDER BY queued_at`, repoID) + if err != nil { + return nil, err + } + defer rows.Close() + + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + if err := rows.Err(); err != nil { + return nil, err + } + + for _, chunk := range chunkStrings(paths, 900) { + qMarks := strings.Repeat("?,", len(chunk)) + qMarks = qMarks[:len(qMarks)-1] + args := make([]any, 0, 3+len(chunk)) + args = append(args, claimAt, claimReason, repoID) + for _, p := range chunk { + args = append(args, p) + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf(` + UPDATE dirty_files + SET queued_at = ?, reason = ? + WHERE repo_id = ? AND path IN (%s) + `, qMarks), args...); err != nil { + return nil, err + } + } + + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + return nil, err + } + committed = true + return paths, nil +} + +// DeleteClaimedDirtyFiles removes previously-claimed rows, but only if their +// queued_at still matches the claim timestamp. If a path was re-queued after +// claim (queued_at changed), it is retained for the next flush. +func (s *Store) DeleteClaimedDirtyFiles(ctx context.Context, repoID int64, paths []string, claimedAt string) error { + if len(paths) == 0 { + return nil + } + if claimedAt == "" { + return fmt.Errorf("claimedAt must be non-empty") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + committed := false + defer func() { + if committed { + return + } + _ = tx.Rollback() + }() + + for _, chunk := range chunkStrings(paths, sqliteInClauseBatchSize) { + qMarks := strings.Repeat("?,", len(chunk)) + qMarks = qMarks[:len(qMarks)-1] + args := make([]any, 0, 3+len(chunk)) + args = append(args, repoID, claimedAt) + for _, p := range chunk { + args = append(args, p) + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` DELETE FROM dirty_files - WHERE repo_id = ? - RETURNING path, queued_at - ) - SELECT path - FROM deleted - ORDER BY queued_at - `, repoID) - if err == nil { - defer rows.Close() - var out []string - for rows.Next() { - var path string - if err := rows.Scan(&path); err != nil { - return nil, err - } - out = append(out, path) + WHERE repo_id = ? AND queued_at = ? AND path IN (%s) + `, qMarks), args...); err != nil { + return err } - return out, rows.Err() } - // Fallback for older SQLite builds/drivers without `RETURNING`. - // We take a write lock up-front so that events queued concurrently cannot be - // inserted until after we delete the drained rows. - conn, connErr := s.db.Conn(ctx) - if connErr != nil { + if err := tx.Commit(); err != nil { + return err + } + committed = true + return nil +} + +func (s *Store) DrainDirtyFiles(ctx context.Context, repoID int64) ([]string, error) { + // Take a write lock up-front so that events queued concurrently cannot be + // inserted until after we delete the drained rows. Also ensures we can safely + // rollback if scanning fails/cancels (avoids losing work on partial reads). + conn, err := s.db.Conn(ctx) + if err != nil { return nil, err } defer conn.Close() - if _, lockErr := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); lockErr != nil { + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { return nil, err } committed := false @@ -3374,33 +3462,88 @@ func (s *Store) DrainDirtyFiles(ctx context.Context, repoID int64) ([]string, er _, _ = conn.ExecContext(ctx, `ROLLBACK`) }() - selRows, selErr := conn.QueryContext(ctx, `SELECT path FROM dirty_files WHERE repo_id = ? ORDER BY queued_at`, repoID) - if selErr != nil { - return nil, selErr + rows, err := conn.QueryContext(ctx, `DELETE FROM dirty_files WHERE repo_id = ? RETURNING path, queued_at`, repoID) + if err == nil { + defer rows.Close() + type drained struct { + path string + queuedAt string + } + var drainedRows []drained + for rows.Next() { + var d drained + if err := rows.Scan(&d.path, &d.queuedAt); err != nil { + return nil, err + } + drainedRows = append(drainedRows, d) + } + if err := rows.Err(); err != nil { + return nil, err + } + sort.Slice(drainedRows, func(i, j int) bool { return drainedRows[i].queuedAt < drainedRows[j].queuedAt }) + out := make([]string, len(drainedRows)) + for i := range drainedRows { + out[i] = drainedRows[i].path + } + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + return nil, err + } + committed = true + return out, nil + } + + // Fallback for older SQLite builds/drivers without `RETURNING`. + if !isSQLiteReturningUnsupported(err) { + return nil, err + } + + selRows, err := conn.QueryContext(ctx, `SELECT path FROM dirty_files WHERE repo_id = ? ORDER BY queued_at`, repoID) + if err != nil { + return nil, err } defer selRows.Close() var out []string for selRows.Next() { var path string - if scanErr := selRows.Scan(&path); scanErr != nil { - return nil, scanErr + if err := selRows.Scan(&path); err != nil { + return nil, err } out = append(out, path) } - if rowsErr := selRows.Err(); rowsErr != nil { - return nil, rowsErr + if err := selRows.Err(); err != nil { + return nil, err } - if _, delErr := conn.ExecContext(ctx, `DELETE FROM dirty_files WHERE repo_id = ?`, repoID); delErr != nil { - return nil, delErr + if _, err := conn.ExecContext(ctx, `DELETE FROM dirty_files WHERE repo_id = ?`, repoID); err != nil { + return nil, err } - if _, commitErr := conn.ExecContext(ctx, `COMMIT`); commitErr != nil { - return nil, commitErr + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + return nil, err } committed = true return out, nil } +func isSQLiteReturningUnsupported(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "returning") && strings.Contains(msg, "syntax") +} + +func chunkStrings(values []string, chunkSize int) [][]string { + if len(values) == 0 || chunkSize <= 0 { + return nil + } + out := make([][]string, 0, (len(values)+chunkSize-1)/chunkSize) + for start := 0; start < len(values); start += chunkSize { + end := min(start+chunkSize, len(values)) + out = append(out, values[start:end]) + } + return out +} + func (s *Store) lookupSymbolID(ctx context.Context, repoID int64, symbol string, symbolID int64) (int64, error) { if symbolID != 0 { return symbolID, nil diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 4310377..cfacabd 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -48,31 +48,44 @@ func TestDirtyFilesQueueAndDrain(t *testing.T) { t.Fatalf("expected dirty files after queueing") } - paths, err := s.DrainDirtyFiles(ctx, repo.ID) + claimedAt := "2026-01-01T00:00:00Z" + paths, err := s.ClaimDirtyFiles(ctx, repo.ID, claimedAt, "test_inflight") if err != nil { - t.Fatalf("DrainDirtyFiles() error = %v", err) + t.Fatalf("ClaimDirtyFiles() error = %v", err) } if len(paths) != 2 { t.Fatalf("expected 2 paths, got %d (%v)", len(paths), paths) } + if ok, err := s.HasDirtyFiles(ctx, repo.ID); err != nil { + t.Fatalf("HasDirtyFiles() error = %v", err) + } else if !ok { + t.Fatalf("expected dirty files after claim") + } + if err := s.DeleteClaimedDirtyFiles(ctx, repo.ID, paths, claimedAt); err != nil { + t.Fatalf("DeleteClaimedDirtyFiles() error = %v", err) + } if ok, err := s.HasDirtyFiles(ctx, repo.ID); err != nil { t.Fatalf("HasDirtyFiles() error = %v", err) } else if ok { - t.Fatalf("expected no dirty files after drain") + t.Fatalf("expected no dirty files after delete") } if err := s.QueueDirtyFiles(ctx, repo.ID, []string{"c.go", "d.go"}, "batch"); err != nil { t.Fatalf("QueueDirtyFiles() error = %v", err) } - paths, err = s.DrainDirtyFiles(ctx, repo.ID) + claimedAt = "2026-01-01T00:00:01Z" + paths, err = s.ClaimDirtyFiles(ctx, repo.ID, claimedAt, "batch_inflight") if err != nil { - t.Fatalf("DrainDirtyFiles() (2) error = %v", err) + t.Fatalf("ClaimDirtyFiles() (2) error = %v", err) } if len(paths) != 2 { t.Fatalf("expected 2 paths from batch, got %d (%v)", len(paths), paths) } + if err := s.DeleteClaimedDirtyFiles(ctx, repo.ID, paths, claimedAt); err != nil { + t.Fatalf("DeleteClaimedDirtyFiles() (2) error = %v", err) + } } func TestListScansIncludesLanguageCoverage(t *testing.T) { diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index fa739f1..e054046 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -92,6 +93,23 @@ func (w *Watcher) Stats() WatchStats { } } +func isRelPathWithinRepo(rel string) bool { + rel = filepath.ToSlash(filepath.Clean(rel)) + return rel != ".." && !strings.HasPrefix(rel, "../") +} + +func isWatcherConfigPath(rel string) bool { + rel = filepath.ToSlash(filepath.Clean(rel)) + rel = strings.TrimPrefix(rel, "./") + if strings.EqualFold(rel, ".codegraphignore") { + return true + } + if strings.EqualFold(rel, filepath.ToSlash(filepath.Join(config.RepoArtifactsDir, "config.json"))) { + return true + } + return false +} + func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, debounce time.Duration) error { w.eventsSeen.Store(0) w.eventsIgnored.Store(0) @@ -120,11 +138,15 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun includes := repoCfg.Include excludes := repoCfg.Exclude + var forceFullScan atomic.Bool + fsw, err := fsnotify.NewWatcher() if err != nil { return err } - defer fsw.Close() + defer func() { _ = fsw.Close() }() + eventsCh := fsw.Events + errorsCh := fsw.Errors addWatchTree := func(root string) error { return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -138,6 +160,13 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun } rel = filepath.Clean(rel) if d.IsDir() && indexer.ShouldSkipDir(rel, excludes) { + if filepath.Base(rel) == config.RepoArtifactsDir { + // Keep watching repo-local config updates even though artifacts are + // never indexable. + if addErr := fsw.Add(path); addErr != nil { + return addErr + } + } return filepath.SkipDir } } @@ -148,36 +177,60 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun }) } + resetWatcher := func() error { + // Rebuild the watcher so include/exclude changes take effect (especially + // for directories that were previously skipped and not watched). + _ = fsw.Close() + next, err := fsnotify.NewWatcher() + if err != nil { + return err + } + fsw = next + eventsCh = fsw.Events + errorsCh = fsw.Errors + return addWatchTree(repoRoot) + } + if err := addWatchTree(repoRoot); err != nil { return err } flush := func() error { w.drainRuns.Add(1) - paths, err := w.store.DrainDirtyFiles(ctx, repoID) + force := forceFullScan.Swap(false) + claimedAt := time.Now().UTC().Format(time.RFC3339Nano) + paths, err := w.store.ClaimDirtyFiles(ctx, repoID, claimedAt, "watch_inflight") if err != nil { return err } - if len(paths) == 0 { + if len(paths) == 0 && !force { w.flushNoop.Add(1) return nil } w.drainPaths.Add(int64(len(paths))) - _, err = w.indexer.Update(ctx, indexer.Options{ + opts := indexer.Options{ RepoRoot: repoRoot, - Paths: paths, ScanKind: "watch", - }) + } + if force { + opts.ScanKind = "watch_config" + } else { + opts.Paths = paths + } + _, err = w.indexer.Update(ctx, opts) if err != nil { - // `DrainDirtyFiles` is destructive; re-queue the drained paths on failure so - // work isn't silently dropped. - if requeueErr := w.store.QueueDirtyFiles(ctx, repoID, paths, "watch_retry"); requeueErr != nil { - return fmt.Errorf("update failed: %w (retry re-queue failed: %v)", err, requeueErr) - } return err } + if len(paths) > 0 { + deleteCtx := context.WithoutCancel(ctx) + if err := w.store.DeleteClaimedDirtyFiles(deleteCtx, repoID, paths, claimedAt); err != nil { + return err + } + } w.updateRuns.Add(1) - w.updatePaths.Add(int64(len(paths))) + if !force { + w.updatePaths.Add(int64(len(paths))) + } return err } @@ -250,7 +303,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun return ctx.Err() case err := <-flushErrCh: return err - case event, ok := <-fsw.Events: + case event, ok := <-eventsCh: if !ok { return nil } @@ -268,6 +321,32 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun continue } rel = filepath.Clean(rel) + if !isRelPathWithinRepo(rel) { + w.eventsIgnored.Add(1) + continue + } + + // Config changes must be handled before ignore filtering since repo-local + // config lives under ignored paths (e.g. `.codegraph/**`). + if isWatcherConfigPath(rel) { + nextCfg, loadErr := config.LoadRepo(repoRoot) + if loadErr == nil { + includes = nextCfg.Include + excludes = nextCfg.Exclude + forceFullScan.Store(true) + if err := resetWatcher(); err != nil { + return err + } + select { + case flushSignalCh <- struct{}{}: + w.flushSignals.Add(1) + default: + w.coalescedSignals.Add(1) + } + } + w.eventsIgnored.Add(1) + continue + } if indexer.ShouldIgnorePath(rel, excludes) { w.eventsIgnored.Add(1) continue @@ -310,7 +389,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun default: w.coalescedSignals.Add(1) } - case err, ok := <-fsw.Errors: + case err, ok := <-errorsCh: if !ok { return nil } From 0351130ce1b35c3755f048965bb60ba867f250ef Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sat, 25 Apr 2026 08:41:34 +0200 Subject: [PATCH 35/59] perf(store): scale ResolveEdges dot-suffix + cap slash-suffix map growth; changelog v1.0.10 (#52) * perf(store): scale ResolveEdges dot-suffix + cap slash-suffix map growth; changelog v1.0.10 * fix(store): restrict dot-tail2 candidate set to single-dot dst_name --- CHANGELOG.md | 32 +++++++++++ internal/store/store.go | 121 ++++++++++++++++++++++++++++++++++------ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f2b1ac..3cc2be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +# Release v1.0.10 - TBD + +## Changes + +### Watch / Incremental Indexing +- **watch:** Apply repo include/exclude consistently; harden dirty-file draining. (#47) +- **watch:** Make dirty queue crash-safe via claim/delete; align watch config behavior. (#50) +- **watch:** Ignore chmod-only + directory-create events; add repeated-work stats. (#36) +- **indexer:** Target cross-file edge resolution on update runs (preserve path-scoped behavior; avoid repo-wide resolve). (#32, #33) + +### Correctness / Data Hygiene +- **store/indexer:** Purge deleted-file graph rows and nullify cross-file symbol references. (#46) + +### Performance (Store / Indexing) +- **store:** Speed up edge resolver (dotted-edge indexes, dot-tail2 strategy, resolver symbol indexes). (#48) +- **store:** Reduce repo-wide resolver cost (de-correlate dot-suffix fallback; shrink slash-suffix symbol maps to unresolved-name set). (unreleased) +- **indexing/store:** Broad batching + reduced statement pressure across symbols/FTS/inserts; add/extend phase timings + write_stats counters. (#20, #21, #22, #24, #25, #26, #28, #29) +- **indexing:** Reduce tokenization allocations; add tokenize timing stats. (#30) + +### CLI / Output Stability +- **json/jsonl:** Stabilize `watch` and `doctor` machine-readable output (event envelopes; arrays always present; disable HTML escaping). (#38, #43) +- **cli/index:** Stabilize `--jsonl` scan payloads/envelopes (scan_kind, parse_ms, correlation fields; dedupe envelopes; handle phase write errors). (#40, #41, #42) + +### Maintenance / Benchmarking +- **clean/doctor:** Add ANALYZE, WAL checkpoint, incremental vacuum; add `doctor --deep` integrity checks; expand DB diagnostics + FTS optimize. (#37, #39) +- **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44) +- **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45) + +### Config / UX +- **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49) +- **cli/help/commands:** Command registry + per-command help; canonical query command names with backward-compatible aliases; help/usage normalization. (#10, #12, #13, #14, #15, #16, #17, #18, #19) + # Release v1.0.9 - 16-04-2026 Improved Node.js repo indexing stability by hard-skipping common generated/tooling directories (for example node_modules and .next), refining default excludes, and clarifying ignore override behavior. diff --git a/internal/store/store.go b/internal/store/store.go index 5e4f17f..622025a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1946,27 +1946,11 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { // Strategy 3c: Dot-suffix fallback (for qualified names without a slash separator). // Keep as a narrower fallback (multi-dot dst_name only) to preserve existing semantics without paying LIKE cost on common pkg.Func cases. - res, err = tx.ExecContext(ctx, ` - UPDATE edges SET dst_symbol_id = ( - SELECT s.id FROM symbols s - WHERE s.repo_id = ? - AND s.qualified_name LIKE '%.' || edges.dst_name - LIMIT 1 - ) - WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND instr(edges.dst_name, '.') > 0 AND instr(edges.dst_name, '/') = 0 - AND instr(substr(edges.dst_name, instr(edges.dst_name, '.') + 1), '.') > 0 - AND EXISTS ( - SELECT 1 FROM symbols s WHERE s.repo_id = ? - AND s.qualified_name LIKE '%.' || edges.dst_name - ) - `, repoID, repoID, repoID) + n, err = s.resolveEdgesByDotSuffix(ctx, tx, repoID) if err != nil { return 0, err } - if n, _ := res.RowsAffected(); n > 0 { - totalResolved += int(n) - } + totalResolved += n // Strategy 4: Method receiver match (e.g., DoSomething matches MyStruct.DoSomething) res, err = tx.ExecContext(ctx, ` @@ -1996,7 +1980,102 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { return totalResolved, nil } +func (s *Store) resolveEdgesByDotSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_edge_dot_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + return 0, err + } + dropped := false + defer func() { + if dropped { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_edge_dot_suffix`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_edge_dot_suffix`); err != nil { + return 0, err + } + + // De-correlate the expensive LIKE suffix match: do the work once per distinct dst_name, + // then apply the result via a fast equality update on edges.dst_name. + // + // This preserves the prior semantics (pick a single match via MIN(id)) while scaling + // with the number of unique names rather than total unresolved edges. + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_edge_dot_suffix(dst_name, dst_symbol_id) + SELECT e.dst_name, + (SELECT MIN(s.id) FROM symbols s WHERE s.repo_id = ? AND s.qualified_name LIKE '%.' || e.dst_name) + FROM ( + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL + AND instr(dst_name, '.') > 0 AND instr(dst_name, '/') = 0 + AND instr(substr(dst_name, instr(dst_name, '.') + 1), '.') > 0 + ) e + WHERE EXISTS ( + SELECT 1 FROM symbols s + WHERE s.repo_id = ? AND s.qualified_name LIKE '%.' || e.dst_name + ) + `, repoID, repoID, repoID); err != nil { + return 0, err + } + + res, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_edge_dot_suffix t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND edges.dst_name IN (SELECT dst_name FROM tmp_edge_dot_suffix) + `, repoID) + if err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_edge_dot_suffix`); err != nil { + return 0, err + } + dropped = true + n, err := res.RowsAffected() + if err != nil { + return 0, err + } + return int(n), nil +} + func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { + // Restrict suffix maps to names that can actually be consumed by the unresolved edge set. + // This avoids building large Go maps for symbols that can't match any current unresolved edges. + neededSuffix := map[string]struct{}{} + neededTail2 := map[string]struct{}{} + needRows, err := tx.QueryContext(ctx, ` + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '/') = 0 + `, repoID) + if err != nil { + return 0, err + } + for needRows.Next() { + var dstName string + if err := needRows.Scan(&dstName); err != nil { + _ = needRows.Close() + return 0, err + } + if dstName == "" { + continue + } + neededSuffix[dstName] = struct{}{} + firstDot := strings.IndexByte(dstName, '.') + if firstDot >= 0 && firstDot == strings.LastIndexByte(dstName, '.') { + // Dot-tail2 optimization only applies to dst_name values with exactly one dot. + neededTail2[dstName] = struct{}{} + } + } + if err := needRows.Err(); err != nil { + _ = needRows.Close() + return 0, err + } + if err := needRows.Close(); err != nil { + return 0, err + } + rows, err := tx.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) if err != nil { return 0, err @@ -2024,6 +2103,9 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI if slash := strings.LastIndexByte(qname, '/'); slash >= 0 && slash+1 < len(qname) { afterSlash = qname[slash+1:] if afterSlash != "" { + if _, ok := neededSuffix[afterSlash]; !ok { + continue + } if cur, ok := minBySuffix[afterSlash]; !ok { minBySuffix[strings.Clone(afterSlash)] = id } else if id < cur { @@ -2041,6 +2123,9 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI } tail2 := afterSlash[start:] if tail2 != "" { + if _, ok := neededTail2[tail2]; !ok { + continue + } if cur, ok := minByDotTail2[tail2]; !ok { minByDotTail2[strings.Clone(tail2)] = id } else if id < cur { From 0c2d82079ac4a7f91949b6095f906d7e12317281 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Sat, 25 Apr 2026 08:48:36 +0200 Subject: [PATCH 36/59] docs(changelog): normalize release summaries across all tagged versions --- CHANGELOG.md | 131 +++++++++++++++++++++------------------------------ 1 file changed, 55 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cc2be8..843054b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,129 +1,108 @@ # Changelog -# Release v1.0.10 - TBD +## Unreleased -## Changes +Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). -### Watch / Incremental Indexing +### Changed - **watch:** Apply repo include/exclude consistently; harden dirty-file draining. (#47) - **watch:** Make dirty queue crash-safe via claim/delete; align watch config behavior. (#50) - **watch:** Ignore chmod-only + directory-create events; add repeated-work stats. (#36) - **indexer:** Target cross-file edge resolution on update runs (preserve path-scoped behavior; avoid repo-wide resolve). (#32, #33) - -### Correctness / Data Hygiene -- **store/indexer:** Purge deleted-file graph rows and nullify cross-file symbol references. (#46) - -### Performance (Store / Indexing) - **store:** Speed up edge resolver (dotted-edge indexes, dot-tail2 strategy, resolver symbol indexes). (#48) -- **store:** Reduce repo-wide resolver cost (de-correlate dot-suffix fallback; shrink slash-suffix symbol maps to unresolved-name set). (unreleased) - **indexing/store:** Broad batching + reduced statement pressure across symbols/FTS/inserts; add/extend phase timings + write_stats counters. (#20, #21, #22, #24, #25, #26, #28, #29) - **indexing:** Reduce tokenization allocations; add tokenize timing stats. (#30) - -### CLI / Output Stability - **json/jsonl:** Stabilize `watch` and `doctor` machine-readable output (event envelopes; arrays always present; disable HTML escaping). (#38, #43) - **cli/index:** Stabilize `--jsonl` scan payloads/envelopes (scan_kind, parse_ms, correlation fields; dedupe envelopes; handle phase write errors). (#40, #41, #42) - -### Maintenance / Benchmarking - **clean/doctor:** Add ANALYZE, WAL checkpoint, incremental vacuum; add `doctor --deep` integrity checks; expand DB diagnostics + FTS optimize. (#37, #39) - **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44) - **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45) - -### Config / UX - **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49) - **cli/help/commands:** Command registry + per-command help; canonical query command names with backward-compatible aliases; help/usage normalization. (#10, #12, #13, #14, #15, #16, #17, #18, #19) -# Release v1.0.9 - 16-04-2026 - -Improved Node.js repo indexing stability by hard-skipping common generated/tooling directories (for example node_modules and .next), refining default excludes, and clarifying ignore override behavior. - -## Changes +### Fixed +- **store/indexer:** Purge deleted-file graph rows and nullify cross-file symbol references. (#46) -### Fixes -- **Indexer:** Established a strict skip policy for common Node.js generated directories (e.g., `node_modules`, `.next`, `.nuxt`). These are now hardcoded and enforced early during filesystem traversal. -- **Indexer:** Clarified ignore override behavior; hardcoded skips are now non-overridable via negation patterns in `.codegraphignore` to ensure predictable indexer performance. -- **Config:** Centralized default exclude patterns to maintain consistency across the CLI and indexer. -- **SQL Hardening:** Added explicit bounds and safety checks for all path-filtering SQL queries. +## v1.0.9 - 2026-04-16 -## Upgrade Notes -- No required migrations or configuration changes. +Improved Node.js repo indexing stability by hard-skipping common generated/tooling directories (for example `node_modules` and `.next`), refining default excludes, and clarifying ignore override behavior. -## v1.0.7 - 27-03-2026 +### Changed +- **indexer:** Enforce strict early skips for common Node.js generated directories; hardcoded skips are non-overridable via `.codegraphignore` negations. (#8) +- **config:** Centralize default exclude patterns to keep CLI/indexer behavior consistent. (#8) ### Fixed +- **sql:** Add explicit bounds/safety checks for path-filtering queries. (#8) +- **build/release:** Carry through `CGO_ENABLED=0` + tree-sitter cross-compilation fixes and release diagnostics. (#4, #5, #6) -- Restored release cross-compilation by splitting tree-sitter adapters behind `//go:build cgo` and using heuristic parsers in `CGO_ENABLED=0` builds. +### Docs +- **readme/changelog:** Update Node.js support status and release notes. (#8) -## v1.0.6 - 26-03-2026 +## v1.0.8 - 2026-03-27 ### Fixed - -- Stopped sending JSON-RPC responses to MCP notifications (`notifications/initialized` and other `notifications/*` methods), which violated the protocol and caused strict clients to fail on connect. -- Changed tool schema `"required": null` to omit the field when empty, fixing JSON Schema validation failures in strict MCP clients. -- Removed non-standard `structuredContent` field from tool call responses to conform to the MCP spec. -- Routed unhandled-method logging through the configured stderr writer instead of Go's default logger. +- **build:** Restore `CGO_ENABLED=0` cross-compilation by splitting tree-sitter adapters behind `//go:build cgo` and using heuristic parsers in no-cgo builds. (#5) ### Changed +- **ci:** Add release build diagnostics to improve cross-platform release debugging. (ci/workflow) -- `NewServer` now accepts an `io.Writer` for error output, giving callers control over diagnostic logging. +## v1.0.7 - 2026-03-27 -### Docs +### Fixed +- **mcp:** Tighten MCP protocol compliance for stricter clients. (#3) -- Added Claude Code MCP setup section to README with `.mcp.json` examples. -- Added missing `list_scans` and `latest_scan_errors` to the MCP tools list in README. -- Added one-line descriptions to all 14 MCP tools in README. +### Docs +- Add `v1.0.6` changelog entry. (docs) -## v1.0.5 - 21-03-2026 +## v1.0.6 - 2026-03-26 ### Fixed - -- Switched the default repo-local database location to `.codegraph/codegraph.sqlite` (while continuing to recognize legacy `codegraph.sqlite` at the repo root) and excluded both from indexing so repeated `index`/`serve` runs stay repo-local without accidental self-indexing. -- Treated the previous global `db_dir` default as a legacy value so existing installs fall forward to repo-local DB behavior without manual config edits. -- Updated Codex MCP setup guidance and examples to use `config.toml` with `startup_timeout_sec = 60`. - -## v1.0.3 - 18-03-2026 +- **mcp:** Stop sending JSON-RPC responses to notifications; fix JSON Schema `required` handling; remove non-standard fields; route unhandled-method logging via configured stderr writer. (mcp) ### Changed +- `NewServer` accepts an `io.Writer` for error output, giving callers control over diagnostic logging. (mcp) -- Simplified core code paths in `internal/store`, `internal/mcp`, `internal/cli`, and parser adapters while preserving behavior. -- Reduced duplicated row-scanning and pagination parsing logic to improve maintainability. -- Cached MCP tool definition payload construction for lower repeated allocation overhead on `tools/list`. +### Docs +- Add Claude Code MCP setup examples; add missing tools to MCP docs list; add short tool descriptions. (readme) -### Notes +## v1.0.5 - 2026-03-21 -- This release is focused on code quality, readability, and safe internal optimization with no intended user-facing breaking changes. +### Fixed +- Default to a repo-local SQLite DB (while continuing to recognize legacy locations) and exclude repo DB artifacts from indexing. (config/store) -## v1.0.2 - 2026-03-18 +### Changed +- Treat prior global `db_dir` default as legacy so existing installs fall forward safely. (config) -### Highlights +## v1.0.4 - 2026-03-18 -- Added `watch --jsonl`, benchmark improvements, and `config init` workflow updates. -- Added/expanded query, export, MCP, and parser capabilities (including Python and heuristic multi-language adapters). -- Improved indexing and store performance with batching, scoped edge resolution, and scan/stat tuning. -- Added cleaner local maintenance workflows (`clean`, doctor improvements, and setup/path guidance updates). +### Docs +- README update to include graph/export usage. (docs) -## v1.0.1 - 18-03-2026 +## v1.0.3 - 2026-03-18 -### Fixes +### Added +- **cli:** `watch`, `benchmark`, `config init`, `clean`, `doctor --fix`, and `--jsonl` output for long-running/indexing workflows. (cli) +- **mcp/query:** Query commands + tools, including offset pagination and supported-languages introspection. (mcp) +- **parser:** Heuristic adapters for major languages plus a Python adapter. (parser) +- **export:** Include symbols + edges in graph exports; support export streaming. (export) -- Corrected Go module path from `github.com/example/localcodegraph` to `github.com/isink17/codegraph`. -- Updated internal imports to match the published module path. -- Updated install docs with actual module path and Go install/PATH troubleshooting guidance. +### Changed +- **indexer/scan:** `.codegraphignore` negation patterns; per-language scan coverage; best-effort parse policy; batched metadata writes and scoped edge resolution. (indexer) +- **performance:** Parallelize indexing and reduce allocation/IO overhead; improve watcher flush/coalescing; add scan phase timings; SQLite/store tuning. (perf) -## v1.0.0 - 18-03-2026 +### Notes +- **licensing:** Relicensed under FSL-1.1 to prevent commercial reselling. (license) -Initial public release of `codegraph`. +## v1.0.2 - 2026-03-18 + +### Fixed +- Installation hardening + README updates to unblock `go install` workflows. (install/docs) -### Highlights +## v1.0.1 - 2026-03-18 -- Added a local-first Go CLI for installation, indexing, updates, stats, graph export, doctor checks, watch mode, and MCP serving. -- Added a SQLite-backed repository graph with explicit migrations and incremental file hashing. -- Added a parser abstraction with a working Go parser adapter and a clean seam for future Tree-sitter adapters. -- Added MCP stdio support with generic tools for indexing, symbol lookup, call graph navigation, impact analysis, related test discovery, semantic search, and graph stats. -- Added release automation for macOS, Linux, and Windows GitHub release artifacts. -- Added agent-oriented documentation for Codex-style clients, Gemini CLI, and Claude-compatible MCP configuration examples. -- Added initial automated tests for install flow, platform paths, incremental indexing, and MCP `graph_stats`. +### Fixed +- Correct Go module path to `github.com/isink17/codegraph`; align imports and install docs accordingly. (install/docs) -### Notes +## v1.0.0 - 2026-03-18 -- This release targets local-first usage on macOS and Linux first, with Windows kept feasible by design. -- The parser subsystem is intentionally shaped for Tree-sitter-backed adapters, while the first shipping implementation uses the Go standard AST for reliability and simple installation. +Initial public release of `codegraph`. From 8794f5dcf816ed80e17e35941d0884f17dcacfa5 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sat, 25 Apr 2026 09:06:20 +0200 Subject: [PATCH 37/59] perf(store): de-correlate ResolveEdges 1/2/4 by distinct dst_name; add ResolveEdgesForNames scale bench (#53) --- .../resolve_edges_for_names_bench_test.go | 98 +++++++++++++++++ internal/store/store.go | 103 ++++++++++++------ 2 files changed, 165 insertions(+), 36 deletions(-) create mode 100644 internal/store/resolve_edges_for_names_bench_test.go diff --git a/internal/store/resolve_edges_for_names_bench_test.go b/internal/store/resolve_edges_for_names_bench_test.go new file mode 100644 index 0000000..9fa842a --- /dev/null +++ b/internal/store/resolve_edges_for_names_bench_test.go @@ -0,0 +1,98 @@ +package store + +import ( + "context" + "fmt" + "path/filepath" + "testing" +) + +func BenchmarkResolveEdgesForNames_CrossFileScale(b *testing.B) { + ctx := context.Background() + dbPath := filepath.Join(b.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + b.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, b.TempDir()) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + const ( + numFiles = 200 + numNames = 1500 + numEdges = 20000 + numPkgs = 50 + dstPerPkg = numNames / numPkgs + ) + + fileIDs := make([]int64, 0, numFiles) + for i := 0; i < numFiles; i++ { + id, err := insertTestFile(ctx, s, repo.ID, fmt.Sprintf("f_%d.go", i)) + if err != nil { + b.Fatalf("insertTestFile() error = %v", err) + } + fileIDs = append(fileIDs, id) + } + + // One src symbol per file to attach edges to. + srcIDs := make([]int64, 0, numFiles) + for i, fileID := range fileIDs { + id, err := insertTestSymbol(ctx, s, repo.ID, fileID, fmt.Sprintf("Src_%d", i), fmt.Sprintf("Src_%d", i)) + if err != nil { + b.Fatalf("insertTestSymbol(src) error = %v", err) + } + srcIDs = append(srcIDs, id) + } + + // Dst symbols spread across files/packages with stable qualified names. + for i := 0; i < numNames; i++ { + pkg := i % numPkgs + name := fmt.Sprintf("Name_%d", i) + qualified := fmt.Sprintf("pkg_%d.%s", pkg, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repo.ID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(dst) error = %v", err) + } + } + + for i := 0; i < numEdges; i++ { + pkg := i % numPkgs + nameIdx := (i % dstPerPkg) + (pkg * dstPerPkg) + if nameIdx >= numNames { + nameIdx = i % numNames + } + dstName := fmt.Sprintf("pkg_%d.Name_%d", pkg, nameIdx) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repo.ID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + names := make([]string, 0, numNames+(numNames/10)) + for i := 0; i < numNames; i++ { + names = append(names, fmt.Sprintf("Name_%d", i)) + } + // Add some duplicates/whitespace to exercise dedupe/trim without changing semantics. + for i := 0; i < numNames/10; i++ { + names = append(names, " "+fmt.Sprintf("Name_%d", i)+" ") + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + if _, err := s.db.ExecContext(ctx, `UPDATE edges SET dst_symbol_id = NULL WHERE repo_id = ?`, repo.ID); err != nil { + b.Fatalf("reset edges error = %v", err) + } + b.StartTimer() + + if _, err := s.ResolveEdgesForNamesWithStats(ctx, repo.ID, names); err != nil { + b.Fatalf("ResolveEdgesForNamesWithStats() error = %v", err) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 622025a..fca6ca9 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1902,13 +1902,26 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { // Strategy 1: Exact qualified name match res, err := tx.ExecContext(ctx, ` - UPDATE edges SET dst_symbol_id = ( - SELECT s.id FROM symbols s - WHERE s.repo_id = ? AND s.qualified_name = edges.dst_name - LIMIT 1 + WITH distinct_names AS ( + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + ), + resolutions AS ( + SELECT + n.dst_name, + ( + SELECT s.id + FROM symbols s + WHERE s.repo_id = ? AND s.qualified_name = n.dst_name + LIMIT 1 + ) AS dst_symbol_id + FROM distinct_names n ) + UPDATE edges + SET dst_symbol_id = (SELECT r.dst_symbol_id FROM resolutions r WHERE r.dst_name = edges.dst_name) WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND EXISTS (SELECT 1 FROM symbols s WHERE s.repo_id = ? AND s.qualified_name = edges.dst_name) + AND edges.dst_name IN (SELECT dst_name FROM resolutions WHERE dst_symbol_id IS NOT NULL) `, repoID, repoID, repoID) if err != nil { return 0, err @@ -1919,14 +1932,27 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { // Strategy 2: Name match (unqualified) res, err = tx.ExecContext(ctx, ` - UPDATE edges SET dst_symbol_id = ( - SELECT s.id FROM symbols s - WHERE s.repo_id = ? AND s.name = edges.dst_name - AND s.kind IN ('function', 'method', 'class', 'type', 'struct', 'interface') - LIMIT 1 + WITH distinct_names AS ( + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + ), + resolutions AS ( + SELECT + n.dst_name, + ( + SELECT s.id + FROM symbols s + WHERE s.repo_id = ? AND s.name = n.dst_name + AND s.kind IN ('function', 'method', 'class', 'type', 'struct', 'interface') + LIMIT 1 + ) AS dst_symbol_id + FROM distinct_names n ) + UPDATE edges + SET dst_symbol_id = (SELECT r.dst_symbol_id FROM resolutions r WHERE r.dst_name = edges.dst_name) WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND EXISTS (SELECT 1 FROM symbols s WHERE s.repo_id = ? AND s.name = edges.dst_name AND s.kind IN ('function', 'method', 'class', 'type', 'struct', 'interface')) + AND edges.dst_name IN (SELECT dst_name FROM resolutions WHERE dst_symbol_id IS NOT NULL) `, repoID, repoID, repoID) if err != nil { return 0, err @@ -1954,18 +1980,28 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { // Strategy 4: Method receiver match (e.g., DoSomething matches MyStruct.DoSomething) res, err = tx.ExecContext(ctx, ` - UPDATE edges SET dst_symbol_id = ( - SELECT s.id FROM symbols s - WHERE s.repo_id = ? - AND s.name = edges.dst_name - AND s.container_name != '' - LIMIT 1 + WITH distinct_names AS ( + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' + ), + resolutions AS ( + SELECT + n.dst_name, + ( + SELECT s.id + FROM symbols s + WHERE s.repo_id = ? + AND s.name = n.dst_name + AND s.container_name != '' + LIMIT 1 + ) AS dst_symbol_id + FROM distinct_names n ) + UPDATE edges + SET dst_symbol_id = (SELECT r.dst_symbol_id FROM resolutions r WHERE r.dst_name = edges.dst_name) WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND EXISTS ( - SELECT 1 FROM symbols s WHERE s.repo_id = ? - AND s.name = edges.dst_name AND s.container_name != '' - ) + AND edges.dst_name IN (SELECT dst_name FROM resolutions WHERE dst_symbol_id IS NOT NULL) `, repoID, repoID, repoID) if err != nil { return 0, err @@ -2379,11 +2415,6 @@ func (s *Store) ResolveEdgesForNamesWithStats(ctx context.Context, repoID int64, } stats.NamesUnique = len(unique) - nameSet := make(map[string]struct{}, len(unique)) - for _, name := range unique { - nameSet[name] = struct{}{} - } - // Candidate selection: // // 1) Use indexed exact matches for dst_name = (covers the common case @@ -2454,16 +2485,16 @@ func (s *Store) ResolveEdgesForNamesWithStats(ctx context.Context, repoID int64, return stats, err } stats.QualifiedScanned++ - if _, ok := targetByID[id]; ok { - continue - } - if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { - if _, ok := nameSet[dstName[dot+1:]]; ok { - targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} - stats.SuffixHits++ + if _, ok := targetByID[id]; ok { + continue + } + if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { + if _, ok := seen[dstName[dot+1:]]; ok { + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.SuffixHits++ + } } } - } if err := rows.Err(); err != nil { _ = rows.Close() return stats, err @@ -2516,7 +2547,7 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] if len(targets) == 0 { return nil } - qualifiedSet := map[string]struct{}{} + qualifiedSet := make(map[string]struct{}, len(targets)) for _, target := range targets { if target.dstName != "" { qualifiedSet[target.dstName] = struct{}{} @@ -2542,7 +2573,7 @@ func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets [] return name } - shortSet := map[string]struct{}{} + shortSet := make(map[string]struct{}, len(targets)) for _, target := range targets { if _, ok := byQualified[target.dstName]; ok { continue From 3392b4b0508650d073e7ae21ac6a1b6f4137e9cd Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sat, 25 Apr 2026 09:20:49 +0200 Subject: [PATCH 38/59] fix(store): scope RelatedTests(file) + deterministic symbol lookup; perf: speed up FindDeadCode with indexes (#54) * fix(store): scope RelatedTests(file) + deterministic symbol lookup; perf: speed up FindDeadCode with indexes * Update internal/query/service.go --- internal/query/service.go | 30 +++-- internal/store/related_tests_test.go | 67 +++++++++++ .../014_references_symbol_id_indexes.sql | 10 ++ .../schema/015_test_links_target_file_idx.sql | 5 + internal/store/store.go | 106 ++++++++++++++---- 5 files changed, 188 insertions(+), 30 deletions(-) create mode 100644 internal/store/related_tests_test.go create mode 100644 internal/store/schema/014_references_symbol_id_indexes.sql create mode 100644 internal/store/schema/015_test_links_target_file_idx.sql diff --git a/internal/query/service.go b/internal/query/service.go index 388d9b4..68f62ec 100644 --- a/internal/query/service.go +++ b/internal/query/service.go @@ -59,12 +59,20 @@ func (s *Service) RelatedTests(ctx context.Context, repoID int64, symbol, file s } func (s *Service) RelatedTestsForFiles(ctx context.Context, repoID int64, files []string, limit, offset int) ([]store.RelatedTest, error) { + if offset < 0 { + offset = 0 + } + if limit <= 0 { + limit = 20 + } +perFileLimit := min(max(50, limit+offset), 1000) + seen := map[string]bool{} var all []store.RelatedTest for _, f := range files { - tests, err := s.store.RelatedTests(ctx, repoID, "", f, 50, 0) + tests, err := s.store.RelatedTests(ctx, repoID, "", f, perFileLimit, 0) if err != nil { - continue + return nil, err } for _, t := range tests { key := t.File + "::" + t.Symbol @@ -74,19 +82,19 @@ func (s *Service) RelatedTestsForFiles(ctx context.Context, repoID int64, files } } } - // Sort by score descending. + // Sort deterministically: score desc, then file, then symbol. sort.Slice(all, func(i, j int) bool { - return all[i].Score > all[j].Score + if all[i].Score != all[j].Score { + return all[i].Score > all[j].Score + } + if all[i].File != all[j].File { + return all[i].File < all[j].File + } + return all[i].Symbol < all[j].Symbol }) // Apply limit/offset. - if offset < 0 { - offset = 0 - } - if limit <= 0 { - limit = 20 - } if offset >= len(all) { - return nil, nil + return []store.RelatedTest{}, nil } end := offset + limit if end > len(all) { diff --git a/internal/store/related_tests_test.go b/internal/store/related_tests_test.go new file mode 100644 index 0000000..a8078dd --- /dev/null +++ b/internal/store/related_tests_test.go @@ -0,0 +1,67 @@ +package store + +import ( + "context" + "path/filepath" + "testing" +) + +func TestRelatedTests_FileScoped(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + targetA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + targetB, err := insertTestFile(ctx, s, repo.ID, "b.go") + if err != nil { + t.Fatalf("insertTestFile(b.go) error = %v", err) + } + testFile, err := insertTestFile(ctx, s, repo.ID, "a_test.go") + if err != nil { + t.Fatalf("insertTestFile(a_test.go) error = %v", err) + } + testSym, err := insertTestSymbol(ctx, s, repo.ID, testFile, "TestA", "TestA") + if err != nil { + t.Fatalf("insertTestSymbol(test) error = %v", err) + } + + // Two links: one for a.go, one for b.go. Querying by file must return only the matching one. + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_file_id, target_symbol_id, reason, score) + VALUES(?, ?, ?, ?, NULL, 'name_match', 0.9) + `, repo.ID, testFile, testSym, targetA); err != nil { + t.Fatalf("insert test_link(a.go) error = %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_file_id, target_symbol_id, reason, score) + VALUES(?, ?, ?, ?, NULL, 'name_match', 0.9) + `, repo.ID, testFile, testSym, targetB); err != nil { + t.Fatalf("insert test_link(b.go) error = %v", err) + } + + got, err := s.RelatedTests(ctx, repo.ID, "", "a.go", 10, 0) + if err != nil { + t.Fatalf("RelatedTests(file=a.go) error = %v", err) + } + if len(got) != 1 { + t.Fatalf("RelatedTests(file=a.go) len=%d, want 1", len(got)) + } + if got[0].File != "a_test.go" { + t.Fatalf("RelatedTests(file=a.go) file=%q, want %q", got[0].File, "a_test.go") + } + if got[0].Symbol != "TestA" { + t.Fatalf("RelatedTests(file=a.go) symbol=%q, want %q", got[0].Symbol, "TestA") + } +} diff --git a/internal/store/schema/014_references_symbol_id_indexes.sql b/internal/store/schema/014_references_symbol_id_indexes.sql new file mode 100644 index 0000000..474da5f --- /dev/null +++ b/internal/store/schema/014_references_symbol_id_indexes.sql @@ -0,0 +1,10 @@ +-- Supports read-heavy queries that filter by symbol_id/context_symbol_id, +-- including FindDeadCode and cleanup paths. +CREATE INDEX IF NOT EXISTS idx_refs_repo_symbol_id +ON references_tbl(repo_id, symbol_id) +WHERE symbol_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_refs_repo_context_symbol_id +ON references_tbl(repo_id, context_symbol_id) +WHERE context_symbol_id IS NOT NULL; + diff --git a/internal/store/schema/015_test_links_target_file_idx.sql b/internal/store/schema/015_test_links_target_file_idx.sql new file mode 100644 index 0000000..86f383f --- /dev/null +++ b/internal/store/schema/015_test_links_target_file_idx.sql @@ -0,0 +1,5 @@ +-- Supports RelatedTests(file=...) lookups without scanning test_links. +CREATE INDEX IF NOT EXISTS idx_test_links_repo_target_file +ON test_links(repo_id, target_file_id) +WHERE target_file_id IS NOT NULL; + diff --git a/internal/store/store.go b/internal/store/store.go index fca6ca9..0fea91b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2485,16 +2485,16 @@ func (s *Store) ResolveEdgesForNamesWithStats(ctx context.Context, repoID int64, return stats, err } stats.QualifiedScanned++ - if _, ok := targetByID[id]; ok { - continue - } - if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { - if _, ok := seen[dstName[dot+1:]]; ok { - targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} - stats.SuffixHits++ - } + if _, ok := targetByID[id]; ok { + continue + } + if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { + if _, ok := seen[dstName[dot+1:]]; ok { + targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.SuffixHits++ } } + } if err := rows.Err(); err != nil { _ = rows.Close() return stats, err @@ -2899,6 +2899,7 @@ func (s *Store) FindCallers(ctx context.Context, repoID int64, symbol string, sy JOIN symbols s ON s.id = e.src_symbol_id JOIN files f ON f.id = s.file_id WHERE e.repo_id = ? AND e.dst_symbol_id = ? + ORDER BY s.qualified_name ASC, s.start_line ASC, s.start_col ASC, s.id ASC LIMIT ? OFFSET ? `, repoID, targetID, safeLimit(limit), safeOffset(offset)) @@ -2920,6 +2921,7 @@ func (s *Store) FindCallees(ctx context.Context, repoID int64, symbol string, sy JOIN symbols s ON s.id = e.dst_symbol_id JOIN files f ON f.id = s.file_id WHERE e.repo_id = ? AND e.src_symbol_id = ? + ORDER BY s.qualified_name ASC, s.start_line ASC, s.start_col ASC, s.id ASC LIMIT ? OFFSET ? `, repoID, srcID, safeLimit(limit), safeOffset(offset)) @@ -2940,6 +2942,10 @@ func (s *Store) ImpactRadius(ctx context.Context, repoID int64, symbols []string queue = append(queue, id) } for _, file := range files { + file = normalizeRepoRelPath(file) + if file == "" { + continue + } rows, err := s.db.QueryContext(ctx, ` SELECT s.id, s.file_id, s.language, s.kind, s.name, s.qualified_name, s.container_name, s.signature, s.visibility, s.start_line, s.start_col, s.end_line, s.end_col, s.doc_summary, s.stable_key, f.path @@ -2959,6 +2965,9 @@ func (s *Store) ImpactRadius(ctx context.Context, repoID int64, symbols []string queue = append(queue, sym.ID) } _ = rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } } if depth <= 0 { depth = 2 @@ -3015,6 +3024,22 @@ func (s *Store) ImpactRadius(ctx context.Context, repoID int64, symbols []string fileList = append(fileList, sym.FilePath) } } + sort.Slice(symbolList, func(i, j int) bool { + if symbolList[i].FilePath != symbolList[j].FilePath { + return symbolList[i].FilePath < symbolList[j].FilePath + } + if symbolList[i].QualifiedName != symbolList[j].QualifiedName { + return symbolList[i].QualifiedName < symbolList[j].QualifiedName + } + if symbolList[i].Range.StartLine != symbolList[j].Range.StartLine { + return symbolList[i].Range.StartLine < symbolList[j].Range.StartLine + } + if symbolList[i].Range.StartCol != symbolList[j].Range.StartCol { + return symbolList[i].Range.StartCol < symbolList[j].Range.StartCol + } + return symbolList[i].ID < symbolList[j].ID + }) + sort.Strings(fileList) return map[string]any{ "symbols": symbolList, "files": fileList, @@ -3042,6 +3067,7 @@ func (s *Store) impactNeighbors(ctx context.Context, repoID int64, frontier []in JOIN symbols s ON s.id = e.src_symbol_id JOIN files f ON f.id = s.file_id WHERE e.repo_id = ? AND e.dst_symbol_id IN (` + placeholders + `) + ORDER BY s.qualified_name ASC, s.start_line ASC, s.start_col ASC, s.id ASC ` if !callers { query = ` @@ -3051,6 +3077,7 @@ func (s *Store) impactNeighbors(ctx context.Context, repoID int64, frontier []in JOIN symbols s ON s.id = e.dst_symbol_id JOIN files f ON f.id = s.file_id WHERE e.repo_id = ? AND e.src_symbol_id IN (` + placeholders + `) AND e.dst_symbol_id IS NOT NULL + ORDER BY s.qualified_name ASC, s.start_line ASC, s.start_col ASC, s.id ASC ` } args := make([]any, 0, len(chunk)+1) @@ -3075,17 +3102,28 @@ func (s *Store) RelatedTests(ctx context.Context, repoID int64, symbol, file str var rows *sql.Rows var err error if file != "" { + file = normalizeRepoRelPath(file) + if file == "" { + return []RelatedTest{}, nil + } + var targetFileID int64 + if err := s.db.QueryRowContext(ctx, `SELECT id FROM files WHERE repo_id = ? AND path = ? LIMIT 1`, repoID, file).Scan(&targetFileID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return []RelatedTest{}, nil + } + return nil, err + } rows, err = s.db.QueryContext(ctx, ` SELECT f.path, COALESCE(s.qualified_name, ''), t.reason, t.score FROM test_links t JOIN files f ON f.id = t.test_file_id LEFT JOIN symbols s ON s.id = t.test_symbol_id - WHERE t.repo_id = ? AND t.test_file_id IN ( - SELECT id FROM files WHERE repo_id = ? AND path LIKE '%_test.go' - ) + WHERE t.repo_id = ? AND t.target_file_id = ? + AND f.path LIKE '%_test.go' + ORDER BY t.score DESC, f.path, COALESCE(s.qualified_name, '') LIMIT ? OFFSET ? - `, repoID, repoID, safeLimit(limit), safeOffset(offset)) + `, repoID, targetFileID, safeLimit(limit), safeOffset(offset)) } else { targetID, err := s.lookupSymbolID(ctx, repoID, symbol, 0) if err != nil { @@ -3097,6 +3135,7 @@ func (s *Store) RelatedTests(ctx context.Context, repoID int64, symbol, file str JOIN files f ON f.id = t.test_file_id LEFT JOIN symbols s ON s.id = t.test_symbol_id WHERE t.repo_id = ? AND t.target_symbol_id = ? + ORDER BY t.score DESC, f.path, COALESCE(s.qualified_name, '') LIMIT ? OFFSET ? `, repoID, targetID, safeLimit(limit), safeOffset(offset)) @@ -3111,6 +3150,7 @@ func (s *Store) RelatedTests(ctx context.Context, repoID int64, symbol, file str if err := rows.Scan(&item.File, &item.Symbol, &item.Reason, &item.Score); err != nil { return nil, err } + item.File = filepath.ToSlash(item.File) out = append(out, item) } return out, rows.Err() @@ -3666,8 +3706,18 @@ func (s *Store) lookupSymbolID(ctx context.Context, repoID int64, symbol string, } var id int64 err := s.db.QueryRowContext(ctx, ` - SELECT id FROM symbols WHERE repo_id = ? AND (qualified_name = ? OR name = ?) LIMIT 1 - `, repoID, symbol, symbol).Scan(&id) + SELECT s.id + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE s.repo_id = ? AND (s.qualified_name = ? OR s.name = ?) + ORDER BY + CASE WHEN s.qualified_name = ? THEN 0 ELSE 1 END, + f.path ASC, + s.start_line ASC, + s.start_col ASC, + s.id ASC + LIMIT 1 + `, repoID, symbol, symbol, symbol).Scan(&id) return id, err } @@ -3679,6 +3729,8 @@ func scanSymbol(scanner interface{ Scan(dest ...any) error }) (graph.Symbol, err ); err != nil { return graph.Symbol{}, err } + // Normalize paths in outputs to be deterministic across platforms and call sites. + sym.FilePath = filepath.ToSlash(sym.FilePath) return sym, nil } @@ -4105,6 +4157,20 @@ func safeOffset(offset int) int { return max(offset, 0) } +func normalizeRepoRelPath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + // Normalize common caller variations: forward slashes, leading ./, etc. + path = filepath.FromSlash(path) + path = filepath.Clean(path) + if path == "." { + return "" + } + return path +} + func quoteFTS(query string) string { tokens := strings.Fields(query) for i, token := range tokens { @@ -4165,11 +4231,13 @@ func (s *Store) FindDeadCode(ctx context.Context, repoID int64, limit, offset in JOIN files f ON f.id = s.file_id WHERE s.repo_id = ? AND s.kind IN ('function', 'method', 'type', 'class', 'struct', 'interface') - AND s.id NOT IN ( - SELECT DISTINCT dst_symbol_id FROM edges WHERE repo_id = ? AND dst_symbol_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM edges e + WHERE e.repo_id = s.repo_id AND e.dst_symbol_id = s.id ) - AND s.id NOT IN ( - SELECT DISTINCT symbol_id FROM references_tbl WHERE repo_id = ? AND symbol_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM references_tbl r + WHERE r.repo_id = s.repo_id AND r.symbol_id = s.id ) AND s.name NOT IN ('main', 'init', 'Main', 'Init') AND s.name NOT LIKE 'Test%' @@ -4177,7 +4245,7 @@ func (s *Store) FindDeadCode(ctx context.Context, repoID int64, limit, offset in AND s.name NOT LIKE 'Example%' ORDER BY f.path, s.start_line LIMIT ? OFFSET ? - `, repoID, repoID, repoID, safeLimit(limit), safeOffset(offset)) + `, repoID, safeLimit(limit), safeOffset(offset)) if err != nil { return nil, err } From affc0829e7933fbcb214d0b69e800dbaa1b527f2 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sun, 26 Apr 2026 16:03:50 +0200 Subject: [PATCH 39/59] perf(store): cache batch-insert SQL + speed up edge src selection in write path (#55) * perf(store): cache batch-insert SQL + speed up edge src selection in write path * fix(store): restore nested-function correctness in srcSymbolChooser --- internal/store/src_symbol_chooser_test.go | 35 ++++++ internal/store/store.go | 127 +++++++++++++++++----- 2 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 internal/store/src_symbol_chooser_test.go diff --git a/internal/store/src_symbol_chooser_test.go b/internal/store/src_symbol_chooser_test.go new file mode 100644 index 0000000..fc5c84e --- /dev/null +++ b/internal/store/src_symbol_chooser_test.go @@ -0,0 +1,35 @@ +package store + +import ( + "testing" + + "github.com/isink17/codegraph/internal/graph" +) + +func TestSrcSymbolChooserChoose_NestedFunctions(t *testing.T) { + stableToID := map[string]int64{ + "outer": 1, + "inner": 2, + } + symbols := []graph.Symbol{ + { + Kind: "function", + StableKey: "outer", + Range: graph.Position{StartLine: 10, EndLine: 50}, + }, + { + Kind: "function", + StableKey: "inner", + Range: graph.Position{StartLine: 20, EndLine: 30}, + }, + } + + c := newSrcSymbolChooser(stableToID, symbols) + + if got := c.Choose(25); got != 2 { + t.Fatalf("Choose(25) = %d, want %d (inner)", got, 2) + } + if got := c.Choose(35); got != 1 { + t.Fatalf("Choose(35) = %d, want %d (outer)", got, 1) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 0fea91b..65f189f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -16,6 +16,7 @@ import ( "slices" "sort" "strings" + "sync" "time" "github.com/isink17/codegraph/internal/graph" @@ -1263,7 +1264,7 @@ func insertParsedFileGraph( parsed graph.ParsedFile, stats *WriteStats, ) error { - stableToID := map[string]int64{} + stableToID := make(map[string]int64, len(parsed.Symbols)) symbolTokenArgs := make([]any, 0, sqliteTokenValuesBatchRows*3) symbolTokenWeights := make(map[string]float64, 64) symbolFTSArgs := make([]any, 0, min(len(parsed.Symbols), sqliteSymbolFTSValuesBatchRows)*6) @@ -1362,9 +1363,10 @@ func insertParsedFileGraph( } if len(parsed.Edges) > 0 { + srcChooser := newSrcSymbolChooser(stableToID, parsed.Symbols) edgeArgs := make([]any, 0, min(len(parsed.Edges), sqliteEdgeValuesBatchRows)*7) for _, edge := range parsed.Edges { - srcID := chooseSrcSymbolID(stableToID, parsed.Symbols, edge.Line) + srcID := srcChooser.Choose(edge.Line) if srcID == 0 { continue } @@ -1418,7 +1420,7 @@ func insertParsedFileGraph( } } - targetKeySet := map[string]struct{}{} + targetKeySet := make(map[string]struct{}, len(parsed.TestLinks)) for _, link := range parsed.TestLinks { if link.TargetStableKey != "" { targetKeySet[link.TargetStableKey] = struct{}{} @@ -1548,21 +1550,12 @@ func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn str if len(args) == 0 { return nil } - // Each row uses 3 params; batch sizes are controlled by sqliteTokenValuesBatchRows to stay under sqliteDefaultMaxVariables. if len(args)%3 != 0 { return fmt.Errorf("invalid token insert args len=%d", len(args)) } rows := len(args) / 3 - var b strings.Builder - fmt.Fprintf(&b, "INSERT INTO %s(%s, token, weight) VALUES ", table, idColumn) - for i := 0; i < rows; i++ { - if i > 0 { - b.WriteString(",") - } - b.WriteString("(?,?,?)") - } - _, err := tx.ExecContext(ctx, b.String(), args...) - if err != nil { + // Each row uses 3 params; batch sizes are controlled by sqliteTokenValuesBatchRows to stay under sqliteDefaultMaxVariables. + if err := execBatchInsert(ctx, tx, table, idColumn+", token, weight", 3, args, stats); err != nil { return err } if stats != nil { @@ -1579,6 +1572,15 @@ func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn str return nil } +type insertSQLKey struct { + table string + columns string + width int + rows int +} + +var insertSQLCache sync.Map // map[insertSQLKey]string + func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, rowsPerBatch int, args []any, stats *WriteStats) error { if len(args) == 0 { return nil @@ -1587,22 +1589,27 @@ func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, row return fmt.Errorf("invalid %s insert args len=%d (expected multiple of %d)", table, len(args), rowsPerBatch) } rowCount := len(args) / rowsPerBatch - var b strings.Builder - fmt.Fprintf(&b, "INSERT INTO %s(%s) VALUES ", table, columns) - for i := 0; i < rowCount; i++ { - if i > 0 { - b.WriteString(",") - } - b.WriteString("(") - for j := 0; j < rowsPerBatch; j++ { - if j > 0 { + key := insertSQLKey{table: table, columns: columns, width: rowsPerBatch, rows: rowCount} + queryAny, ok := insertSQLCache.Load(key) + if !ok { + var b strings.Builder + fmt.Fprintf(&b, "INSERT INTO %s(%s) VALUES ", table, columns) + for i := 0; i < rowCount; i++ { + if i > 0 { b.WriteString(",") } - b.WriteString("?") + b.WriteString("(") + for j := 0; j < rowsPerBatch; j++ { + if j > 0 { + b.WriteString(",") + } + b.WriteString("?") + } + b.WriteString(")") } - b.WriteString(")") + queryAny, _ = insertSQLCache.LoadOrStore(key, b.String()) } - _, err := tx.ExecContext(ctx, b.String(), args...) + _, err := tx.ExecContext(ctx, queryAny.(string), args...) if err != nil { return err } @@ -1620,6 +1627,12 @@ func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, row case "file_imports": stats.ImportInsertBatches++ stats.ImportInsertRows += rowCount + case "symbol_tokens": + stats.SymbolTokenInsertBatches++ + stats.SymbolTokenInsertRows += rowCount + case "file_tokens": + stats.FileTokenInsertBatches++ + stats.FileTokenInsertRows += rowCount } stats.TotalExecStatements++ } @@ -1651,16 +1664,70 @@ func firstFunctionID(stableToID map[string]int64, symbols []graph.Symbol) int64 return 0 } -func chooseSrcSymbolID(stableToID map[string]int64, symbols []graph.Symbol, line int) int64 { +type funcSpan struct { + start int + end int + id int64 +} + +type srcSymbolChooser struct { + spans []funcSpan + fallback int64 + monotonic bool +} + +func newSrcSymbolChooser(stableToID map[string]int64, symbols []graph.Symbol) srcSymbolChooser { + out := srcSymbolChooser{ + spans: make([]funcSpan, 0, 16), + monotonic: true, + } + lastStart := -1 for _, sym := range symbols { if sym.Kind != "function" { continue } - if line >= sym.Range.StartLine && line <= sym.Range.EndLine { - return stableToID[sym.StableKey] + id := stableToID[sym.StableKey] + if id == 0 { + continue + } + if out.fallback == 0 { + out.fallback = id + } + start := sym.Range.StartLine + if start < lastStart { + out.monotonic = false + } + lastStart = start + out.spans = append(out.spans, funcSpan{start: start, end: sym.Range.EndLine, id: id}) + } + return out +} + +func (c srcSymbolChooser) Choose(line int) int64 { + if len(c.spans) == 0 { + return 0 + } + if !c.monotonic { + for _, span := range c.spans { + if line >= span.start && line <= span.end { + return span.id + } + } + return c.fallback + } + i := sort.Search(len(c.spans), func(i int) bool { return c.spans[i].start > line }) - 1 + if i < 0 { + return c.fallback + } + // Spans are monotonic by start line, but can be nested/overlapping. + // Scan backward from the last start<=line and return the most-specific match. + for j := i; j >= 0; j-- { + span := c.spans[j] + if line >= span.start && line <= span.end { + return span.id } } - return firstFunctionID(stableToID, symbols) + return c.fallback } func (s *Store) MarkMissingDeleted(ctx context.Context, repoID, scanID int64) (int, error) { From 8b3a4d7b46abdeb8bf5b2093255866af5ab9be73 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sun, 26 Apr 2026 17:25:10 +0200 Subject: [PATCH 40/59] fix(store/doctor): harden DB pragmas and migration lifecycle correctness (#56) * fix(store/doctor): harden DB pragmas and migration lifecycle correctness * store: build SQLite file DSNs via net/url; reuse conn across migrations --- internal/doctor/doctor.go | 12 ++- internal/store/db_lifecycle_test.go | 71 +++++++++++++++++ internal/store/store.go | 118 +++++++++++++++++++++++++--- 3 files changed, 188 insertions(+), 13 deletions(-) create mode 100644 internal/store/db_lifecycle_test.go diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index bcc0507..062df7f 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -165,7 +165,11 @@ func inspectDB(ctx context.Context, dbPath string) (*DBInfo, error) { if err != nil { return nil, err } - db, err := sql.Open(store.SQLiteDriverName(), dbPath) + dsn, err := store.BuildSQLiteDSN(dbPath, store.OpenOptions{}, false, true) + if err != nil { + return nil, err + } + db, err := sql.Open(store.SQLiteDriverName(), dsn) if err != nil { return nil, err } @@ -183,7 +187,11 @@ func inspectDB(ctx context.Context, dbPath string) (*DBInfo, error) { } func inspectDeepDB(ctx context.Context, dbPath string) (*DBDeepInfo, error) { - db, err := sql.Open(store.SQLiteDriverName(), dbPath) + dsn, err := store.BuildSQLiteDSN(dbPath, store.OpenOptions{}, false, true) + if err != nil { + return nil, err + } + db, err := sql.Open(store.SQLiteDriverName(), dsn) if err != nil { return nil, err } diff --git a/internal/store/db_lifecycle_test.go b/internal/store/db_lifecycle_test.go new file mode 100644 index 0000000..23cd90b --- /dev/null +++ b/internal/store/db_lifecycle_test.go @@ -0,0 +1,71 @@ +package store + +import ( + "context" + "path/filepath" + "sync" + "testing" +) + +func TestOpen_AppliesConnectionPragmasAcrossPool(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + s.db.SetMaxOpenConns(4) + s.db.SetMaxIdleConns(4) + + conn1, err := s.db.Conn(ctx) + if err != nil { + t.Fatalf("Conn() (1) error = %v", err) + } + defer conn1.Close() + conn2, err := s.db.Conn(ctx) + if err != nil { + t.Fatalf("Conn() (2) error = %v", err) + } + defer conn2.Close() + + var fk1, fk2 int64 + if err := conn1.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&fk1); err != nil { + t.Fatalf("PRAGMA foreign_keys (1) error = %v", err) + } + if err := conn2.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&fk2); err != nil { + t.Fatalf("PRAGMA foreign_keys (2) error = %v", err) + } + if fk1 != 1 || fk2 != 1 { + t.Fatalf("expected foreign_keys=1 on both conns, got %d and %d", fk1, fk2) + } +} + +func TestOpen_ConcurrentMigrateIsSafe(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + + const workers = 8 + var wg sync.WaitGroup + wg.Add(workers) + + errs := make(chan error, workers) + for i := 0; i < workers; i++ { + go func() { + defer wg.Done() + s, err := Open(dbPath) + if err == nil { + _ = s.Close() + } + errs <- err + }() + } + wg.Wait() + close(errs) + + for err := range errs { + if err != nil { + t.Fatalf("concurrent Open() error = %v", err) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 65f189f..cd2d1f5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -11,6 +11,7 @@ import ( "fmt" "io/fs" "math" + "net/url" "os" "path/filepath" "slices" @@ -259,7 +260,11 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { } else if st.Size() == 0 { isNewDB = true } - db, err := sql.Open(sqliteDriverName, path) + dsn, err := BuildSQLiteDSN(path, opts, isNewDB, false) + if err != nil { + return nil, err + } + db, err := sql.Open(sqliteDriverName, dsn) if err != nil { return nil, err } @@ -273,6 +278,62 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { return s, nil } +func BuildSQLiteDSN(path string, opts OpenOptions, isNewDB bool, readOnly bool) (string, error) { + pragmas := buildPragmas(isNewDB, opts.PerformanceProfile) + values := url.Values{} + if readOnly { + values.Set("mode", "ro") + } + for _, pragma := range pragmas { + values.Add("_pragma", pragma) + } + p := filepath.ToSlash(path) + u := url.URL{Scheme: "file"} + if strings.HasPrefix(p, "//") { + rest := strings.TrimPrefix(p, "//") + host, tail, _ := strings.Cut(rest, "/") + u.Host = host + u.Path = "/" + tail + } else { + if len(p) >= 2 && p[1] == ':' { + p = "/" + p + } + u.Path = p + } + u.RawQuery = values.Encode() + return u.String(), nil +} + +func buildPragmas(isNewDB bool, profile string) []string { + base := []string{ + `journal_mode(WAL)`, + `busy_timeout(5000)`, + `foreign_keys(ON)`, + `cache_size(-65536)`, + } + if isNewDB { + // auto_vacuum is only reliably applied for a brand-new DB before any tables are created. + // For existing DBs, switching auto_vacuum requires VACUUM; we avoid doing that implicitly. + base = append(base, `auto_vacuum(INCREMENTAL)`) + + // page_size is only reliably applied for a brand-new DB before any tables are created. + // For existing DBs, changing page_size requires VACUUM; we avoid doing that implicitly. + base = append(base, `page_size(8192)`) + } + perf := strings.ToLower(strings.TrimSpace(profile)) + switch perf { + case "", "balanced": + base = append(base, `synchronous(NORMAL)`, `temp_store(MEMORY)`) + case "durable": + base = append(base, `synchronous(FULL)`) + case "fast": + base = append(base, `synchronous(OFF)`, `temp_store(MEMORY)`) + default: + base = append(base, `synchronous(NORMAL)`, `temp_store(MEMORY)`) + } + return base +} + func applyPragmas(db *sql.DB, isNewDB bool, profile string) error { base := []string{ `PRAGMA journal_mode = WAL;`, @@ -322,41 +383,64 @@ func (s *Store) Close() error { } func (s *Store) Migrate() error { + ctx := context.Background() entries, err := fs.ReadDir(migrationFS, "schema") if err != nil { return err } sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + conn, err := s.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() for _, entry := range entries { name := entry.Name() var version int if _, err := fmt.Sscanf(name, "%d_", &version); err != nil { continue } - exists, err := hasMigration(s.db, version) + deadline := time.Now().Add(6 * time.Second) + for { + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + if strings.Contains(err.Error(), "database is locked") || strings.Contains(err.Error(), "SQLITE_BUSY") { + if time.Now().After(deadline) { + return err + } + time.Sleep(50 * time.Millisecond) + continue + } + return err + } + break + } + + exists, err := hasMigrationConn(ctx, conn, version) if err != nil { + _, _ = conn.ExecContext(ctx, `ROLLBACK`) return err } if exists { + if _, err := conn.ExecContext(ctx, `ROLLBACK`); err != nil { + return err + } continue } sqlBytes, err := migrationFS.ReadFile(filepath.ToSlash(filepath.Join("schema", name))) if err != nil { + _, _ = conn.ExecContext(ctx, `ROLLBACK`) return err } - tx, err := s.db.Begin() - if err != nil { + if _, err := conn.ExecContext(ctx, string(sqlBytes)); err != nil { + _, _ = conn.ExecContext(ctx, `ROLLBACK`) return err } - if _, err := tx.Exec(string(sqlBytes)); err != nil { - _ = tx.Rollback() + if _, err := conn.ExecContext(ctx, `INSERT INTO schema_migrations(version, applied_at) VALUES(?, ?)`, version, time.Now().UTC().Format(time.RFC3339)); err != nil { + _, _ = conn.ExecContext(ctx, `ROLLBACK`) return err } - if _, err := tx.Exec(`INSERT INTO schema_migrations(version, applied_at) VALUES(?, ?)`, version, time.Now().UTC().Format(time.RFC3339)); err != nil { - _ = tx.Rollback() - return err - } - if err := tx.Commit(); err != nil { + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + _, _ = conn.ExecContext(ctx, `ROLLBACK`) return err } } @@ -375,6 +459,18 @@ func hasMigration(db *sql.DB, version int) (bool, error) { return false, err } +func hasMigrationConn(ctx context.Context, conn *sql.Conn, version int) (bool, error) { + var exists int + err := conn.QueryRowContext(ctx, `SELECT COUNT(1) FROM schema_migrations WHERE version = ?`, version).Scan(&exists) + if err == nil { + return exists > 0, nil + } + if strings.Contains(err.Error(), "no such table") { + return false, nil + } + return false, err +} + func CanonicalRepoPath(path string) (string, error) { abs, err := filepath.Abs(path) if err != nil { From 61db68cbfb2a870c11a5e31d590df4600fb484b4 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sun, 26 Apr 2026 17:57:59 +0200 Subject: [PATCH 41/59] bench(store): add chooser + callers/callees benchmarks; retry applyPragmas on SQLITE_BUSY (#57) * bench(store): add chooser + callers/callees benchmarks; retry applyPragmas on SQLITE_BUSY * store(sqlite): unify busy-retry policy; make SQLITE_BUSY detection driver-aware --- internal/store/find_callers_bench_test.go | 131 ++++++++++++++++++ internal/store/sqlite_busy_cgo.go | 22 +++ internal/store/sqlite_busy_modernc.go | 25 ++++ .../store/src_symbol_chooser_bench_test.go | 57 ++++++++ internal/store/store.go | 36 +++-- 5 files changed, 257 insertions(+), 14 deletions(-) create mode 100644 internal/store/find_callers_bench_test.go create mode 100644 internal/store/sqlite_busy_cgo.go create mode 100644 internal/store/sqlite_busy_modernc.go create mode 100644 internal/store/src_symbol_chooser_bench_test.go diff --git a/internal/store/find_callers_bench_test.go b/internal/store/find_callers_bench_test.go new file mode 100644 index 0000000..b7c2f47 --- /dev/null +++ b/internal/store/find_callers_bench_test.go @@ -0,0 +1,131 @@ +package store + +import ( + "context" + "fmt" + "path/filepath" + "testing" +) + +func BenchmarkStoreFindCallers_Hub(b *testing.B) { + ctx := context.Background() + dbPath := filepath.Join(b.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + b.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, b.TempDir()) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + fileID, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + b.Fatalf("insertTestFile() error = %v", err) + } + + const n = 2000 + ids := make([]int64, 0, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("Fn_%d", i) + qualified := "pkg." + name + id, err := insertTestSymbol(ctx, s, repo.ID, fileID, name, qualified) + if err != nil { + b.Fatalf("insertTestSymbol() error = %v", err) + } + ids = append(ids, id) + } + hubID := ids[0] + hubQName := "pkg.Fn_0" + + // Many callers -> hub. + for i := 1; i < n; i++ { + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, evidence, file_id, line) + VALUES(?, ?, ?, ?, 'call', '', ?, 1) + `, repo.ID, ids[i], hubID, hubQName, fileID); err != nil { + b.Fatalf("insert edge callers->hub error = %v", err) + } + } + + // Hub -> many callees (so we can also bench FindCallees with same fixture). + for i := 1; i < n; i++ { + dstQName := fmt.Sprintf("pkg.Fn_%d", i) + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, evidence, file_id, line) + VALUES(?, ?, ?, ?, 'call', '', ?, 1) + `, repo.ID, hubID, ids[i], dstQName, fileID); err != nil { + b.Fatalf("insert edge hub->callee error = %v", err) + } + } + + b.Logf("symbols=%d edges=%d", n, 2*(n-1)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + callers, err := s.FindCallers(ctx, repo.ID, hubQName, 0, 20, 0) + if err != nil { + b.Fatalf("FindCallers() error = %v", err) + } + if len(callers) == 0 { + b.Fatalf("expected non-empty callers") + } + } +} + +func BenchmarkStoreFindCallees_Hub(b *testing.B) { + ctx := context.Background() + dbPath := filepath.Join(b.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + b.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, b.TempDir()) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + fileID, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + b.Fatalf("insertTestFile() error = %v", err) + } + + const n = 2000 + ids := make([]int64, 0, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("Fn_%d", i) + qualified := "pkg." + name + id, err := insertTestSymbol(ctx, s, repo.ID, fileID, name, qualified) + if err != nil { + b.Fatalf("insertTestSymbol() error = %v", err) + } + ids = append(ids, id) + } + hubID := ids[0] + hubQName := "pkg.Fn_0" + + for i := 1; i < n; i++ { + dstQName := fmt.Sprintf("pkg.Fn_%d", i) + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, evidence, file_id, line) + VALUES(?, ?, ?, ?, 'call', '', ?, 1) + `, repo.ID, hubID, ids[i], dstQName, fileID); err != nil { + b.Fatalf("insert edge hub->callee error = %v", err) + } + } + + b.Logf("symbols=%d edges=%d", n, n-1) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + callees, err := s.FindCallees(ctx, repo.ID, hubQName, 0, 20, 0) + if err != nil { + b.Fatalf("FindCallees() error = %v", err) + } + if len(callees) == 0 { + b.Fatalf("expected non-empty callees") + } + } +} diff --git a/internal/store/sqlite_busy_cgo.go b/internal/store/sqlite_busy_cgo.go new file mode 100644 index 0000000..7f1a425 --- /dev/null +++ b/internal/store/sqlite_busy_cgo.go @@ -0,0 +1,22 @@ +//go:build sqlite_cgo && cgo + +package store + +import ( + "errors" + "strings" + + "github.com/mattn/go-sqlite3" +) + +func isSQLiteBusy(err error) bool { + if err == nil { + return false + } + var se sqlite3.Error + if errors.As(err, &se) { + return se.Code == sqlite3.ErrBusy || se.Code == sqlite3.ErrLocked + } + s := err.Error() + return strings.Contains(s, "database is locked") || strings.Contains(s, "SQLITE_BUSY") || strings.Contains(s, "SQLITE_LOCKED") +} diff --git a/internal/store/sqlite_busy_modernc.go b/internal/store/sqlite_busy_modernc.go new file mode 100644 index 0000000..6841df6 --- /dev/null +++ b/internal/store/sqlite_busy_modernc.go @@ -0,0 +1,25 @@ +//go:build !sqlite_cgo + +package store + +import ( + "errors" + "strings" + + sqlite "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +func isSQLiteBusy(err error) bool { + if err == nil { + return false + } + var se *sqlite.Error + if errors.As(err, &se) { + code := se.Code() + const sqliteLockedSharedcache = sqlite3.SQLITE_LOCKED | (1 << 8) + return code == sqlite3.SQLITE_BUSY || code == sqlite3.SQLITE_LOCKED || code == sqliteLockedSharedcache + } + s := err.Error() + return strings.Contains(s, "database is locked") || strings.Contains(s, "SQLITE_BUSY") || strings.Contains(s, "SQLITE_LOCKED") +} diff --git a/internal/store/src_symbol_chooser_bench_test.go b/internal/store/src_symbol_chooser_bench_test.go new file mode 100644 index 0000000..5fff0da --- /dev/null +++ b/internal/store/src_symbol_chooser_bench_test.go @@ -0,0 +1,57 @@ +package store + +import ( + "testing" + + "github.com/isink17/codegraph/internal/graph" +) + +var benchSinkInt64 int64 + +func BenchmarkSrcSymbolChooserChoose_NestedMonotonic(b *testing.B) { + stableToID := map[string]int64{ + "outer": 1, + "inner": 2, + "after": 3, + } + symbols := []graph.Symbol{ + {Kind: "function", StableKey: "outer", Range: graph.Position{StartLine: 10, EndLine: 100}}, + {Kind: "function", StableKey: "inner", Range: graph.Position{StartLine: 30, EndLine: 40}}, + {Kind: "function", StableKey: "after", Range: graph.Position{StartLine: 120, EndLine: 130}}, + } + c := newSrcSymbolChooser(stableToID, symbols) + + // Correctness sanity (nested: outside inner but inside outer). + if got := c.Choose(50); got != 1 { + b.Fatalf("Choose(50)=%d, want 1 (outer)", got) + } + + lines := []int{11, 35, 50, 99, 121, 200} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchSinkInt64 = c.Choose(lines[i%len(lines)]) + } +} + +func BenchmarkSrcSymbolChooserChoose_NonMonotonic(b *testing.B) { + stableToID := map[string]int64{ + "a": 1, + "b": 2, + "c": 3, + } + // Start lines are not monotonic => linear scan path. + symbols := []graph.Symbol{ + {Kind: "function", StableKey: "b", Range: graph.Position{StartLine: 20, EndLine: 30}}, + {Kind: "function", StableKey: "a", Range: graph.Position{StartLine: 10, EndLine: 100}}, + {Kind: "function", StableKey: "c", Range: graph.Position{StartLine: 120, EndLine: 130}}, + } + c := newSrcSymbolChooser(stableToID, symbols) + + lines := []int{11, 25, 50, 121, 200} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchSinkInt64 = c.Choose(lines[i%len(lines)]) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index cd2d1f5..9fc75bd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -268,7 +268,9 @@ func OpenWithOptions(path string, opts OpenOptions) (*Store, error) { if err != nil { return nil, err } - if err := applyPragmas(db, isNewDB, opts.PerformanceProfile); err != nil { + if err := withSQLiteBusyRetry(6*time.Second, 50*time.Millisecond, func() error { + return applyPragmas(db, isNewDB, opts.PerformanceProfile) + }); err != nil { return nil, err } s := &Store{db: db} @@ -304,6 +306,20 @@ func BuildSQLiteDSN(path string, opts OpenOptions, isNewDB bool, readOnly bool) return u.String(), nil } +func withSQLiteBusyRetry(timeout, interval time.Duration, fn func() error) error { + deadline := time.Now().Add(timeout) + for { + if err := fn(); err != nil { + if isSQLiteBusy(err) && time.Now().Before(deadline) { + time.Sleep(interval) + continue + } + return err + } + return nil + } +} + func buildPragmas(isNewDB bool, profile string) []string { base := []string{ `journal_mode(WAL)`, @@ -400,19 +416,11 @@ func (s *Store) Migrate() error { if _, err := fmt.Sscanf(name, "%d_", &version); err != nil { continue } - deadline := time.Now().Add(6 * time.Second) - for { - if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { - if strings.Contains(err.Error(), "database is locked") || strings.Contains(err.Error(), "SQLITE_BUSY") { - if time.Now().After(deadline) { - return err - } - time.Sleep(50 * time.Millisecond) - continue - } - return err - } - break + if err := withSQLiteBusyRetry(6*time.Second, 50*time.Millisecond, func() error { + _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`) + return err + }); err != nil { + return err } exists, err := hasMigrationConn(ctx, conn, version) From d617c1dbec5784293096fb63a7381b8feca515e3 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Sun, 26 Apr 2026 18:23:54 +0200 Subject: [PATCH 42/59] store: cache symbol insert SQL + IN placeholders to reduce write-path allocs (#58) * store: cache symbol insert SQL + IN placeholders to reduce write-path allocs * store: use LoadOrStore in SQL caches and rename symbolInsertSQL arg --- internal/store/store.go | 65 +++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 9fc75bd..8c4e3ba 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1166,9 +1166,7 @@ func deleteFileGraphsBatch(ctx context.Context, tx *sql.Tx, fileIDs []int64, sta end = len(ids) } chunk := ids[start:end] - placeholders := strings.Repeat("?,", len(chunk)) - placeholders = strings.TrimSuffix(placeholders, ",") - query := sqlPrefix + placeholders + sqlSuffix + query := sqlPrefix + sqlitePlaceholders(len(chunk)) + sqlSuffix args := make([]any, len(chunk)) for i, id := range chunk { args[i] = id @@ -1572,14 +1570,8 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID } args := make([]any, len(symbols)*15) - var b strings.Builder - b.WriteString("INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) VALUES ") argIdx := 0 - for i, sym := range symbols { - if i > 0 { - b.WriteString(",") - } - b.WriteString("(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + for _, sym := range symbols { args[argIdx+0] = repoID args[argIdx+1] = fileID args[argIdx+2] = sym.Language @@ -1597,9 +1589,8 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID args[argIdx+14] = sym.StableKey argIdx += 15 } - b.WriteString(" RETURNING id, stable_key") - rows, err := tx.QueryContext(ctx, b.String(), args...) + rows, err := tx.QueryContext(ctx, symbolInsertSQL(len(symbols)), args...) if err != nil { return nil, err } @@ -1631,6 +1622,34 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID return out, nil } +var symbolInsertSQLCache sync.Map // map[int]string + +func symbolInsertSQL(n int) string { + if n <= 0 { + return "" + } + if v, ok := symbolInsertSQLCache.Load(n); ok { + return v.(string) + } + const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) VALUES " + const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + const suffix = " RETURNING id, stable_key" + + var b strings.Builder + b.Grow(len(prefix) + n*(len(row)+1) + len(suffix)) + b.WriteString(prefix) + for i := 0; i < n; i++ { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(row) + } + b.WriteString(suffix) + s := b.String() + actual, _ := symbolInsertSQLCache.LoadOrStore(n, s) + return actual.(string) +} + func execSymbolFTSInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { if len(args) == 0 { return nil @@ -1685,6 +1704,28 @@ type insertSQLKey struct { var insertSQLCache sync.Map // map[insertSQLKey]string +var sqlitePlaceholdersCache sync.Map // map[int]string + +func sqlitePlaceholders(n int) string { + if n <= 0 { + return "" + } + if v, ok := sqlitePlaceholdersCache.Load(n); ok { + return v.(string) + } + var b strings.Builder + b.Grow(n*2 - 1) + for i := 0; i < n; i++ { + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('?') + } + s := b.String() + actual, _ := sqlitePlaceholdersCache.LoadOrStore(n, s) + return actual.(string) +} + func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, rowsPerBatch int, args []any, stats *WriteStats) error { if len(args) == 0 { return nil From e2047d16827c5f7727710e2ee65a042ff49f5787 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 08:59:29 +0200 Subject: [PATCH 43/59] add per-strategy resolver microbenches (slash-suffix, dot-tail2, dot-suffix) (#59) * add per-strategy resolver microbenches (slash-suffix, dot-tail2, dot-suffix) * simplify upsertBenchRepo return type to int64 --- .../resolve_edges_strategies_bench_test.go | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 internal/store/resolve_edges_strategies_bench_test.go diff --git a/internal/store/resolve_edges_strategies_bench_test.go b/internal/store/resolve_edges_strategies_bench_test.go new file mode 100644 index 0000000..89203f0 --- /dev/null +++ b/internal/store/resolve_edges_strategies_bench_test.go @@ -0,0 +1,260 @@ +package store + +import ( + "context" + "fmt" + "path/filepath" + "testing" +) + +// BenchmarkResolveEdgesBySlashSuffix_SlashOnly drives only the slash-suffix +// branch inside resolveEdgesBySlashSuffix: dst_name has zero dots so the +// dot-tail2 needed-set stays empty, and symbol qualified_names contain a slash +// so the slash-suffix map is populated. +func BenchmarkResolveEdgesBySlashSuffix_SlashOnly(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 100 + numNames = 1000 + numNoiseSyms = 1000 + numEdges = 5000 + dstQNamePrefix = "github.com/org/repo/pkg/" + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + // Slash-qualified target symbols: qualified_name has a '/', name part has no dot. + for i := 0; i < numNames; i++ { + name := fmt.Sprintf("Func_%d", i) + qualified := fmt.Sprintf("%spkg_%d/%s", dstQNamePrefix, i%50, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(slash dst) error = %v", err) + } + } + // Noise symbols (no slash, irrelevant to this strategy). + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("noise.%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + // Unresolved edges: dst_name = simple "Func_" with no dot, no slash. + for i := 0; i < numEdges; i++ { + dstName := fmt.Sprintf("Func_%d", i%numNames) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesBySlashSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + +// BenchmarkResolveEdgesBySlashSuffix_DotTail2 drives only the dot-tail2 branch +// inside resolveEdgesBySlashSuffix: dst_name has exactly one dot (so it lands +// in neededTail2), and symbol qualified_names have no slash but enough dot +// segments that only the tail2 logic resolves them. +func BenchmarkResolveEdgesBySlashSuffix_DotTail2(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 100 + numNames = 1000 + numNoiseSyms = 1000 + numEdges = 5000 + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + // Dot-tail2 target symbols: no slash, qualified_name has a leading segment + // before the matched 2-segment tail (e.g., "io.pkg_3.Func_42"). + for i := 0; i < numNames; i++ { + name := fmt.Sprintf("Func_%d", i) + qualified := fmt.Sprintf("io.pkg_%d.%s", i%50, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(dot-tail2 dst) error = %v", err) + } + } + // Noise symbols with slashes (excluded from dot-tail2 by the slash-branch + // `continue` because afterSlash isn't in neededSuffix). + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("github.com/org/repo/noise/%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + // Unresolved edges: dst_name = "pkg_X.Func_Y" (exactly one dot, no slash). + for i := 0; i < numEdges; i++ { + dstName := fmt.Sprintf("pkg_%d.Func_%d", i%50, i%numNames) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesBySlashSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + +// BenchmarkResolveEdgesByDotSuffix drives resolveEdgesByDotSuffix: dst_name has +// at least two dots and no slash, so the multi-dot pre-filter accepts it and +// the LIKE '%.' || dst_name suffix scan is exercised. +func BenchmarkResolveEdgesByDotSuffix(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 100 + numNames = 1000 + numNoiseSyms = 1000 + numEdges = 5000 + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + // Multi-dot target symbols: qualified_name = "x." so the LIKE + // '%.' suffix match resolves them. + for i := 0; i < numNames; i++ { + dstSuffix := fmt.Sprintf("a_%d.b_%d.c_%d", i%50, i%25, i) + qualified := "x." + dstSuffix + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, fmt.Sprintf("c_%d", i), qualified); err != nil { + b.Fatalf("insertTestSymbol(dot-suffix dst) error = %v", err) + } + } + // Noise symbols with single-dot or slash names (skipped by the strategy filters). + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("noise.%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + // Unresolved edges: dst_name has exactly two dots and no slash so it passes + // the strategy's multi-dot pre-filter. + for i := 0; i < numEdges; i++ { + nameIdx := i % numNames + dstName := fmt.Sprintf("a_%d.b_%d.c_%d", nameIdx%50, nameIdx%25, nameIdx) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesByDotSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesByDotSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + +// openBenchStore opens a fresh sqlite store under b.TempDir(). +func openBenchStore(b *testing.B) *Store { + b.Helper() + dbPath := filepath.Join(b.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + b.Fatalf("Open() error = %v", err) + } + return s +} + +func upsertBenchRepo(ctx context.Context, b *testing.B, s *Store) int64 { + b.Helper() + repo, err := s.UpsertRepo(ctx, b.TempDir()) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + return repo.ID +} + +func makeBenchFiles(ctx context.Context, b *testing.B, s *Store, repoID int64, n int) []int64 { + b.Helper() + out := make([]int64, 0, n) + for i := 0; i < n; i++ { + id, err := insertTestFile(ctx, s, repoID, fmt.Sprintf("f_%d.go", i)) + if err != nil { + b.Fatalf("insertTestFile() error = %v", err) + } + out = append(out, id) + } + return out +} + +func makeBenchSrcSymbols(ctx context.Context, b *testing.B, s *Store, repoID int64, fileIDs []int64) []int64 { + b.Helper() + out := make([]int64, 0, len(fileIDs)) + for i, fileID := range fileIDs { + id, err := insertTestSymbol(ctx, s, repoID, fileID, fmt.Sprintf("Src_%d", i), fmt.Sprintf("Src_%d", i)) + if err != nil { + b.Fatalf("insertTestSymbol(src) error = %v", err) + } + out = append(out, id) + } + return out +} From 69a580f8144326fe5dd94fa39e62a072e735d130 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 09:31:12 +0200 Subject: [PATCH 44/59] trim residual write-path allocations + add multi-file batch bench + fix duplicate token stats (#60) --- internal/store/store.go | 74 +++++++++++++++++++----------- internal/store/store_bench_test.go | 58 +++++++++++++++++++++++ 2 files changed, 105 insertions(+), 27 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 8c4e3ba..2615e5c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1669,6 +1669,14 @@ func execSymbolFTSInsert(ctx context.Context, tx *sql.Tx, args []any, stats *Wri return nil } +// Pre-baked column lists for the only two (table, idColumn) call sites +// (`symbol_tokens`/`symbol_id` and `file_tokens`/`file_id`). Avoids +// reallocating the column-list string per token-insert call. +const ( + symbolTokensInsertColumns = "symbol_id, token, weight" + fileTokensInsertColumns = "file_id, token, weight" +) + func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn string, args []any, stats *WriteStats) error { if len(args) == 0 { return nil @@ -1676,23 +1684,20 @@ func execTokenTriplesInsert(ctx context.Context, tx *sql.Tx, table, idColumn str if len(args)%3 != 0 { return fmt.Errorf("invalid token insert args len=%d", len(args)) } - rows := len(args) / 3 - // Each row uses 3 params; batch sizes are controlled by sqliteTokenValuesBatchRows to stay under sqliteDefaultMaxVariables. - if err := execBatchInsert(ctx, tx, table, idColumn+", token, weight", 3, args, stats); err != nil { - return err - } - if stats != nil { - switch table { - case "symbol_tokens": - stats.SymbolTokenInsertBatches++ - stats.SymbolTokenInsertRows += rows - case "file_tokens": - stats.FileTokenInsertBatches++ - stats.FileTokenInsertRows += rows - } - stats.TotalExecStatements++ + var columns string + switch idColumn { + case "symbol_id": + columns = symbolTokensInsertColumns + case "file_id": + columns = fileTokensInsertColumns + default: + columns = idColumn + ", token, weight" } - return nil + // Each row uses 3 params; batch sizes are controlled by sqliteTokenValuesBatchRows + // to stay under sqliteDefaultMaxVariables. Stats counters (Symbol/FileToken*, + // TotalExecStatements) are bumped exclusively in execBatchInsert to avoid + // double-counting from this wrapper. + return execBatchInsert(ctx, tx, table, columns, 3, args, stats) } type insertSQLKey struct { @@ -1737,20 +1742,25 @@ func execBatchInsert(ctx context.Context, tx *sql.Tx, table, columns string, row key := insertSQLKey{table: table, columns: columns, width: rowsPerBatch, rows: rowCount} queryAny, ok := insertSQLCache.Load(key) if !ok { + // Build the row tuple once via the cached placeholder string, then + // concatenate it rowCount times. Avoids the inner per-row loops and + // the fmt.Fprintf allocation in the previous implementation. + tuple := "(" + sqlitePlaceholders(rowsPerBatch) + ")" + const prefix = "INSERT INTO " + const sep = "(" + const valuesKw = ") VALUES " var b strings.Builder - fmt.Fprintf(&b, "INSERT INTO %s(%s) VALUES ", table, columns) + b.Grow(len(prefix) + len(table) + len(sep) + len(columns) + len(valuesKw) + rowCount*(len(tuple)+1)) + b.WriteString(prefix) + b.WriteString(table) + b.WriteString(sep) + b.WriteString(columns) + b.WriteString(valuesKw) for i := 0; i < rowCount; i++ { if i > 0 { - b.WriteString(",") + b.WriteByte(',') } - b.WriteString("(") - for j := 0; j < rowsPerBatch; j++ { - if j > 0 { - b.WriteString(",") - } - b.WriteString("?") - } - b.WriteString(")") + b.WriteString(tuple) } queryAny, _ = insertSQLCache.LoadOrStore(key, b.String()) } @@ -1822,8 +1832,18 @@ type srcSymbolChooser struct { } func newSrcSymbolChooser(stableToID map[string]int64, symbols []graph.Symbol) srcSymbolChooser { + // Pre-size to the upper bound of function symbols (capped to keep the + // allocation small for files with many non-function symbols). Avoids + // the 16->32->64 doubling sequence for files with >16 functions. + capHint := len(symbols) + if capHint > 64 { + capHint = 64 + } + if capHint < 8 { + capHint = 8 + } out := srcSymbolChooser{ - spans: make([]funcSpan, 0, 16), + spans: make([]funcSpan, 0, capHint), monotonic: true, } lastStart := -1 diff --git a/internal/store/store_bench_test.go b/internal/store/store_bench_test.go index 0d52c5e..3da6504 100644 --- a/internal/store/store_bench_test.go +++ b/internal/store/store_bench_test.go @@ -86,6 +86,64 @@ func BenchmarkStoreReplaceFileGraphWriteHeavy(b *testing.B) { } } +// BenchmarkStoreReplaceFileGraphsBatchWriteHeavy stresses the multi-file batch +// path (`ReplaceFileGraphsBatchWithStats`) so per-batch hot-loop allocations +// are visible in isolation from single-file overhead. +func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-batch-write-heavy.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + const ( + filesPerBatch = 16 + symbolsPerFile = 40 + refsPerSymbol = 2 + edgesPerSymbol = 2 + ) + + parsed := heavyParsedFile(symbolsPerFile, refsPerSymbol, edgesPerSymbol) + inputs := make([]store.ReplaceFileGraphInput, 0, filesPerBatch) + for j := 0; j < filesPerBatch; j++ { + inputs = append(inputs, store.ReplaceFileGraphInput{ + Path: fmt.Sprintf("bench/batch_%d.go", j), + Language: "go", + SizeBytes: 1024, + MtimeUnixNS: 0, + ContentHash: "", + Parsed: parsed, + }) + } + + stats := &store.WriteStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Vary mtime/hash per iteration so the upsert always writes. + hash := strconv.Itoa(i) + for j := range inputs { + inputs[j].MtimeUnixNS = int64(i) + inputs[j].ContentHash = hash + } + if _, err := s.ReplaceFileGraphsBatchWithStats(ctx, repo.ID, 1, inputs, stats); err != nil { + b.Fatalf("ReplaceFileGraphsBatchWithStats() error = %v", err) + } + } +} + func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64) { b.Helper() repoRoot := b.TempDir() From 4b92f8e9cf93eabb779be4ca07688fcee60d40f6 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:02:54 +0200 Subject: [PATCH 45/59] perf(store/indexer): trim repo-wide change-detection floor cost (#61) --- internal/indexer/indexer_bench_test.go | 42 ++++++++++++++++++++++++++ internal/store/store.go | 23 +++++++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index 8449443..260a41e 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -45,6 +45,48 @@ func BenchmarkIndexerIndex(b *testing.B) { } } +// BenchmarkIndexerNoOpUpdateRepoWide measures the floor cost of a repo-wide +// update where 0 files have changed: existing-files load + filesystem walk + +// per-file (mtime,size) compare + mark-seen flushes, with no parses or graph +// rewrites. Surfaces ExistingLoadMS / WalkMS regressions on large repos. +func BenchmarkIndexerNoOpUpdateRepoWide(b *testing.B) { + ctx := context.Background() + repoRoot := b.TempDir() + files := 200 + if v := os.Getenv("CODEGRAPH_BENCH_FILES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + files = n + } + } + createGoFixtureRepo(b, repoRoot, files) + dbPath := filepath.Join(b.TempDir(), "bench-noop-update.sqlite") + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", sqliteBenchProfile()) + b.Logf("fixture_files=%d", files) + + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()}) + if err != nil { + b.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, Options{RepoRoot: repoRoot}); err != nil { + b.Fatalf("Index() error = %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + summary, err := idx.Update(ctx, Options{RepoRoot: repoRoot}) + if err != nil { + b.Fatalf("Update() error = %v", err) + } + if summary.FilesChanged != 0 { + b.Fatalf("expected 0 files changed on no-op update, got %d", summary.FilesChanged) + } + } +} + func BenchmarkIndexerUpdateOneFile(b *testing.B) { ctx := context.Background() repoRoot := b.TempDir() diff --git a/internal/store/store.go b/internal/store/store.go index 2615e5c..7aad227 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -748,11 +748,20 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { return out, nil } +// ExistingFiles returns active (non-deleted) file records for the repo, +// projecting only the columns the change-detection path actually consumes +// (`path`, `size_bytes`, `mtime_unix_ns`, `content_sha256`). `ID` and +// `Language` on the returned `FileRecord` are intentionally left zero. +// +// Filtering `is_deleted = 0` server-side avoids materializing tombstone rows +// that can never match a walked path, shrinking both the query result set +// and the resulting Go map. `IsDeleted` is therefore always false on returned +// records. func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]FileRecord, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, path, language, size_bytes, mtime_unix_ns, content_sha256, is_deleted + SELECT path, size_bytes, mtime_unix_ns, content_sha256 FROM files - WHERE repo_id = ? + WHERE repo_id = ? AND is_deleted = 0 `, repoID) if err != nil { return nil, err @@ -760,6 +769,8 @@ func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]Fil return scanExistingFiles(rows) } +// ExistingFilesForPaths is the path-scoped sibling of ExistingFiles; same +// projection and same `is_deleted = 0` filter apply. func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths []string) (map[string]FileRecord, error) { out := make(map[string]FileRecord, len(paths)) if len(paths) == 0 { @@ -771,9 +782,9 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ chunk := paths[start:end] placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") query := ` - SELECT id, path, language, size_bytes, mtime_unix_ns, content_sha256, is_deleted + SELECT path, size_bytes, mtime_unix_ns, content_sha256 FROM files - WHERE repo_id = ? AND path IN (` + placeholders + `) + WHERE repo_id = ? AND is_deleted = 0 AND path IN (` + placeholders + `) ` args := make([]any, 0, len(chunk)+1) args = append(args, repoID) @@ -800,11 +811,9 @@ func scanExistingFiles(rows *sql.Rows) (map[string]FileRecord, error) { out := map[string]FileRecord{} for rows.Next() { var rec FileRecord - var isDeleted int - if err := rows.Scan(&rec.ID, &rec.Path, &rec.Language, &rec.SizeBytes, &rec.MtimeUnixNS, &rec.ContentHash, &isDeleted); err != nil { + if err := rows.Scan(&rec.Path, &rec.SizeBytes, &rec.MtimeUnixNS, &rec.ContentHash); err != nil { return nil, err } - rec.IsDeleted = isDeleted == 1 out[rec.Path] = rec } return out, rows.Err() From 848e8b967b3f9b4dcc9ae94551ff1b731d2d53fe Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:27:41 +0200 Subject: [PATCH 46/59] perf(export): page no-focus JSONPaged via ExportSymbolsPage/EdgesPage (#62) * perf(export): page no-focus JSONPaged via ExportSymbolsPage/EdgesPage * fix(export): stably order symbols and edges in JSONPaged fallback --- internal/export/export.go | 37 ++++++++++++++--- internal/export/export_test.go | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/internal/export/export.go b/internal/export/export.go index d45f243..d209c87 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -8,7 +8,9 @@ import ( "sort" "strings" + "github.com/isink17/codegraph/internal/graph" "github.com/isink17/codegraph/internal/query" + "github.com/isink17/codegraph/internal/store" ) type Service struct { @@ -36,17 +38,42 @@ func (s *Service) JSON(ctx context.Context, repoID int64) ([]byte, error) { }, "", " ") } +// JSONPaged returns a JSON-encoded subset of the graph. When the caller +// requests a bounded page over the whole repo (symbol == "" && limit > 0) +// rows are streamed straight from the paging helpers so peak memory is +// O(page), not O(repo). The unbounded (limit <= 0) and focused (symbol != "") +// paths still go through GraphSnapshot since they intentionally want the full +// snapshot or a bounded impact subgraph respectively. func (s *Service) JSONPaged(ctx context.Context, repoID int64, symbol string, depth, limit, offset int) ([]byte, error) { stats, err := s.query.Stats(ctx, repoID) if err != nil { return nil, err } - symbols, edges, err := s.query.GraphSnapshot(ctx, repoID, symbol, depth) - if err != nil { - return nil, err + var symbols []graph.Symbol + var edges []store.ExportEdge + if symbol == "" && limit > 0 { + symbols, err = s.query.ExportSymbolsPage(ctx, repoID, limit, offset) + if err != nil { + return nil, err + } + edges, err = s.query.ExportEdgesPage(ctx, repoID, limit, offset) + if err != nil { + return nil, err + } + } else { + symbols, edges, err = s.query.GraphSnapshot(ctx, repoID, symbol, depth) + if err != nil { + return nil, err + } + // GraphSnapshot returns slices in unspecified order (the no-focus + // loaders have no ORDER BY, and loadEdgesForExport's dedup map + // further randomises focused-edge order). Sort by ID ASC to match + // the optimized paging branch and make pageSlice deterministic. + sort.Slice(symbols, func(i, j int) bool { return symbols[i].ID < symbols[j].ID }) + sort.Slice(edges, func(i, j int) bool { return edges[i].ID < edges[j].ID }) + symbols = pageSlice(symbols, limit, offset) + edges = pageSlice(edges, limit, offset) } - symbols = pageSlice(symbols, limit, offset) - edges = pageSlice(edges, limit, offset) return json.MarshalIndent(map[string]any{ "repo": stats.RepoRoot, "stats": stats, diff --git a/internal/export/export_test.go b/internal/export/export_test.go index ce583ab..ea44cb3 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -2,6 +2,8 @@ package export import ( "context" + "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -57,6 +59,76 @@ func main() { helper() } } } +// TestJSONPagedNoFocusUsesPagingHelpers verifies that the bounded-page +// no-focus path (`symbol == "" && limit > 0`) returns at most `limit` +// symbols/edges and that subsequent pages cover disjoint ranges. This is the +// path that previously materialized the entire repo via GraphSnapshot before +// slicing client-side. +func TestJSONPagedNoFocusUsesPagingHelpers(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + for i := 0; i < 8; i++ { + writeFile(t, filepath.Join(repoRoot, fmt.Sprintf("file_%d.go", i)), + fmt.Sprintf("package main\nfunc helper%d() {}\nfunc main%d() { helper%d() }\n", i, i, i)) + } + + s, err := store.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + + svc := New(query.New(s, nil)) + + page0, err := svc.JSONPaged(ctx, repo.ID, "", 0, 4, 0) + if err != nil { + t.Fatalf("JSONPaged(page0) error = %v", err) + } + page1, err := svc.JSONPaged(ctx, repo.ID, "", 0, 4, 4) + if err != nil { + t.Fatalf("JSONPaged(page1) error = %v", err) + } + + parsed0 := struct { + Symbols []map[string]any `json:"symbols"` + }{} + parsed1 := struct { + Symbols []map[string]any `json:"symbols"` + }{} + if err := json.Unmarshal(page0, &parsed0); err != nil { + t.Fatalf("Unmarshal page0: %v", err) + } + if err := json.Unmarshal(page1, &parsed1); err != nil { + t.Fatalf("Unmarshal page1: %v", err) + } + if len(parsed0.Symbols) == 0 || len(parsed0.Symbols) > 4 { + t.Fatalf("page0 symbol count = %d, want 1..4", len(parsed0.Symbols)) + } + if len(parsed1.Symbols) > 4 { + t.Fatalf("page1 symbol count = %d, want <=4", len(parsed1.Symbols)) + } + // Pages must not overlap on stable_key. + keys := map[string]bool{} + for _, sym := range parsed0.Symbols { + if k, ok := sym["stable_key"].(string); ok && k != "" { + keys[k] = true + } + } + for _, sym := range parsed1.Symbols { + if k, ok := sym["stable_key"].(string); ok && k != "" && keys[k] { + t.Fatalf("page1 overlaps page0 on stable_key %q", k) + } + } +} + func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { From 009ad7104d768ab94780de530860f6596cc063e3 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:24:37 +0200 Subject: [PATCH 47/59] perf(store/resolver): early-return resolveEdgesBySlashSuffix when no unresolved edges (#64) * perf(store/resolver): early-return resolveEdgesBySlashSuffix when no unresolved edges * Update internal/store/store.go --- .../resolve_edges_strategies_bench_test.go | 56 +++++++++++++++++++ internal/store/store.go | 6 ++ 2 files changed, 62 insertions(+) diff --git a/internal/store/resolve_edges_strategies_bench_test.go b/internal/store/resolve_edges_strategies_bench_test.go index 89203f0..11c1646 100644 --- a/internal/store/resolve_edges_strategies_bench_test.go +++ b/internal/store/resolve_edges_strategies_bench_test.go @@ -213,6 +213,62 @@ func BenchmarkResolveEdgesByDotSuffix(b *testing.B) { } } +// BenchmarkResolveEdgesBySlashSuffix_NoUnresolved measures the steady-state +// floor cost when every edge is already resolved (no candidate dst_names for +// either suffix strategy). Captures the win from skipping the full symbols +// scan when the needed-set is empty. +func BenchmarkResolveEdgesBySlashSuffix_NoUnresolved(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 100 + numSyms = 2000 + numNoiseSyms = 1000 + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + _ = makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + for i := 0; i < numSyms; i++ { + name := fmt.Sprintf("Func_%d", i) + qualified := fmt.Sprintf("github.com/org/repo/pkg_%d/%s", i%50, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol() error = %v", err) + } + } + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("noise.%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + // Intentionally insert no unresolved edges: the SELECT DISTINCT dst_name + // query returns zero rows, so neededSuffix and neededTail2 stay empty. + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesBySlashSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + // openBenchStore opens a fresh sqlite store under b.TempDir(). func openBenchStore(b *testing.B) *Store { b.Helper() diff --git a/internal/store/store.go b/internal/store/store.go index 7aad227..6bc5bef 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2352,6 +2352,12 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI if err := needRows.Close(); err != nil { return 0, err } + // Skip the full symbols scan entirely when no current unresolved edge can + // be satisfied by either suffix strategy. This is the common case after + // upstream strategies have already absorbed the resolvable edges. +if len(neededSuffix) == 0 { + return 0, nil + } rows, err := tx.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) if err != nil { From 1d1717c9909452d5d3bd8e45d42707e5ad083394 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:32:15 +0200 Subject: [PATCH 48/59] =?UTF-8?q?docs(changelog):=20catch=20up=20Unrelease?= =?UTF-8?q?d=20with=20#52=E2=80=93#64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843054b..f3e3f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,18 +10,32 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **watch:** Ignore chmod-only + directory-create events; add repeated-work stats. (#36) - **indexer:** Target cross-file edge resolution on update runs (preserve path-scoped behavior; avoid repo-wide resolve). (#32, #33) - **store:** Speed up edge resolver (dotted-edge indexes, dot-tail2 strategy, resolver symbol indexes). (#48) +- **store:** Scale `ResolveEdges` dot-suffix path and cap slash-suffix map growth; restrict dot-tail2 candidate set to single-dot `dst_name`. (#52) +- **store:** De-correlate `ResolveEdges` strategies 1/2/4 by distinct `dst_name`; add `BenchmarkResolveEdgesForNames_CrossFileScale`. (#53) +- **store:** Cache batch-insert SQL and speed up edge-source selection in the write path via `srcSymbolChooser` span/binary-search. (#55) +- **store:** Cache symbol insert SQL + IN-placeholder strings to reduce write-path allocations. (#58) +- **store:** Steady-state guard skips the full symbols scan in `resolveEdgesBySlashSuffix` when no unresolved non-slash edge remains; ~31× faster / ~1200× fewer allocs on the new no-unresolved bench. (#64) +- **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) +- **store:** Trim residual write-path allocations in `ReplaceFileGraphsBatch`; add multi-file batch bench. (#60) +- **export:** Page no-focus `JSONPaged` directly via `ExportSymbolsPage` / `ExportEdgesPage` so peak memory is O(page) instead of O(repo) on the bounded-page CLI path. (#62) - **indexing/store:** Broad batching + reduced statement pressure across symbols/FTS/inserts; add/extend phase timings + write_stats counters. (#20, #21, #22, #24, #25, #26, #28, #29) - **indexing:** Reduce tokenization allocations; add tokenize timing stats. (#30) - **json/jsonl:** Stabilize `watch` and `doctor` machine-readable output (event envelopes; arrays always present; disable HTML escaping). (#38, #43) - **cli/index:** Stabilize `--jsonl` scan payloads/envelopes (scan_kind, parse_ms, correlation fields; dedupe envelopes; handle phase write errors). (#40, #41, #42) - **clean/doctor:** Add ANALYZE, WAL checkpoint, incremental vacuum; add `doctor --deep` integrity checks; expand DB diagnostics + FTS optimize. (#37, #39) - **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44) +- **store/bench:** Add chooser + callers/callees benchmarks; retry `applyPragmas` on `SQLITE_BUSY`. (#57) +- **store/bench:** Add per-strategy resolver microbenchmarks (slash-suffix, dot-tail2, dot-suffix). (#59) +- **store:** Speed up `FindDeadCode` with new `idx_refs_repo_symbol_id` / `idx_refs_repo_context_symbol_id` indexes. (#54) - **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45) - **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49) - **cli/help/commands:** Command registry + per-command help; canonical query command names with backward-compatible aliases; help/usage normalization. (#10, #12, #13, #14, #15, #16, #17, #18, #19) ### Fixed - **store/indexer:** Purge deleted-file graph rows and nullify cross-file symbol references. (#46) +- **store:** Scope `RelatedTests(file)` correctly via `target_file_id`; make symbol lookup deterministic. (#54) +- **store/doctor:** Harden DB pragmas and migration lifecycle correctness; concurrent-open coverage via `TestOpen_ConcurrentMigrateIsSafe`. (#56) +- **store:** Fix duplicate token-stat counting in batched writes. (#60) ## v1.0.9 - 2026-04-16 From 5f7a358c0756cce38d9bca9ec1cdedaa4093b0f6 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:13:29 +0200 Subject: [PATCH 49/59] perf(indexer): overlap ExistingFiles load with FS walk on repo-wide scans (#65) * perf(indexer): overlap ExistingFiles load with FS walk on repo-wide scans * Update internal/indexer/indexer.go --- internal/indexer/indexer.go | 72 +++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 468ad99..e80216b 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -128,18 +128,33 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } pathScoped := len(candidateSet) > 0 - var existing map[string]store.FileRecord + // Run the existing-files load concurrently with the filesystem walk so + // the floor cost of a no-op repo-wide update is roughly max(loadMS, walkMS) + // instead of loadMS+walkMS. Workers block on `existingReady` before + // reading the resulting map; the FS-walk producer fills the (now + // larger) `tasks` buffer in parallel. On load failure a monitor goroutine + // cancels `ctxRun` so the producer/workers drain cleanly. + var ( + existing map[string]store.FileRecord + loadErr error + existingLoadMS int64 + ) existingLoadStarted := time.Now() - if len(candidateSet) > 0 { - existing, err = i.store.ExistingFilesForPaths(ctx, repo.ID, candidatePaths) - } else { - existing, err = i.store.ExistingFiles(ctx, repo.ID) - } - summary.ExistingLoadMS = time.Since(existingLoadStarted).Milliseconds() - if err != nil { - _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) - return summary, err - } + existingReady := make(chan struct{}) + go func() { + defer close(existingReady) + var ( + m map[string]store.FileRecord + err error + ) + if pathScoped { + m, err = i.store.ExistingFilesForPaths(ctx, repo.ID, candidatePaths) + } else { + m, err = i.store.ExistingFiles(ctx, repo.ID) + } + existingLoadMS = time.Since(existingLoadStarted).Milliseconds() + existing, loadErr = m, err + }() workerCount := runtime.NumCPU() if workerCount < 2 { @@ -152,7 +167,19 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err ctxRun, cancel := context.WithCancel(ctx) defer cancel() - tasks := make(chan fileTask, workerCount*2) + go func() { + <-existingReady + if loadErr != nil { + cancel() + } + }() + + // `tasks` is sized so the FS-walk producer can keep filling while the + // existing-files load is still in flight. With cap=workerCount*2 the + // producer would stall on a full channel after ~16 dirents, defeating + // the overlap on large repos. workerCount*64 (≤512 entries × ~200B + // per fileTask ≈ ~100KB) keeps peak buffer trivial. + tasks := make(chan fileTask, workerCount*64) results := make(chan fileResult, workerCount*2) producerErr := make(chan error, 1) walkStart := time.Now() @@ -256,6 +283,14 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err wg.Add(1) go func() { defer wg.Done() + <-existingReady + if loadErr != nil { + // Drain producer-buffered tasks so the producer's + // send-side select can observe ctxRun cancellation and exit. + for range tasks { + } + return + } for task := range tasks { res := processFileTask(ctxRun, task, existing[task.rel], opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy) select { @@ -525,6 +560,10 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } walkErr := <-producerErr + // Safe to read existingLoadMS here: the workers' `<-existingReady` happens- + // before any workers exit, and wg.Wait() (closing `results`) joined them + // before this consumer-loop returned. + summary.ExistingLoadMS = existingLoadMS summary.WalkMS = time.Since(walkStart).Milliseconds() summary.ProcessWallMS = time.Since(processWallStart).Milliseconds() summary.TaskMS = taskDur.Milliseconds() @@ -540,9 +579,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if writeStats != (store.WriteStats{}) { summary.WriteStats = &writeStats } - if runErr == nil && walkErr != nil && walkErr != context.Canceled { - runErr = walkErr - } +if (runErr == nil || errors.Is(runErr, context.Canceled)) && walkErr != nil && !errors.Is(walkErr, context.Canceled) { + runErr = walkErr +} +if (runErr == nil || errors.Is(runErr, context.Canceled)) && loadErr != nil { + runErr = loadErr +} if runErr != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", runErr.Error()) return summary, runErr From b445a1f90a851d889d2f3af47dd195e92f2471de Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:00:19 +0200 Subject: [PATCH 50/59] perf(store/resolver): schema-backed slash-suffix path via qualified_suffix (#66) --- internal/indexer/indexer.go | 13 +- .../store/resolve_edges_for_names_test.go | 6 +- .../resolve_edges_strategies_bench_test.go | 68 ++++++ .../schema/016_symbol_qualified_suffix.sql | 32 +++ internal/store/store.go | 211 +++++++++++------- 5 files changed, 239 insertions(+), 91 deletions(-) create mode 100644 internal/store/schema/016_symbol_qualified_suffix.sql diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index e80216b..56d5dee 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "io/fs" @@ -579,12 +580,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err if writeStats != (store.WriteStats{}) { summary.WriteStats = &writeStats } -if (runErr == nil || errors.Is(runErr, context.Canceled)) && walkErr != nil && !errors.Is(walkErr, context.Canceled) { - runErr = walkErr -} -if (runErr == nil || errors.Is(runErr, context.Canceled)) && loadErr != nil { - runErr = loadErr -} + if (runErr == nil || errors.Is(runErr, context.Canceled)) && walkErr != nil && !errors.Is(walkErr, context.Canceled) { + runErr = walkErr + } + if (runErr == nil || errors.Is(runErr, context.Canceled)) && loadErr != nil { + runErr = loadErr + } if runErr != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", runErr.Error()) return summary, runErr diff --git a/internal/store/resolve_edges_for_names_test.go b/internal/store/resolve_edges_for_names_test.go index 46f356a..febdd45 100644 --- a/internal/store/resolve_edges_for_names_test.go +++ b/internal/store/resolve_edges_for_names_test.go @@ -130,10 +130,10 @@ func insertTestSymbol(ctx context.Context, s *Store, repoID, fileID int64, name, res, err := s.db.ExecContext(ctx, ` INSERT INTO symbols( repo_id, file_id, language, kind, name, qualified_name, - start_line, start_col, end_line, end_col, stable_key + start_line, start_col, end_line, end_col, stable_key, qualified_suffix ) - VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?) - `, repoID, fileID, "go", "function", name, qualified, qualified) + VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?, ?) + `, repoID, fileID, "go", "function", name, qualified, qualified, qualifiedSuffix(qualified)) if err != nil { return 0, err } diff --git a/internal/store/resolve_edges_strategies_bench_test.go b/internal/store/resolve_edges_strategies_bench_test.go index 11c1646..9494ea5 100644 --- a/internal/store/resolve_edges_strategies_bench_test.go +++ b/internal/store/resolve_edges_strategies_bench_test.go @@ -213,6 +213,74 @@ func BenchmarkResolveEdgesByDotSuffix(b *testing.B) { } } +// BenchmarkResolveEdgesBySlashSuffix_SlashOnlyLargeScale stresses the slash +// branch with 20k slash-qualified targets + 20k noise symbols. The Go-scan + +// hash-filter path is O(symbols), so it grows ~linearly with this fixture +// size; the schema-backed indexed JOIN against `qualified_suffix` +// (migration 016) grows ~O(unique_needed_names) and should pull ahead at +// scale. Companion bench to `_SlashOnly` (2k symbols), which is too small for +// the scan cost to dominate. +func BenchmarkResolveEdgesBySlashSuffix_SlashOnlyLargeScale(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 200 + numNames = 20000 + numNoiseSyms = 20000 + numEdges = 5000 + dstQNamePrefix = "github.com/org/repo/pkg/" + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + for i := 0; i < numNames; i++ { + name := fmt.Sprintf("Func_%d", i) + qualified := fmt.Sprintf("%spkg_%d/%s", dstQNamePrefix, i%50, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(slash dst) error = %v", err) + } + } + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("noise.%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + for i := 0; i < numEdges; i++ { + dstName := fmt.Sprintf("Func_%d", i%numNames) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesBySlashSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + // BenchmarkResolveEdgesBySlashSuffix_NoUnresolved measures the steady-state // floor cost when every edge is already resolved (no candidate dst_names for // either suffix strategy). Captures the win from skipping the full symbols diff --git a/internal/store/schema/016_symbol_qualified_suffix.sql b/internal/store/schema/016_symbol_qualified_suffix.sql new file mode 100644 index 0000000..b3a1a2c --- /dev/null +++ b/internal/store/schema/016_symbol_qualified_suffix.sql @@ -0,0 +1,32 @@ +-- Strategy 3a: indexed slash-suffix resolution. +-- +-- `qualified_suffix` stores the substring of `qualified_name` after the last +-- '/' for slash-containing names, '' otherwise. This lets `resolveEdgesBySlashSuffix` +-- replace its full repo-wide symbols scan with an indexed equality JOIN +-- (tmp_needed.dst_name = symbols.qualified_suffix) under (repo_id, qualified_suffix). +-- +-- Backfill uses a recursive CTE because SQLite has no `reverse()` / +-- last-occurrence builtin: each step strips one '/'-separated prefix until +-- none remain, and the terminal row (instr=0) is the after-last-slash value. +-- For non-slash qnames the column stays at its DEFAULT '' (and is excluded +-- from the partial index). + +ALTER TABLE symbols ADD COLUMN qualified_suffix TEXT NOT NULL DEFAULT ''; + +WITH RECURSIVE strip(id, rest) AS ( + SELECT id, qualified_name FROM symbols WHERE instr(qualified_name, '/') > 0 + UNION ALL + SELECT id, substr(rest, instr(rest, '/') + 1) + FROM strip + WHERE instr(rest, '/') > 0 +) +UPDATE symbols +SET qualified_suffix = ( + SELECT s2.rest FROM strip s2 + WHERE s2.id = symbols.id AND instr(s2.rest, '/') = 0 +) +WHERE instr(qualified_name, '/') > 0; + +CREATE INDEX IF NOT EXISTS idx_symbols_repo_qsuffix + ON symbols(repo_id, qualified_suffix) + WHERE qualified_suffix != ''; diff --git a/internal/store/store.go b/internal/store/store.go index 6bc5bef..d517af1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1578,7 +1578,7 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID return map[string]int64{}, nil } - args := make([]any, len(symbols)*15) + args := make([]any, len(symbols)*16) argIdx := 0 for _, sym := range symbols { args[argIdx+0] = repoID @@ -1596,7 +1596,8 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID args[argIdx+12] = sym.Range.EndCol args[argIdx+13] = sym.DocSummary args[argIdx+14] = sym.StableKey - argIdx += 15 + args[argIdx+15] = qualifiedSuffix(sym.QualifiedName) + argIdx += 16 } rows, err := tx.QueryContext(ctx, symbolInsertSQL(len(symbols)), args...) @@ -1640,8 +1641,8 @@ func symbolInsertSQL(n int) string { if v, ok := symbolInsertSQLCache.Load(n); ok { return v.(string) } - const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key) VALUES " - const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key, qualified_suffix) VALUES " + const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" const suffix = " RETURNING id, stable_key" var b strings.Builder @@ -2257,6 +2258,19 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { return totalResolved, nil } +// qualifiedSuffix returns the substring of qname after the last '/' for +// slash-containing names, '' otherwise. It mirrors the resolver's `afterSlash` +// logic and is the value persisted in `symbols.qualified_suffix` (migration +// 016) so `resolveEdgesBySlashSuffix` can do an indexed equality JOIN against +// `idx_symbols_repo_qsuffix` instead of a repo-wide symbols scan + Go-side +// hash filter. +func qualifiedSuffix(qname string) string { + if i := strings.LastIndexByte(qname, '/'); i >= 0 && i+1 < len(qname) { + return qname[i+1:] + } + return "" +} + func (s *Store) resolveEdgesByDotSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_edge_dot_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { return 0, err @@ -2355,20 +2369,124 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI // Skip the full symbols scan entirely when no current unresolved edge can // be satisfied by either suffix strategy. This is the common case after // upstream strategies have already absorbed the resolvable edges. -if len(neededSuffix) == 0 { + if len(neededSuffix) == 0 { return 0, nil } + totalResolved := 0 + + // Slash-suffix path: indexed equality JOIN against the persisted + // `qualified_suffix` column (`idx_symbols_repo_qsuffix`, migration 016) + // instead of a `SELECT id, qualified_name FROM symbols WHERE repo_id = ?` + // scan + Go-side neededSuffix hash filter. The MIN(s.id) preserves the + // stable-tie-breaker behaviour of the prior `minBySuffix` map. + { + needNames := make([]string, 0, len(neededSuffix)) + for name := range neededSuffix { + needNames = append(needNames, name) + } + + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_resolver_needed_suffix(dst_name TEXT PRIMARY KEY)`); err != nil { + return 0, err + } + droppedNeeded := false + defer func() { + if droppedNeeded { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_resolver_needed_suffix`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_resolver_needed_suffix`); err != nil { + return 0, err + } + + // Keep well under SQLite's default variable limit (999). + const maxPerInsert = 400 + for start := 0; start < len(needNames); start += maxPerInsert { + end := min(start+maxPerInsert, len(needNames)) + chunk := needNames[start:end] + var b strings.Builder + b.WriteString(`INSERT OR IGNORE INTO tmp_resolver_needed_suffix(dst_name) VALUES `) + args := make([]any, 0, len(chunk)) + for i, name := range chunk { + if i > 0 { + b.WriteString(",") + } + b.WriteString("(?)") + args = append(args, name) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + return 0, err + } + } + + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + return 0, err + } + droppedSlashSuffix := false + defer func() { + if droppedSlashSuffix { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_symbol_slash_suffix`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { + return 0, err + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_symbol_slash_suffix(dst_name, dst_symbol_id) + SELECT n.dst_name, MIN(s.id) + FROM tmp_resolver_needed_suffix n + JOIN symbols s + ON s.repo_id = ? + AND s.qualified_suffix = n.dst_name + WHERE s.qualified_suffix != '' + GROUP BY n.dst_name + `, repoID); err != nil { + return 0, err + } + + updateRes, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_slash_suffix t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) + `, repoID) + if err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { + return 0, err + } + droppedSlashSuffix = true + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_resolver_needed_suffix`); err != nil { + return 0, err + } + droppedNeeded = true + n, err := updateRes.RowsAffected() + if err != nil { + return 0, err + } + totalResolved += int(n) + } + + // Dot-tail2 path: this branch matches `last-2-dot-segments(afterSlash)` + // rather than `afterSlash` itself, so a single `qualified_suffix` column + // can't serve it without schema sprawl (see OPTIMIZATIONS.md Batch 3). + // Keep the Go-side scan but only run it when a current unresolved edge + // could actually consume tail2 candidates. + if len(neededTail2) == 0 { + return totalResolved, nil + } rows, err := tx.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) if err != nil { - return 0, err + return totalResolved, err } type resolvedSuffix struct { dstName string dstID int64 } - // Map: suffix (after last '/') -> minimum symbol id. Using MIN keeps behavior stable when multiple symbols share the same suffix. - minBySuffix := map[string]int64{} // Map: dot tail (last 2 segments after last '/') -> minimum symbol id. minByDotTail2 := map[string]int64{} for rows.Next() { @@ -2376,7 +2494,7 @@ if len(neededSuffix) == 0 { var qname string if err := rows.Scan(&id, &qname); err != nil { _ = rows.Close() - return 0, err + return totalResolved, err } if qname == "" { continue @@ -2385,17 +2503,6 @@ if len(neededSuffix) == 0 { afterSlash := qname if slash := strings.LastIndexByte(qname, '/'); slash >= 0 && slash+1 < len(qname) { afterSlash = qname[slash+1:] - if afterSlash != "" { - if _, ok := neededSuffix[afterSlash]; !ok { - continue - } - if cur, ok := minBySuffix[afterSlash]; !ok { - minBySuffix[strings.Clone(afterSlash)] = id - } else if id < cur { - // Update value only; existing map key remains the cloned string. - minBySuffix[afterSlash] = id - } - } } lastDot := strings.LastIndexByte(afterSlash, '.') @@ -2420,70 +2527,10 @@ if len(neededSuffix) == 0 { } if err := rows.Err(); err != nil { _ = rows.Close() - return 0, err + return totalResolved, err } if err := rows.Close(); err != nil { - return 0, err - } - totalResolved := 0 - - if len(minBySuffix) > 0 { - resolved := make([]resolvedSuffix, 0, len(minBySuffix)) - for dstName, dstID := range minBySuffix { - resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) - } - if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_slash_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { - return 0, err - } - droppedSlashSuffix := false - defer func() { - if droppedSlashSuffix { - return - } - _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_symbol_slash_suffix`) - }() - if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_slash_suffix`); err != nil { - return 0, err - } - - // Keep well under SQLite's default variable limit (999). - const maxPairsPerInsert = 400 - for start := 0; start < len(resolved); start += maxPairsPerInsert { - end := min(start+maxPairsPerInsert, len(resolved)) - chunk := resolved[start:end] - var b strings.Builder - b.WriteString(`INSERT INTO tmp_symbol_slash_suffix(dst_name, dst_symbol_id) VALUES `) - args := make([]any, 0, len(chunk)*2) - for i, r := range chunk { - if i > 0 { - b.WriteString(",") - } - b.WriteString("(?,?)") - args = append(args, r.dstName, r.dstID) - } - if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { - return 0, err - } - } - - updateRes, err := tx.ExecContext(ctx, ` - UPDATE edges - SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_slash_suffix t WHERE t.dst_name = edges.dst_name) - WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL - AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_slash_suffix) - `, repoID) - if err != nil { - return 0, err - } - if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_slash_suffix`); err != nil { - return 0, err - } - droppedSlashSuffix = true - n, err := updateRes.RowsAffected() - if err != nil { - return 0, err - } - totalResolved += int(n) + return totalResolved, err } if len(minByDotTail2) > 0 { From 395676cbb0e00190a17770043e0b535dcc472206 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:00:59 +0200 Subject: [PATCH 51/59] perf(export): writer-based JSONStream for unbounded JSON export (#67) * perf(export): writer-based JSONStream for unbounded JSON export * fix(export): make JSONStream's encodeIndented robust to first-line prefixing --- CHANGELOG.md | 3 + internal/cli/app.go | 6 ++ internal/export/export.go | 159 +++++++++++++++++++++++++++++++++ internal/export/export_test.go | 76 ++++++++++++++++ 4 files changed, 244 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e3f3c..2c55801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store:** Cache symbol insert SQL + IN-placeholder strings to reduce write-path allocations. (#58) - **store:** Steady-state guard skips the full symbols scan in `resolveEdgesBySlashSuffix` when no unresolved non-slash edge remains; ~31× faster / ~1200× fewer allocs on the new no-unresolved bench. (#64) - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) +- **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) +- **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) +- **export:** Add writer-based `JSONStream` for the unbounded JSON export path; CLI `graph export --format json` (no `--limit`, no focus) now streams via `ExportSymbolsPage` / `ExportEdgesPage` to stdout, dropping peak memory from ~3× O(repo) (full symbol + edge slices plus marshalled bytes) to O(pageSize). DOT path still buffered. - **store:** Trim residual write-path allocations in `ReplaceFileGraphsBatch`; add multi-file batch bench. (#60) - **export:** Page no-focus `JSONPaged` directly via `ExportSymbolsPage` / `ExportEdgesPage` so peak memory is O(page) instead of O(repo) on the bounded-page CLI path. (#62) - **indexing/store:** Broad batching + reduced statement pressure across symbols/FTS/inserts; add/extend phase timings + write_stats counters. (#20, #21, #22, #24, #25, #26, #28, #29) diff --git a/internal/cli/app.go b/internal/cli/app.go index 8a9e4b0..94695aa 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1414,6 +1414,12 @@ func runGraph(ctx context.Context, cfg config.Config, stdout io.Writer, args []s _, err = stdout.Write(out) return err } + // Unbounded no-focus JSON: stream straight to stdout via paged loaders so + // peak memory is O(pageSize) instead of O(repo). Bounded/focused paths + // keep the byte-slice JSONPaged shape (single MarshalIndent call). + if selectedSymbol == "" && *limit <= 0 { + return exp.JSONStream(ctx, stdout, repoID, 0) + } out, err := exp.JSONPaged(ctx, repoID, selectedSymbol, 2, *limit, *offset) if err != nil { return err diff --git a/internal/export/export.go b/internal/export/export.go index d209c87..8684610 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -1,6 +1,8 @@ package export import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" @@ -82,6 +84,163 @@ func (s *Service) JSONPaged(ctx context.Context, repoID int64, symbol string, de }, "", " ") } +// JSONStream emits the same single-object shape as `JSON()` (keys: edges, +// repo, stats, symbols — alphabetical to match `MarshalIndent` over a +// `map[string]any`) but pages symbols/edges via `ExportSymbolsPage` / +// `ExportEdgesPage` and writes incrementally to `w`. Peak memory is +// O(pageSize) instead of O(repo): the prior `JSON()` path materialised the +// full symbol + edge slices via `GraphSnapshot("")` and then allocated the +// JSON byte slice, holding ~3× repo-scale memory. The CLI default JSON +// export (`graph export` without `--limit`) routes through this for stdout. +func (s *Service) JSONStream(ctx context.Context, w io.Writer, repoID int64, pageSize int) (err error) { + if pageSize <= 0 { + pageSize = 500 + } + stats, err := s.query.Stats(ctx, repoID) + if err != nil { + return err + } + + bw := bufio.NewWriterSize(w, 64*1024) + defer func() { + if flushErr := bw.Flush(); flushErr != nil && err == nil { + err = flushErr + } + }() + + write := func(s string) error { + _, e := bw.WriteString(s) + return e + } + encodeIndented := func(v any, prefix, indent string) error { + b, e := json.MarshalIndent(v, prefix, indent) + if e != nil { + return e + } + // JSONStream writes the first-line indentation context manually + // before each call. Strip a leading prefix from MarshalIndent's + // output so the first line is never double-indented — defensive + // against the encoder's contract changing or a future swap to one + // that does prefix the first line. + if len(prefix) > 0 && bytes.HasPrefix(b, []byte(prefix)) { + b = b[len(prefix):] + } + _, e = bw.Write(b) + return e + } + + if err = write("{\n"); err != nil { + return err + } + + // "edges": [ ] + if err = write(` "edges": [`); err != nil { + return err + } + first := true + offset := 0 + for { + edges, e := s.query.ExportEdgesPage(ctx, repoID, pageSize, offset) + if e != nil { + return e + } + if len(edges) == 0 { + break + } + for _, edge := range edges { + if first { + if err = write("\n "); err != nil { + return err + } + first = false + } else { + if err = write(",\n "); err != nil { + return err + } + } + if err = encodeIndented(edge, " ", " "); err != nil { + return err + } + } + offset += len(edges) + } + if !first { + if err = write("\n "); err != nil { + return err + } + } + if err = write("],\n"); err != nil { + return err + } + + // "repo": "..." + repoBytes, err := json.Marshal(stats.RepoRoot) + if err != nil { + return err + } + if err = write(` "repo": `); err != nil { + return err + } + if _, err = bw.Write(repoBytes); err != nil { + return err + } + if err = write(",\n"); err != nil { + return err + } + + // "stats": { ... } + if err = write(` "stats": `); err != nil { + return err + } + if err = encodeIndented(stats, " ", " "); err != nil { + return err + } + if err = write(",\n"); err != nil { + return err + } + + // "symbols": [ ] + if err = write(` "symbols": [`); err != nil { + return err + } + first = true + offset = 0 + for { + symbols, e := s.query.ExportSymbolsPage(ctx, repoID, pageSize, offset) + if e != nil { + return e + } + if len(symbols) == 0 { + break + } + for _, sym := range symbols { + if first { + if err = write("\n "); err != nil { + return err + } + first = false + } else { + if err = write(",\n "); err != nil { + return err + } + } + if err = encodeIndented(sym, " ", " "); err != nil { + return err + } + } + offset += len(symbols) + } + if !first { + if err = write("\n "); err != nil { + return err + } + } + if err = write("]\n}\n"); err != nil { + return err + } + return nil +} + func (s *Service) StreamJSONL(ctx context.Context, w io.Writer, repoID int64, pageSize int) error { if pageSize <= 0 { pageSize = 500 diff --git a/internal/export/export_test.go b/internal/export/export_test.go index ea44cb3..255397b 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -1,6 +1,7 @@ package export import ( + "bytes" "context" "encoding/json" "fmt" @@ -129,6 +130,81 @@ func TestJSONPagedNoFocusUsesPagingHelpers(t *testing.T) { } } +// TestJSONStreamMatchesJSONShape validates that the writer-based unbounded +// path emits the same top-level shape as the byte-slice JSON() path +// (`repo`, `stats`, `symbols`, `edges`) and the same row identities. Peak +// memory drops from O(repo) to O(pageSize) is structural — not measured +// here, just the output equivalence is asserted. +func TestJSONStreamMatchesJSONShape(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + for i := 0; i < 6; i++ { + writeFile(t, filepath.Join(repoRoot, fmt.Sprintf("file_%d.go", i)), + fmt.Sprintf("package main\nfunc helper%d() {}\nfunc main%d() { helper%d() }\n", i, i, i)) + } + s, err := store.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + svc := New(query.New(s, nil)) + + jsonOut, err := svc.JSON(ctx, repo.ID) + if err != nil { + t.Fatalf("JSON() error = %v", err) + } + var buf bytes.Buffer + // Force a tiny page size to exercise the page-boundary stitching path. + if err := svc.JSONStream(ctx, &buf, repo.ID, 2); err != nil { + t.Fatalf("JSONStream() error = %v", err) + } + + type doc struct { + Repo string `json:"repo"` + Stats map[string]any `json:"stats"` + Symbols []map[string]any `json:"symbols"` + Edges []map[string]any `json:"edges"` + } + var refDoc, streamDoc doc + if err := json.Unmarshal(jsonOut, &refDoc); err != nil { + t.Fatalf("Unmarshal(JSON output): %v", err) + } + if err := json.Unmarshal(buf.Bytes(), &streamDoc); err != nil { + t.Fatalf("Unmarshal(JSONStream output): %v\n%s", err, buf.String()) + } + if refDoc.Repo != streamDoc.Repo || refDoc.Repo == "" { + t.Fatalf("repo mismatch: ref=%q stream=%q", refDoc.Repo, streamDoc.Repo) + } + if len(refDoc.Symbols) == 0 || len(streamDoc.Symbols) != len(refDoc.Symbols) { + t.Fatalf("symbols count mismatch: ref=%d stream=%d", len(refDoc.Symbols), len(streamDoc.Symbols)) + } + if len(refDoc.Edges) == 0 || len(streamDoc.Edges) != len(refDoc.Edges) { + t.Fatalf("edges count mismatch: ref=%d stream=%d", len(refDoc.Edges), len(streamDoc.Edges)) + } + // Stable_key sets must match exactly (paged loader is ORDER BY id, JSON + // path goes through GraphSnapshot — both end up covering all symbols). + refKeys := map[string]bool{} + for _, sym := range refDoc.Symbols { + if k, ok := sym["stable_key"].(string); ok { + refKeys[k] = true + } + } + for _, sym := range streamDoc.Symbols { + k, _ := sym["stable_key"].(string) + if !refKeys[k] { + t.Fatalf("stream symbol stable_key %q not in JSON() output", k) + } + } +} + func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { From 1fb80817699297a084b2d17b223bf54fa4dc6561 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:24:29 +0200 Subject: [PATCH 52/59] fix(store): delete test_links pointing at purged target files (#68) * fix(store): delete test_links pointing at purged target files * fix(store): populate test_links.target_file_id during indexing --- CHANGELOG.md | 1 + internal/store/purge_ghost_rows_test.go | 197 ++++++++++++++++++ internal/store/store.go | 87 +++++++- .../store/test_links_target_file_id_test.go | 91 ++++++++ 4 files changed, 365 insertions(+), 11 deletions(-) create mode 100644 internal/store/purge_ghost_rows_test.go create mode 100644 internal/store/test_links_target_file_id_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c55801..f79ecca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **cli/help/commands:** Command registry + per-command help; canonical query command names with backward-compatible aliases; help/usage normalization. (#10, #12, #13, #14, #15, #16, #17, #18, #19) ### Fixed +- **store:** Purge-time delete of `test_links` whose `target_file_id` points at a deleted file. Previously the symbol-side companion (`target_symbol_id`) was nulled but the row survived with a stale `target_file_id`, surfacing as a ghost association in `RelatedTests(file=path)` because the path→id lookup does not filter `is_deleted`. Covered by new `TestPurgeDeletes_TestLinksTargetingDeletedFile` and `TestPurgeNullifiesEdgeAndRefSymbolReferences`. - **store/indexer:** Purge deleted-file graph rows and nullify cross-file symbol references. (#46) - **store:** Scope `RelatedTests(file)` correctly via `target_file_id`; make symbol lookup deterministic. (#54) - **store/doctor:** Harden DB pragmas and migration lifecycle correctness; concurrent-open coverage via `TestOpen_ConcurrentMigrateIsSafe`. (#56) diff --git a/internal/store/purge_ghost_rows_test.go b/internal/store/purge_ghost_rows_test.go new file mode 100644 index 0000000..f0f4f6d --- /dev/null +++ b/internal/store/purge_ghost_rows_test.go @@ -0,0 +1,197 @@ +package store + +import ( + "context" + "path/filepath" + "testing" +) + +// TestPurgeDeletes_TestLinksTargetingDeletedFile pins down the +// target_file_id ghost: when a target file is purged, the existing nullify +// pass only nulls test_links.target_symbol_id; without this fix the row +// survives with a stale target_file_id pointing at the soft-deleted file +// row, and RelatedTests(file=path) would surface it as a ghost association. +func TestPurgeDeletes_TestLinksTargetingDeletedFile(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + scanID, _, err := s.BeginScan(ctx, repo.ID, "index") + if err != nil { + t.Fatalf("BeginScan() error = %v", err) + } + + targetA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + targetSym, err := insertTestSymbol(ctx, s, repo.ID, targetA, "Helper", "pkg/a.Helper") + if err != nil { + t.Fatalf("insertTestSymbol(target) error = %v", err) + } + testFile, err := insertTestFile(ctx, s, repo.ID, "a_test.go") + if err != nil { + t.Fatalf("insertTestFile(a_test.go) error = %v", err) + } + testSym, err := insertTestSymbol(ctx, s, repo.ID, testFile, "TestA", "pkg/a.TestA") + if err != nil { + t.Fatalf("insertTestSymbol(test) error = %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO test_links(repo_id, test_file_id, test_symbol_id, target_file_id, target_symbol_id, reason, score) + VALUES(?, ?, ?, ?, ?, 'name_match', 0.9) + `, repo.ID, testFile, testSym, targetA, targetSym); err != nil { + t.Fatalf("insert test_link error = %v", err) + } + + // Mark the target file deleted, then purge. The test file is unrelated to + // this scan and remains live (it has its own last_scan_id). + if _, err := s.MarkFilesDeletedBatch(ctx, repo.ID, scanID, []string{"a.go"}); err != nil { + t.Fatalf("MarkFilesDeletedBatch error = %v", err) + } + purged, err := s.PurgeDeletedFileGraphsForScan(ctx, repo.ID, scanID) + if err != nil { + t.Fatalf("PurgeDeletedFileGraphsForScan error = %v", err) + } + if purged != 1 { + t.Fatalf("PurgeDeletedFileGraphsForScan purged = %d, want 1", purged) + } + + // The test_links row targeting a.go must be gone — not just nulled. + var remaining int + if err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(1) FROM test_links WHERE repo_id = ? AND target_file_id = ? + `, repo.ID, targetA).Scan(&remaining); err != nil { + t.Fatalf("count test_links error = %v", err) + } + if remaining != 0 { + t.Fatalf("test_links targeting deleted file remaining = %d, want 0", remaining) + } + + // And RelatedTests(file=a.go) must surface zero ghosts. The path lookup + // itself still resolves to the soft-deleted file row, which is exactly + // the surface that previously leaked. + got, err := s.RelatedTests(ctx, repo.ID, "", "a.go", 10, 0) + if err != nil { + t.Fatalf("RelatedTests(file=a.go) error = %v", err) + } + if len(got) != 0 { + t.Fatalf("RelatedTests(file=a.go) after purge = %d entries, want 0: %+v", len(got), got) + } +} + +// TestPurgeNullifiesEdgeAndRefSymbolReferences covers the symmetric paths +// already wired up: edges.dst_symbol_id, references_tbl.symbol_id, and +// references_tbl.context_symbol_id must all be NULL (not dangling) after the +// target file's symbols are physically deleted. +func TestPurgeNullifiesEdgeAndRefSymbolReferences(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + scanID, _, err := s.BeginScan(ctx, repo.ID, "index") + if err != nil { + t.Fatalf("BeginScan() error = %v", err) + } + + defFile, err := insertTestFile(ctx, s, repo.ID, "def.go") + if err != nil { + t.Fatalf("insertTestFile(def.go) error = %v", err) + } + defSym, err := insertTestSymbol(ctx, s, repo.ID, defFile, "Helper", "pkg/def.Helper") + if err != nil { + t.Fatalf("insertTestSymbol(def) error = %v", err) + } + useFile, err := insertTestFile(ctx, s, repo.ID, "use.go") + if err != nil { + t.Fatalf("insertTestFile(use.go) error = %v", err) + } + useSym, err := insertTestSymbol(ctx, s, repo.ID, useFile, "Caller", "pkg/use.Caller") + if err != nil { + t.Fatalf("insertTestSymbol(use) error = %v", err) + } + + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, file_id, line) + VALUES(?, ?, ?, ?, 'call', ?, 1) + `, repo.ID, useSym, defSym, "Helper", useFile); err != nil { + t.Fatalf("insert edge error = %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO references_tbl(repo_id, file_id, symbol_id, context_symbol_id, ref_kind, name, start_line, start_col, end_line, end_col) + VALUES(?, ?, ?, ?, 'call', ?, 1, 1, 1, 1) + `, repo.ID, useFile, defSym, defSym, "Helper"); err != nil { + t.Fatalf("insert reference error = %v", err) + } + + if _, err := s.MarkFilesDeletedBatch(ctx, repo.ID, scanID, []string{"def.go"}); err != nil { + t.Fatalf("MarkFilesDeletedBatch error = %v", err) + } + if _, err := s.PurgeDeletedFileGraphsForScan(ctx, repo.ID, scanID); err != nil { + t.Fatalf("PurgeDeletedFileGraphsForScan error = %v", err) + } + + var dangling int + if err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(1) FROM edges WHERE repo_id = ? AND dst_symbol_id IS NOT NULL AND dst_symbol_id NOT IN (SELECT id FROM symbols) + `, repo.ID).Scan(&dangling); err != nil { + t.Fatalf("count dangling edges error = %v", err) + } + if dangling != 0 { + t.Fatalf("dangling edges.dst_symbol_id = %d, want 0", dangling) + } + if err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(1) FROM references_tbl WHERE repo_id = ? AND symbol_id IS NOT NULL AND symbol_id NOT IN (SELECT id FROM symbols) + `, repo.ID).Scan(&dangling); err != nil { + t.Fatalf("count dangling refs.symbol_id error = %v", err) + } + if dangling != 0 { + t.Fatalf("dangling references_tbl.symbol_id = %d, want 0", dangling) + } + if err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(1) FROM references_tbl WHERE repo_id = ? AND context_symbol_id IS NOT NULL AND context_symbol_id NOT IN (SELECT id FROM symbols) + `, repo.ID).Scan(&dangling); err != nil { + t.Fatalf("count dangling refs.context_symbol_id error = %v", err) + } + if dangling != 0 { + t.Fatalf("dangling references_tbl.context_symbol_id = %d, want 0", dangling) + } + + // Stats counters reflect physical row counts. After purge the dst-side + // rows (the def file's symbol) are gone, but the use-file rows remain. + stats, err := s.Stats(ctx, repo.ID) + if err != nil { + t.Fatalf("Stats() error = %v", err) + } + if stats.Files != 1 { + t.Fatalf("Stats.Files post-purge = %d, want 1", stats.Files) + } + if stats.Symbols != 1 { + t.Fatalf("Stats.Symbols post-purge = %d, want 1", stats.Symbols) + } + // The edge and reference rows survive (they belong to the live use file) + // but are now unresolved (dst_symbol_id / symbol_id are NULL). + if stats.Edges != 1 { + t.Fatalf("Stats.Edges post-purge = %d, want 1", stats.Edges) + } + if stats.References != 1 { + t.Fatalf("Stats.References post-purge = %d, want 1", stats.References) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index d517af1..797f6f4 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -55,9 +55,9 @@ const ( // sqliteImportValuesBatchRows controls multi-row inserts into file_imports where each row uses 3 parameters. // 300*3=900 variables, staying under sqliteDefaultMaxVariables. sqliteImportValuesBatchRows = 300 - // sqliteTestLinkValuesBatchRows controls multi-row inserts into test_links where each row uses 6 parameters. - // 150*6=900 variables, staying under sqliteDefaultMaxVariables. - sqliteTestLinkValuesBatchRows = 150 + // sqliteTestLinkValuesBatchRows controls multi-row inserts into test_links where each row uses 7 parameters. + // 128*7=896 variables, staying under sqliteDefaultMaxVariables. + sqliteTestLinkValuesBatchRows = 128 // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 15 parameters. // 60*15=900 variables, staying under sqliteDefaultMaxVariables. @@ -1541,23 +1541,25 @@ func insertParsedFileGraph( for key := range targetKeySet { targetKeys = append(targetKeys, key) } - targetStableToID, err := resolveSymbolsByStableKeysQuery(ctx, tx, repoID, targetKeys) + targetStableToRef, err := resolveSymbolFilesByStableKeysQuery(ctx, tx, repoID, targetKeys) if err != nil { return err } if len(parsed.TestLinks) > 0 { - testLinkArgs := make([]any, 0, min(len(parsed.TestLinks), sqliteTestLinkValuesBatchRows)*6) + testLinkArgs := make([]any, 0, min(len(parsed.TestLinks), sqliteTestLinkValuesBatchRows)*7) for _, link := range parsed.TestLinks { var testSymbolID any var targetSymbolID any + var targetFileID any if id := stableToID[link.TestSymbolKey]; id != 0 { testSymbolID = id } - if id, ok := targetStableToID[link.TargetStableKey]; ok { - targetSymbolID = id + if ref, ok := targetStableToRef[link.TargetStableKey]; ok { + targetSymbolID = ref.id + targetFileID = ref.fileID } - testLinkArgs = append(testLinkArgs, repoID, fileID, testSymbolID, targetSymbolID, link.Reason, link.Score) - if len(testLinkArgs) >= sqliteTestLinkValuesBatchRows*6 { + testLinkArgs = append(testLinkArgs, repoID, fileID, testSymbolID, targetFileID, targetSymbolID, link.Reason, link.Score) + if len(testLinkArgs) >= sqliteTestLinkValuesBatchRows*7 { if err := execTestLinksInsert(ctx, tx, testLinkArgs, stats); err != nil { return err } @@ -1813,7 +1815,7 @@ func execUnresolvedEdgesInsert(ctx context.Context, tx *sql.Tx, args []any, stat } func execTestLinksInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { - return execBatchInsert(ctx, tx, "test_links", "repo_id, test_file_id, test_symbol_id, target_symbol_id, reason, score", 6, args, stats) + return execBatchInsert(ctx, tx, "test_links", "repo_id, test_file_id, test_symbol_id, target_file_id, target_symbol_id, reason, score", 7, args, stats) } func execImportsInsert(ctx context.Context, tx *sql.Tx, args []any, stats *WriteStats) error { @@ -2049,6 +2051,18 @@ func nullifyDeletedSymbolReferences(ctx context.Context, tx *sql.Tx, repoID int6 } symbolIDs := `SELECT id FROM symbols WHERE file_id IN (` + placeholders + `)` + // test_links pointing AT one of the deleted target files become meaningless + // once the target file is gone (target_symbol_id will be nulled out below + // alongside the other symbol-level fan-out, but target_file_id has no + // nullification path of its own — without this delete the row would survive + // with a stale target_file_id and surface in RelatedTests(file=...). + if _, err := tx.ExecContext(ctx, ` + DELETE FROM test_links + WHERE repo_id = ? AND target_file_id IN (`+placeholders+`) + `, args...); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = NULL @@ -2096,6 +2110,16 @@ func nullifyDeletedSymbolReferencesFromTemp(ctx context.Context, tx *sql.Tx, rep return err } + // See nullifyDeletedSymbolReferences: rows with target_file_id pointing at + // any deleted file become orphans even after target_symbol_id is nulled, + // which would surface in RelatedTests(file=...). Delete them up front. + if _, err := tx.ExecContext(ctx, ` + DELETE FROM test_links + WHERE repo_id = ? AND target_file_id IN (SELECT id FROM tmp_delete_file_ids) + `, repoID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, ` UPDATE edges SET dst_symbol_id = NULL @@ -2259,7 +2283,7 @@ func (s *Store) ResolveEdges(ctx context.Context, repoID int64) (int, error) { } // qualifiedSuffix returns the substring of qname after the last '/' for -// slash-containing names, '' otherwise. It mirrors the resolver's `afterSlash` +// slash-containing names, ” otherwise. It mirrors the resolver's `afterSlash` // logic and is the value persisted in `symbols.qualified_suffix` (migration // 016) so `resolveEdgesBySlashSuffix` can do an indexed equality JOIN against // `idx_symbols_repo_qsuffix` instead of a repo-wide symbols scan + Go-side @@ -3052,6 +3076,47 @@ type queryContexter interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } +// symbolFileRef pairs a resolved symbol id with the file_id that hosts it. +// Used by the test_links insert path to populate target_file_id alongside +// target_symbol_id so PurgeDeletedFileGraphsForScan can match rows whose +// target file was deleted. +type symbolFileRef struct { + id int64 + fileID int64 +} + +func resolveSymbolFilesByStableKeysQuery(ctx context.Context, q queryContexter, repoID int64, stableKeys []string) (map[string]symbolFileRef, error) { + out := map[string]symbolFileRef{} + if len(stableKeys) == 0 { + return out, nil + } + placeholders := strings.TrimRight(strings.Repeat("?,", len(stableKeys)), ",") + query := ` + SELECT stable_key, id, file_id + FROM symbols + WHERE repo_id = ? AND stable_key IN (` + placeholders + `) + ` + args := make([]any, 0, len(stableKeys)+1) + args = append(args, repoID) + for _, key := range stableKeys { + args = append(args, key) + } + rows, err := q.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var key string + var ref symbolFileRef + if err := rows.Scan(&key, &ref.id, &ref.fileID); err != nil { + return nil, err + } + out[key] = ref + } + return out, rows.Err() +} + func resolveSymbolsByStableKeysQuery(ctx context.Context, q queryContexter, repoID int64, stableKeys []string) (map[string]int64, error) { out := map[string]int64{} if len(stableKeys) == 0 { diff --git a/internal/store/test_links_target_file_id_test.go b/internal/store/test_links_target_file_id_test.go new file mode 100644 index 0000000..ab6123e --- /dev/null +++ b/internal/store/test_links_target_file_id_test.go @@ -0,0 +1,91 @@ +package store_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/isink17/codegraph/internal/indexer" + "github.com/isink17/codegraph/internal/parser" + goparser "github.com/isink17/codegraph/internal/parser/golang" + "github.com/isink17/codegraph/internal/store" +) + +// TestIndexer_TestLinksPopulateTargetFileID drives the real indexer over a +// tiny pkg/_test.go pair and asserts that test_links rows store +// target_file_id pointing at the target file. Without this column being +// populated, the purge-time DELETE in nullifyDeletedSymbolReferences cannot +// match (target_file_id IS NULL) and ghost test associations leak — and +// RelatedTests(file=...) will not even return live associations because it +// joins on target_file_id. +func TestIndexer_TestLinksPopulateTargetFileID(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(repoRoot, "helper.go"), []byte(`package pkg + +func Helper() {} +`), 0o644); err != nil { + t.Fatalf("write helper.go: %v", err) + } + if err := os.WriteFile(filepath.Join(repoRoot, "helper_test.go"), []byte(`package pkg + +import "testing" + +func TestHelper(t *testing.T) { Helper() } +`), 0o644); err != nil { + t.Fatalf("write helper_test.go: %v", err) + } + + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + // RelatedTests(file=helper.go) joins on test_links.target_file_id, so a + // non-empty result proves target_file_id is populated end-to-end. + got, err := s.RelatedTests(ctx, repo.ID, "", "helper.go", 10, 0) + if err != nil { + t.Fatalf("RelatedTests(file=helper.go) error = %v", err) + } + if len(got) == 0 { + t.Fatalf("RelatedTests(file=helper.go) returned 0 rows; expected at least one (target_file_id likely NULL)") + } + foundTestFile := false + for _, r := range got { + if r.File == "helper_test.go" { + foundTestFile = true + break + } + } + if !foundTestFile { + t.Fatalf("RelatedTests did not surface helper_test.go: %+v", got) + } + + // End-to-end purge: deleting helper.go must drop the linked row, which + // only works if target_file_id was populated so the purge DELETE matches. + if err := os.Remove(filepath.Join(repoRoot, "helper.go")); err != nil { + t.Fatalf("Remove(helper.go) error = %v", err) + } + if _, err := idx.Update(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Update(after delete) error = %v", err) + } + gotAfter, err := s.RelatedTests(ctx, repo.ID, "", "helper.go", 10, 0) + if err != nil { + t.Fatalf("RelatedTests(file=helper.go) post-purge error = %v", err) + } + if len(gotAfter) != 0 { + t.Fatalf("RelatedTests(file=helper.go) post-purge = %d rows, want 0: %+v", len(gotAfter), gotAfter) + } +} From 3c76a0633d4738315be3f5733dec2d91a450e6f1 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:56:48 +0200 Subject: [PATCH 53/59] perf(export): stream unbounded DOT export via paged DISTINCT nodes + edges (#69) * perf(export): stream unbounded DOT export via paged DISTINCT nodes + edges * refactor(export): dedupe DOT edge formatting into a single helper --- CHANGELOG.md | 1 + internal/cli/app.go | 8 +++ internal/export/export.go | 107 ++++++++++++++++++++++++++++----- internal/export/export_test.go | 85 ++++++++++++++++++++++++++ internal/query/service.go | 6 +- internal/store/store.go | 30 +++++++++ 6 files changed, 222 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f79ecca..d226170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **export:** Add writer-based `JSONStream` for the unbounded JSON export path; CLI `graph export --format json` (no `--limit`, no focus) now streams via `ExportSymbolsPage` / `ExportEdgesPage` to stdout, dropping peak memory from ~3× O(repo) (full symbol + edge slices plus marshalled bytes) to O(pageSize). DOT path still buffered. +- **export:** Add writer-based `DOTStream` for the unbounded DOT export path; CLI `graph export --format dot` (no focus symbol) now streams nodes via a new server-side `ExportDOTNodeNamesPage` (`SELECT DISTINCT qualified_name … ORDER BY qualified_name LIMIT/OFFSET`) and edges via `ExportEdgesPage`, dropping peak memory from O(repo) (full `[]graph.Symbol` + `[]ExportEdge` slices + `strings.Builder` buffer) to O(pageSize). Output preserves the existing `digraph codegraph { … }` framing and alphabetically-sorted dedup'd node lines; edge order follows `ORDER BY id ASC` (already unspecified per `DOT()`'s contract). Focused DOT (`--symbol`) keeps the bounded `GraphSnapshot` path. - **store:** Trim residual write-path allocations in `ReplaceFileGraphsBatch`; add multi-file batch bench. (#60) - **export:** Page no-focus `JSONPaged` directly via `ExportSymbolsPage` / `ExportEdgesPage` so peak memory is O(page) instead of O(repo) on the bounded-page CLI path. (#62) - **indexing/store:** Broad batching + reduced statement pressure across symbols/FTS/inserts; add/extend phase timings + write_stats counters. (#20, #21, #22, #24, #25, #26, #28, #29) diff --git a/internal/cli/app.go b/internal/cli/app.go index 94695aa..6d606cd 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1407,6 +1407,14 @@ func runGraph(ctx context.Context, cfg config.Config, stdout io.Writer, args []s return exp.StreamJSONL(ctx, stdout, repoID, *limit) } if *format == "dot" { + // Unbounded no-focus DOT: stream nodes via paged DISTINCT + // qualified_name and edges via ExportEdgesPage so peak memory is + // O(pageSize) instead of O(repo). Focused DOT (--symbol) keeps the + // byte-slice GraphSnapshot path because the bounded subgraph is + // already O(subgraph) and matches the focused-edge dedup shape. + if selectedSymbol == "" { + return exp.DOTStream(ctx, stdout, repoID, 0) + } out, err := exp.DOT(ctx, repoID, selectedSymbol, 2) if err != nil { return err diff --git a/internal/export/export.go b/internal/export/export.go index 8684610..3c7643f 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -291,6 +291,73 @@ func (s *Service) StreamJSONL(ctx context.Context, w io.Writer, repoID int64, pa return enc.Encode(map[string]any{"type": "graph_done"}) } +// DOTStream emits the same `digraph codegraph { ... }` shape as `DOT()` +// (no-focus only) but pages nodes via `ExportDOTNodeNamesPage` (server-side +// `SELECT DISTINCT qualified_name ... ORDER BY qualified_name`) and edges +// via `ExportEdgesPage`, writing incrementally to `w`. Peak memory drops +// from O(repo) (full `[]graph.Symbol` + `[]ExportEdge` slices plus the +// strings.Builder buffer) to O(pageSize). Edge ordering follows +// `ExportEdgesPage` (ID ASC) instead of `loadEdgesForExport`'s insertion +// order; both are unspecified by `DOT()`'s contract. +func (s *Service) DOTStream(ctx context.Context, w io.Writer, repoID int64, pageSize int) (err error) { + if pageSize <= 0 { + pageSize = 500 + } + bw := bufio.NewWriterSize(w, 64*1024) + defer func() { + if flushErr := bw.Flush(); flushErr != nil && err == nil { + err = flushErr + } + }() + + if _, err = bw.WriteString("digraph codegraph {\n"); err != nil { + return err + } + + offset := 0 + for { + names, e := s.query.ExportDOTNodeNamesPage(ctx, repoID, pageSize, offset) + if e != nil { + return e + } + if len(names) == 0 { + break + } + for _, n := range names { + if _, err = fmt.Fprintf(bw, " %q;\n", n); err != nil { + return err + } + } + offset += len(names) + } + + offset = 0 + for { + edges, e := s.query.ExportEdgesPage(ctx, repoID, pageSize, offset) + if e != nil { + return e + } + if len(edges) == 0 { + break + } + for _, edge := range edges { + line := dotEdgeLine(edge) + if line == "" { + continue + } + if _, err = bw.WriteString(line); err != nil { + return err + } + } + offset += len(edges) + } + + if _, err = bw.WriteString("}\n"); err != nil { + return err + } + return nil +} + func (s *Service) DOT(ctx context.Context, repoID int64, symbol string, depth int) ([]byte, error) { symbols, edges, err := s.query.GraphSnapshot(ctx, repoID, symbol, depth) if err != nil { @@ -313,26 +380,38 @@ func (s *Service) DOT(ctx context.Context, repoID int64, symbol string, depth in b.WriteString(fmt.Sprintf(" %q;\n", n)) } for _, edge := range edges { - src := edge.SrcQualifiedName - if src == "" { - src = fmt.Sprintf("symbol#%d", edge.SrcSymbolID) - } - dst := edge.DstQualifiedName - attrs := []string{} - if dst == "" { - dst = edge.DstName - if dst == "" { - continue - } - attrs = append(attrs, `style="dashed"`) + line := dotEdgeLine(edge) + if line == "" { + continue } - attrs = append(attrs, fmt.Sprintf(`label=%q`, edge.Kind)) - b.WriteString(fmt.Sprintf(" %q -> %q [%s];\n", src, dst, strings.Join(attrs, ","))) + b.WriteString(line) } b.WriteString("}\n") return []byte(b.String()), nil } +// dotEdgeLine renders a single DOT edge line (with trailing newline) or "" +// when the edge has neither a resolved nor a fallback dst name. Shared by +// DOT() and DOTStream so the resolved-vs-dashed-vs-skipped attribute logic +// lives in exactly one place. +func dotEdgeLine(edge store.ExportEdge) string { + src := edge.SrcQualifiedName + if src == "" { + src = fmt.Sprintf("symbol#%d", edge.SrcSymbolID) + } + dst := edge.DstQualifiedName + attrs := []string{} + if dst == "" { + dst = edge.DstName + if dst == "" { + return "" + } + attrs = append(attrs, `style="dashed"`) + } + attrs = append(attrs, fmt.Sprintf(`label=%q`, edge.Kind)) + return fmt.Sprintf(" %q -> %q [%s];\n", src, dst, strings.Join(attrs, ",")) +} + func pageSlice[T any](items []T, limit, offset int) []T { start := clampOffset(offset, len(items)) end := clampEnd(start, limit, len(items)) diff --git a/internal/export/export_test.go b/internal/export/export_test.go index 255397b..e31ef50 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" @@ -205,6 +206,90 @@ func TestJSONStreamMatchesJSONShape(t *testing.T) { } } +// TestDOTStreamMatchesDOTShape pins down that the writer-based no-focus +// DOT path emits the same node set and the same digraph framing as the +// byte-slice DOT() path. Edge ordering differs (DOTStream pages via +// ExportEdgesPage's ORDER BY id; DOT() iterates GraphSnapshot's +// loadEdgesForExport result), so this test asserts node parity and +// edge-line set parity rather than byte equality. +func TestDOTStreamMatchesDOTShape(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + for i := 0; i < 6; i++ { + writeFile(t, filepath.Join(repoRoot, fmt.Sprintf("file_%d.go", i)), + fmt.Sprintf("package main\nfunc helper%d() {}\nfunc main%d() { helper%d() }\n", i, i, i)) + } + s, err := store.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + svc := New(query.New(s, nil)) + + refOut, err := svc.DOT(ctx, repo.ID, "", 0) + if err != nil { + t.Fatalf("DOT() error = %v", err) + } + var buf bytes.Buffer + // Force a tiny page size so the stitching path runs. + if err := svc.DOTStream(ctx, &buf, repo.ID, 2); err != nil { + t.Fatalf("DOTStream() error = %v", err) + } + + splitDOT := func(b []byte) (header string, nodes, edges []string, footer string) { + lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n") + if len(lines) < 2 { + t.Fatalf("DOT output too short:\n%s", b) + } + header = lines[0] + footer = lines[len(lines)-1] + for _, ln := range lines[1 : len(lines)-1] { + trim := strings.TrimSpace(ln) + switch { + case strings.Contains(trim, "->"): + edges = append(edges, trim) + case strings.HasSuffix(trim, ";"): + nodes = append(nodes, trim) + } + } + return + } + refHdr, refNodes, refEdges, refFtr := splitDOT(refOut) + strHdr, strNodes, strEdges, strFtr := splitDOT(buf.Bytes()) + + if refHdr != strHdr || refHdr != "digraph codegraph {" { + t.Fatalf("header mismatch: ref=%q stream=%q", refHdr, strHdr) + } + if refFtr != strFtr || refFtr != "}" { + t.Fatalf("footer mismatch: ref=%q stream=%q", refFtr, strFtr) + } + if len(refNodes) == 0 || len(refEdges) == 0 { + t.Fatalf("ref produced no nodes/edges; nodes=%d edges=%d", len(refNodes), len(refEdges)) + } + if !reflect.DeepEqual(refNodes, strNodes) { + t.Fatalf("node lines mismatch:\nref=%v\nstream=%v", refNodes, strNodes) + } + refEdgeSet := map[string]int{} + for _, e := range refEdges { + refEdgeSet[e]++ + } + strEdgeSet := map[string]int{} + for _, e := range strEdges { + strEdgeSet[e]++ + } + if !reflect.DeepEqual(refEdgeSet, strEdgeSet) { + t.Fatalf("edge multiset mismatch:\nref=%v\nstream=%v", refEdgeSet, strEdgeSet) + } +} + func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { diff --git a/internal/query/service.go b/internal/query/service.go index 68f62ec..4a77de2 100644 --- a/internal/query/service.go +++ b/internal/query/service.go @@ -65,7 +65,7 @@ func (s *Service) RelatedTestsForFiles(ctx context.Context, repoID int64, files if limit <= 0 { limit = 20 } -perFileLimit := min(max(50, limit+offset), 1000) + perFileLimit := min(max(50, limit+offset), 1000) seen := map[string]bool{} var all []store.RelatedTest @@ -139,6 +139,10 @@ func (s *Service) ExportEdgesPage(ctx context.Context, repoID int64, limit, offs return s.store.ExportEdgesPage(ctx, repoID, limit, offset) } +func (s *Service) ExportDOTNodeNamesPage(ctx context.Context, repoID int64, limit, offset int) ([]string, error) { + return s.store.ExportDOTNodeNamesPage(ctx, repoID, limit, offset) +} + func (s *Service) TraceDependencies(ctx context.Context, repoID int64, symbol string, direction string, maxDepth int) ([]map[string]any, error) { return s.store.TraceDependencies(ctx, repoID, symbol, direction, maxDepth) } diff --git a/internal/store/store.go b/internal/store/store.go index 797f6f4..d57c52f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3626,6 +3626,36 @@ func (s *Store) ExportSymbolsPage(ctx context.Context, repoID int64, limit, offs return scanSymbols(rows) } +// ExportDOTNodeNamesPage yields the unique, non-empty qualified_name values +// for the repo in stable alphabetical order, paged. It exists so DOTStream +// can emit deduped + sorted DOT node lines without materialising the full +// `[]graph.Symbol` slice (peak memory on the no-focus DOT path was +// previously O(repo) in `Symbol`-sized rows; this trims it to O(pageSize) +// in plain strings). +func (s *Store) ExportDOTNodeNamesPage(ctx context.Context, repoID int64, limit, offset int) ([]string, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT DISTINCT qualified_name + FROM symbols + WHERE repo_id = ? AND qualified_name <> '' + ORDER BY qualified_name ASC + LIMIT ? + OFFSET ? + `, repoID, safeLimit(limit), safeOffset(offset)) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]string, 0, limit) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + func (s *Store) ExportEdgesPage(ctx context.Context, repoID int64, limit, offset int) ([]ExportEdge, error) { rows, err := s.db.QueryContext(ctx, ` SELECT e.id, e.src_symbol_id, COALESCE(src.qualified_name, ''), e.dst_symbol_id, COALESCE(dst.qualified_name, ''), e.dst_name, e.edge_kind, COALESCE(f.path, ''), e.line From e5471d15ba76c2b6f010d51ca2bc58abf52d2eaa Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:25:29 +0200 Subject: [PATCH 54/59] perf(indexer): slim existing-files map value type to drop floor cost (#70) * perf(indexer): slim existing-files map value type to drop floor cost * refactor(store): scan existing-files metadata directly into caller's map --- CHANGELOG.md | 1 + internal/indexer/indexer.go | 12 +++---- internal/indexer/indexer_test.go | 2 +- internal/store/store.go | 59 +++++++++++++++++++------------- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d226170..dd53603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store:** Steady-state guard skips the full symbols scan in `resolveEdgesBySlashSuffix` when no unresolved non-slash edge remains; ~31× faster / ~1200× fewer allocs on the new no-unresolved bench. (#64) - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) +- **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **export:** Add writer-based `JSONStream` for the unbounded JSON export path; CLI `graph export --format json` (no `--limit`, no focus) now streams via `ExportSymbolsPage` / `ExportEdgesPage` to stdout, dropping peak memory from ~3× O(repo) (full symbol + edge slices plus marshalled bytes) to O(pageSize). DOT path still buffered. - **export:** Add writer-based `DOTStream` for the unbounded DOT export path; CLI `graph export --format dot` (no focus symbol) now streams nodes via a new server-side `ExportDOTNodeNamesPage` (`SELECT DISTINCT qualified_name … ORDER BY qualified_name LIMIT/OFFSET`) and edges via `ExportEdgesPage`, dropping peak memory from O(repo) (full `[]graph.Symbol` + `[]ExportEdge` slices + `strings.Builder` buffer) to O(pageSize). Output preserves the existing `digraph codegraph { … }` framing and alphabetically-sorted dedup'd node lines; edge order follows `ORDER BY id ASC` (already unspecified per `DOT()`'s contract). Focused DOT (`--symbol`) keeps the bounded `GraphSnapshot` path. diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 56d5dee..a7fa29a 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -136,7 +136,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err // larger) `tasks` buffer in parallel. On load failure a monitor goroutine // cancels `ctxRun` so the producer/workers drain cleanly. var ( - existing map[string]store.FileRecord + existing map[string]store.ExistingFileMeta loadErr error existingLoadMS int64 ) @@ -145,7 +145,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err go func() { defer close(existingReady) var ( - m map[string]store.FileRecord + m map[string]store.ExistingFileMeta err error ) if pathScoped { @@ -293,7 +293,8 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return } for task := range tasks { - res := processFileTask(ctxRun, task, existing[task.rel], opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy) + prev, hasPrev := existing[task.rel] + res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy) select { case results <- res: case <-ctxRun.Done(): @@ -693,13 +694,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, nil } -func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string) fileResult { +func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string) fileResult { result := fileResult{task: task} started := time.Now() defer func() { result.processDur = time.Since(started) }() - hasPrev := prev.Path != "" if len(allowedLanguages) > 0 && task.language != "" && !slices.Contains(allowedLanguages, task.language) { if hasPrev { @@ -709,7 +709,7 @@ func processFileTask(ctx context.Context, task fileTask, prev store.FileRecord, } return result } - if hasPrev && !prev.IsDeleted && !force && prev.SizeBytes == task.info.Size() && prev.MtimeUnixNS == task.info.ModTime().UnixNano() { + if hasPrev && !force && prev.SizeBytes == task.info.Size() && prev.MtimeUnixNS == task.info.ModTime().UnixNano() { result.action = "mark_seen" return result } diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 35cfcea..d1d2078 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -303,7 +303,7 @@ func main() {} } } -func mapKeys(m map[string]store.FileRecord) []string { +func mapKeys(m map[string]store.ExistingFileMeta) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) diff --git a/internal/store/store.go b/internal/store/store.go index d57c52f..1681437 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -748,16 +748,24 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { return out, nil } +// ExistingFileMeta is the slim per-row value the change-detection path +// actually consumes. It intentionally omits the `Path` (held by the map +// key), `ID`, `Language`, and `IsDeleted` fields — none are read by the +// indexer worker, and dropping them shrinks the per-entry footprint of the +// repo-wide existing-files map. On a 2k/5k-file no-op the trim is the +// dominant remaining alloc on this path. +type ExistingFileMeta struct { + SizeBytes int64 + MtimeUnixNS int64 + ContentHash string +} + // ExistingFiles returns active (non-deleted) file records for the repo, -// projecting only the columns the change-detection path actually consumes -// (`path`, `size_bytes`, `mtime_unix_ns`, `content_sha256`). `ID` and -// `Language` on the returned `FileRecord` are intentionally left zero. -// -// Filtering `is_deleted = 0` server-side avoids materializing tombstone rows -// that can never match a walked path, shrinking both the query result set -// and the resulting Go map. `IsDeleted` is therefore always false on returned -// records. -func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]FileRecord, error) { +// keyed by path with a slim `ExistingFileMeta` value. The projection is +// the change-detection minimum (`size_bytes`, `mtime_unix_ns`, +// `content_sha256`); `is_deleted = 0` is applied server-side so tombstone +// rows never reach the Go map. +func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]ExistingFileMeta, error) { rows, err := s.db.QueryContext(ctx, ` SELECT path, size_bytes, mtime_unix_ns, content_sha256 FROM files @@ -766,13 +774,17 @@ func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]Fil if err != nil { return nil, err } - return scanExistingFiles(rows) + out := map[string]ExistingFileMeta{} + if err := scanExistingFileMetasInto(rows, out); err != nil { + return nil, err + } + return out, nil } // ExistingFilesForPaths is the path-scoped sibling of ExistingFiles; same // projection and same `is_deleted = 0` filter apply. -func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths []string) (map[string]FileRecord, error) { - out := make(map[string]FileRecord, len(paths)) +func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths []string) (map[string]ExistingFileMeta, error) { + out := make(map[string]ExistingFileMeta, len(paths)) if len(paths) == 0 { return out, nil } @@ -795,28 +807,27 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ if err != nil { return nil, err } - records, err := scanExistingFiles(rows) - if err != nil { + if err := scanExistingFileMetasInto(rows, out); err != nil { return nil, err } - for k, v := range records { - out[k] = v - } } return out, nil } -func scanExistingFiles(rows *sql.Rows) (map[string]FileRecord, error) { +// scanExistingFileMetasInto scans rows directly into `dst` so callers can +// reuse a pre-allocated destination map across chunked queries instead of +// paying for a per-chunk map alloc + merge-loop. +func scanExistingFileMetasInto(rows *sql.Rows, dst map[string]ExistingFileMeta) error { defer rows.Close() - out := map[string]FileRecord{} for rows.Next() { - var rec FileRecord - if err := rows.Scan(&rec.Path, &rec.SizeBytes, &rec.MtimeUnixNS, &rec.ContentHash); err != nil { - return nil, err + var path string + var meta ExistingFileMeta + if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS, &meta.ContentHash); err != nil { + return err } - out[rec.Path] = rec + dst[path] = meta } - return out, rows.Err() + return rows.Err() } func scanScanRecords(rows *sql.Rows) ([]ScanRecord, error) { From 66ffe3627b0747f46b4640f21732db7e16dc7199 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:57:06 +0200 Subject: [PATCH 55/59] perf(store/resolver): schema-back dot-tail2 path via persisted column (#71) --- CHANGELOG.md | 1 + .../store/resolve_edges_for_names_test.go | 6 +- .../resolve_edges_strategies_bench_test.go | 75 ++++++++ .../store/schema/017_symbol_dot_tail2.sql | 45 +++++ internal/store/store.go | 163 +++++++++--------- 5 files changed, 205 insertions(+), 85 deletions(-) create mode 100644 internal/store/schema/017_symbol_dot_tail2.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index dd53603..c88abbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) +- **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default). - **export:** Add writer-based `JSONStream` for the unbounded JSON export path; CLI `graph export --format json` (no `--limit`, no focus) now streams via `ExportSymbolsPage` / `ExportEdgesPage` to stdout, dropping peak memory from ~3× O(repo) (full symbol + edge slices plus marshalled bytes) to O(pageSize). DOT path still buffered. - **export:** Add writer-based `DOTStream` for the unbounded DOT export path; CLI `graph export --format dot` (no focus symbol) now streams nodes via a new server-side `ExportDOTNodeNamesPage` (`SELECT DISTINCT qualified_name … ORDER BY qualified_name LIMIT/OFFSET`) and edges via `ExportEdgesPage`, dropping peak memory from O(repo) (full `[]graph.Symbol` + `[]ExportEdge` slices + `strings.Builder` buffer) to O(pageSize). Output preserves the existing `digraph codegraph { … }` framing and alphabetically-sorted dedup'd node lines; edge order follows `ORDER BY id ASC` (already unspecified per `DOT()`'s contract). Focused DOT (`--symbol`) keeps the bounded `GraphSnapshot` path. - **store:** Trim residual write-path allocations in `ReplaceFileGraphsBatch`; add multi-file batch bench. (#60) diff --git a/internal/store/resolve_edges_for_names_test.go b/internal/store/resolve_edges_for_names_test.go index febdd45..c3f7d60 100644 --- a/internal/store/resolve_edges_for_names_test.go +++ b/internal/store/resolve_edges_for_names_test.go @@ -130,10 +130,10 @@ func insertTestSymbol(ctx context.Context, s *Store, repoID, fileID int64, name, res, err := s.db.ExecContext(ctx, ` INSERT INTO symbols( repo_id, file_id, language, kind, name, qualified_name, - start_line, start_col, end_line, end_col, stable_key, qualified_suffix + start_line, start_col, end_line, end_col, stable_key, qualified_suffix, dot_tail2 ) - VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?, ?) - `, repoID, fileID, "go", "function", name, qualified, qualified, qualifiedSuffix(qualified)) + VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?, ?, ?) + `, repoID, fileID, "go", "function", name, qualified, qualified, qualifiedSuffix(qualified), dotTail2(qualified)) if err != nil { return 0, err } diff --git a/internal/store/resolve_edges_strategies_bench_test.go b/internal/store/resolve_edges_strategies_bench_test.go index 9494ea5..ea46f6d 100644 --- a/internal/store/resolve_edges_strategies_bench_test.go +++ b/internal/store/resolve_edges_strategies_bench_test.go @@ -281,6 +281,81 @@ func BenchmarkResolveEdgesBySlashSuffix_SlashOnlyLargeScale(b *testing.B) { } } +// BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale stresses the +// dot-tail2 sub-branch at the same scale as `_SlashOnlyLargeScale` (40k +// total symbols). With the pre-016 / pre-dot_tail2 design the dot-tail2 +// branch always materialised the full repo symbols table in Go and +// derived `tail2` per row — that grows ~O(symbols). The migration-017 +// schema-backed path turns the same matching into an indexed JOIN against +// `symbols.dot_tail2`, which grows ~O(unique_needed_names) instead. +func BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 200 + numNames = 20000 + numNoiseSyms = 20000 + numEdges = 5000 + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + // Dot-tail2 targets: no slash, qualified_name has a leading segment + // before the matched 2-segment tail (e.g. "io.pkg_3.Func_42"). Mirrors + // `_DotTail2`'s qualified-name shape, scaled up to 20k. + for i := 0; i < numNames; i++ { + name := fmt.Sprintf("Func_%d", i) + qualified := fmt.Sprintf("io.pkg_%d.%s", i%50, name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(dot-tail2 dst) error = %v", err) + } + } + // Slash-bearing noise symbols. Their `afterSlash` won't be in + // `neededSuffix` so they don't feed the slash branch, and once + // `dot_tail2` is in the schema they're indexed-out by the partial-index + // `WHERE dot_tail2 != ''` filter (slash-bearing names with no dot in + // `afterSlash` get an empty `dot_tail2`). + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("github.com/org/repo/noise/%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + for i := 0; i < numEdges; i++ { + dstName := fmt.Sprintf("pkg_%d.Func_%d", i%50, i%numNames) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesBySlashSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesBySlashSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + // BenchmarkResolveEdgesBySlashSuffix_NoUnresolved measures the steady-state // floor cost when every edge is already resolved (no candidate dst_names for // either suffix strategy). Captures the win from skipping the full symbols diff --git a/internal/store/schema/017_symbol_dot_tail2.sql b/internal/store/schema/017_symbol_dot_tail2.sql new file mode 100644 index 0000000..9ee835a --- /dev/null +++ b/internal/store/schema/017_symbol_dot_tail2.sql @@ -0,0 +1,45 @@ +-- Strategy 3b: indexed dot-tail2 resolution. +-- +-- `dot_tail2` stores the last two dot-separated segments of `afterSlash` +-- (the substring after the last '/' for slash-containing names, the full +-- qualified_name for non-slash names). It mirrors the in-memory derivation +-- in `resolveEdgesBySlashSuffix`'s dot-tail2 branch, which previously +-- materialised every symbol row into Go and rebuilt this string per row +-- on every resolve pass. Persisting it lets the resolver replace that scan +-- with an indexed equality JOIN against `(repo_id, dot_tail2)`. +-- +-- Backfill walks two recursive CTEs. The first reuses migration 016's +-- `qualified_suffix` (or falls back to `qualified_name` when no '/') so we +-- don't restrip slashes here. The second strips one '.' prefix +-- per step; the terminal row's `prev_rest` is exactly the last-2-segment +-- tail. Rows whose afterSlash has no '.' keep the column at its DEFAULT '' +-- and are excluded from the partial index. + +ALTER TABLE symbols ADD COLUMN dot_tail2 TEXT NOT NULL DEFAULT ''; + +WITH RECURSIVE + after_slash(id, s) AS ( + SELECT id, + CASE + WHEN qualified_suffix != '' THEN qualified_suffix + ELSE qualified_name + END + FROM symbols + ), + strip_dot(id, rest, prev_rest) AS ( + SELECT id, s, '' FROM after_slash WHERE instr(s, '.') > 0 + UNION ALL + SELECT id, substr(rest, instr(rest, '.') + 1), rest + FROM strip_dot + WHERE instr(rest, '.') > 0 + ) +UPDATE symbols +SET dot_tail2 = ( + SELECT prev_rest FROM strip_dot + WHERE strip_dot.id = symbols.id AND instr(strip_dot.rest, '.') = 0 +) +WHERE id IN (SELECT id FROM after_slash WHERE instr(s, '.') > 0); + +CREATE INDEX IF NOT EXISTS idx_symbols_repo_dot_tail2 + ON symbols(repo_id, dot_tail2) + WHERE dot_tail2 != ''; diff --git a/internal/store/store.go b/internal/store/store.go index 1681437..94ebcbd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -59,9 +59,9 @@ const ( // 128*7=896 variables, staying under sqliteDefaultMaxVariables. sqliteTestLinkValuesBatchRows = 128 - // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 15 parameters. - // 60*15=900 variables, staying under sqliteDefaultMaxVariables. - sqliteSymbolValuesBatchRows = 60 + // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 17 parameters. + // 58*17=986 variables, staying under sqliteDefaultMaxVariables. + sqliteSymbolValuesBatchRows = 58 // sqliteSymbolFTSValuesBatchRows controls multi-row inserts into symbol_fts where each row uses 6 parameters. // 150*6=900 variables, staying under sqliteDefaultMaxVariables. sqliteSymbolFTSValuesBatchRows = 150 @@ -1591,7 +1591,7 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID return map[string]int64{}, nil } - args := make([]any, len(symbols)*16) + args := make([]any, len(symbols)*17) argIdx := 0 for _, sym := range symbols { args[argIdx+0] = repoID @@ -1610,7 +1610,8 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID args[argIdx+13] = sym.DocSummary args[argIdx+14] = sym.StableKey args[argIdx+15] = qualifiedSuffix(sym.QualifiedName) - argIdx += 16 + args[argIdx+16] = dotTail2(sym.QualifiedName) + argIdx += 17 } rows, err := tx.QueryContext(ctx, symbolInsertSQL(len(symbols)), args...) @@ -1654,8 +1655,8 @@ func symbolInsertSQL(n int) string { if v, ok := symbolInsertSQLCache.Load(n); ok { return v.(string) } - const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key, qualified_suffix) VALUES " - const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key, qualified_suffix, dot_tail2) VALUES " + const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" const suffix = " RETURNING id, stable_key" var b strings.Builder @@ -2306,6 +2307,28 @@ func qualifiedSuffix(qname string) string { return "" } +// dotTail2 returns the last two dot-separated segments of `qname`'s +// after-slash portion (matching the SQL CASE in migration 017's backfill). +// Empty when there's no dot in afterSlash. Persisted in +// `symbols.dot_tail2` so `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch +// can do an indexed equality JOIN against `idx_symbols_repo_dot_tail2` +// instead of a repo-wide symbols scan + Go-side string slicing. +func dotTail2(qname string) string { + afterSlash := qualifiedSuffix(qname) + if afterSlash == "" { + afterSlash = qname + } + lastDot := strings.LastIndexByte(afterSlash, '.') + if lastDot < 0 || lastDot+1 >= len(afterSlash) { + return "" + } + start := 0 + if prevDot := strings.LastIndexByte(afterSlash[:lastDot], '.'); prevDot >= 0 { + start = prevDot + 1 + } + return afterSlash[start:] +} + func (s *Store) resolveEdgesByDotSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_edge_dot_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { return 0, err @@ -2505,74 +2528,53 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI totalResolved += int(n) } - // Dot-tail2 path: this branch matches `last-2-dot-segments(afterSlash)` - // rather than `afterSlash` itself, so a single `qualified_suffix` column - // can't serve it without schema sprawl (see OPTIMIZATIONS.md Batch 3). - // Keep the Go-side scan but only run it when a current unresolved edge - // could actually consume tail2 candidates. + // Dot-tail2 path: this branch matches `last-2-dot-segments(afterSlash)`. + // Schema-backed by `symbols.dot_tail2` (migration 017) + partial index + // `idx_symbols_repo_dot_tail2`, so the same matching is now an indexed + // equality JOIN instead of a Go-side full repo symbols scan + per-row + // string slicing. if len(neededTail2) == 0 { return totalResolved, nil } - - rows, err := tx.QueryContext(ctx, `SELECT id, qualified_name FROM symbols WHERE repo_id = ?`, repoID) - if err != nil { - return totalResolved, err - } - type resolvedSuffix struct { - dstName string - dstID int64 - } - // Map: dot tail (last 2 segments after last '/') -> minimum symbol id. - minByDotTail2 := map[string]int64{} - for rows.Next() { - var id int64 - var qname string - if err := rows.Scan(&id, &qname); err != nil { - _ = rows.Close() - return totalResolved, err - } - if qname == "" { - continue + { + needNames := make([]string, 0, len(neededTail2)) + for name := range neededTail2 { + needNames = append(needNames, name) } - afterSlash := qname - if slash := strings.LastIndexByte(qname, '/'); slash >= 0 && slash+1 < len(qname) { - afterSlash = qname[slash+1:] + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_resolver_needed_tail2(dst_name TEXT PRIMARY KEY)`); err != nil { + return 0, err } - - lastDot := strings.LastIndexByte(afterSlash, '.') - if lastDot >= 0 && lastDot+1 < len(afterSlash) { - start := 0 - if prevDot := strings.LastIndexByte(afterSlash[:lastDot], '.'); prevDot >= 0 { - start = prevDot + 1 + droppedNeededTail2 := false + defer func() { + if droppedNeededTail2 { + return } - tail2 := afterSlash[start:] - if tail2 != "" { - if _, ok := neededTail2[tail2]; !ok { - continue - } - if cur, ok := minByDotTail2[tail2]; !ok { - minByDotTail2[strings.Clone(tail2)] = id - } else if id < cur { - // Update value only; existing map key remains the cloned string. - minByDotTail2[tail2] = id + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_resolver_needed_tail2`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_resolver_needed_tail2`); err != nil { + return 0, err + } + + const maxPerInsert = 400 + for start := 0; start < len(needNames); start += maxPerInsert { + end := min(start+maxPerInsert, len(needNames)) + chunk := needNames[start:end] + var b strings.Builder + b.WriteString(`INSERT OR IGNORE INTO tmp_resolver_needed_tail2(dst_name) VALUES `) + args := make([]any, 0, len(chunk)) + for i, name := range chunk { + if i > 0 { + b.WriteString(",") } + b.WriteString("(?)") + args = append(args, name) + } + if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { + return 0, err } } - } - if err := rows.Err(); err != nil { - _ = rows.Close() - return totalResolved, err - } - if err := rows.Close(); err != nil { - return totalResolved, err - } - if len(minByDotTail2) > 0 { - resolved := make([]resolvedSuffix, 0, len(minByDotTail2)) - for dstName, dstID := range minByDotTail2 { - resolved = append(resolved, resolvedSuffix{dstName: dstName, dstID: dstID}) - } if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_dot_tail2(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { return 0, err } @@ -2587,24 +2589,17 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI return 0, err } - // Keep well under SQLite's default variable limit (999). - const maxPairsPerInsert = 400 - for start := 0; start < len(resolved); start += maxPairsPerInsert { - end := min(start+maxPairsPerInsert, len(resolved)) - chunk := resolved[start:end] - var b strings.Builder - b.WriteString(`INSERT INTO tmp_symbol_dot_tail2(dst_name, dst_symbol_id) VALUES `) - args := make([]any, 0, len(chunk)*2) - for i, r := range chunk { - if i > 0 { - b.WriteString(",") - } - b.WriteString("(?,?)") - args = append(args, r.dstName, r.dstID) - } - if _, err := tx.ExecContext(ctx, b.String(), args...); err != nil { - return 0, err - } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_symbol_dot_tail2(dst_name, dst_symbol_id) + SELECT n.dst_name, MIN(s.id) + FROM tmp_resolver_needed_tail2 n + JOIN symbols s + ON s.repo_id = ? + AND s.dot_tail2 = n.dst_name + WHERE s.dot_tail2 != '' + GROUP BY n.dst_name + `, repoID); err != nil { + return 0, err } updateRes, err := tx.ExecContext(ctx, ` @@ -2622,6 +2617,10 @@ func (s *Store) resolveEdgesBySlashSuffix(ctx context.Context, tx *sql.Tx, repoI return 0, err } droppedDotTail2 = true + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_resolver_needed_tail2`); err != nil { + return 0, err + } + droppedNeededTail2 = true n, err := updateRes.RowsAffected() if err != nil { return 0, err From eaf4e00339d174b62558f1174145fb4001fed6c6 Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:34:22 +0200 Subject: [PATCH 56/59] perf(store/resolver): schema-back 2-dot dot-suffix path via dot_tail3 (#72) --- CHANGELOG.md | 1 + .../store/resolve_edges_for_names_test.go | 6 +- .../resolve_edges_strategies_bench_test.go | 72 +++++++++ .../store/schema/018_symbol_dot_tail3.sql | 60 +++++++ internal/store/store.go | 148 +++++++++++++++++- 5 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 internal/store/schema/018_symbol_dot_tail3.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c88abbe..4f4c387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default). +- **store/resolver:** Schema-backed dot-suffix path for the dominant 2-dot dst_name case (e.g. `pkg.Sub.Func`) via persisted `symbols.dot_tail3` (migration 018) + partial index `idx_symbols_repo_dot_tail3`. New `resolveEdgesByDotTail3` runs as a schema-backed prelude inside `resolveEdgesByDotSuffix` (indexed equality JOIN against `dot_tail3` for dst_names with exactly 2 dots and no slash), leaving only ≥3-dot dst_names to the original `LIKE '%.' || dst_name` fallback. Measured on new `BenchmarkResolveEdgesByDotSuffixLargeScale` (40k symbols, count=3 × benchtime=2x median, A/B): ns/op 73,216,905,800 (73.2 s) → 74,127,650 (74 ms) — **~988× faster**. Symbol insert SQL bumped to 18 columns (batch row cap 58→55). - **export:** Add writer-based `JSONStream` for the unbounded JSON export path; CLI `graph export --format json` (no `--limit`, no focus) now streams via `ExportSymbolsPage` / `ExportEdgesPage` to stdout, dropping peak memory from ~3× O(repo) (full symbol + edge slices plus marshalled bytes) to O(pageSize). DOT path still buffered. - **export:** Add writer-based `DOTStream` for the unbounded DOT export path; CLI `graph export --format dot` (no focus symbol) now streams nodes via a new server-side `ExportDOTNodeNamesPage` (`SELECT DISTINCT qualified_name … ORDER BY qualified_name LIMIT/OFFSET`) and edges via `ExportEdgesPage`, dropping peak memory from O(repo) (full `[]graph.Symbol` + `[]ExportEdge` slices + `strings.Builder` buffer) to O(pageSize). Output preserves the existing `digraph codegraph { … }` framing and alphabetically-sorted dedup'd node lines; edge order follows `ORDER BY id ASC` (already unspecified per `DOT()`'s contract). Focused DOT (`--symbol`) keeps the bounded `GraphSnapshot` path. - **store:** Trim residual write-path allocations in `ReplaceFileGraphsBatch`; add multi-file batch bench. (#60) diff --git a/internal/store/resolve_edges_for_names_test.go b/internal/store/resolve_edges_for_names_test.go index c3f7d60..b645677 100644 --- a/internal/store/resolve_edges_for_names_test.go +++ b/internal/store/resolve_edges_for_names_test.go @@ -130,10 +130,10 @@ func insertTestSymbol(ctx context.Context, s *Store, repoID, fileID int64, name, res, err := s.db.ExecContext(ctx, ` INSERT INTO symbols( repo_id, file_id, language, kind, name, qualified_name, - start_line, start_col, end_line, end_col, stable_key, qualified_suffix, dot_tail2 + start_line, start_col, end_line, end_col, stable_key, qualified_suffix, dot_tail2, dot_tail3 ) - VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?, ?, ?) - `, repoID, fileID, "go", "function", name, qualified, qualified, qualifiedSuffix(qualified), dotTail2(qualified)) + VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?, ?, ?, ?) + `, repoID, fileID, "go", "function", name, qualified, qualified, qualifiedSuffix(qualified), dotTail2(qualified), dotTail3(qualified)) if err != nil { return 0, err } diff --git a/internal/store/resolve_edges_strategies_bench_test.go b/internal/store/resolve_edges_strategies_bench_test.go index ea46f6d..342dba8 100644 --- a/internal/store/resolve_edges_strategies_bench_test.go +++ b/internal/store/resolve_edges_strategies_bench_test.go @@ -213,6 +213,78 @@ func BenchmarkResolveEdgesByDotSuffix(b *testing.B) { } } +// BenchmarkResolveEdgesByDotSuffixLargeScale stresses +// resolveEdgesByDotSuffix at the same scale as `_SlashOnlyLargeScale` / +// `_DotTail2LargeScale` (40k total symbols, of which 20k carry a multi-dot +// `qualified_name` shape that the LIKE-suffix subquery must scan against). +// dst_names match the strategy's multi-dot, no-slash pre-filter, and the +// per-row correlated subquery `s.qualified_name LIKE '%.' || e.dst_name` +// is unindexable — this fixture is the natural stress for whether the +// remaining LIKE path warrants a schema-backed equality JOIN. +func BenchmarkResolveEdgesByDotSuffixLargeScale(b *testing.B) { + ctx := context.Background() + s := openBenchStore(b) + defer s.Close() + + repoID := upsertBenchRepo(ctx, b, s) + + const ( + numFiles = 200 + numNames = 20000 + numNoiseSyms = 20000 + numEdges = 5000 + ) + + fileIDs := makeBenchFiles(ctx, b, s, repoID, numFiles) + srcIDs := makeBenchSrcSymbols(ctx, b, s, repoID, fileIDs) + + // Multi-dot target symbols: qualified_name has a leading segment before + // the matched 3-segment tail, so `LIKE '%.' || dst_name` matches the + // last three segments. Mirrors the small-scale `_DotSuffix` shape. + for i := 0; i < numNames; i++ { + dstSuffix := fmt.Sprintf("a_%d.b_%d.c_%d", i%50, i%25, i) + qualified := "x." + dstSuffix + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, fmt.Sprintf("c_%d", i), qualified); err != nil { + b.Fatalf("insertTestSymbol(dot-suffix dst) error = %v", err) + } + } + for i := 0; i < numNoiseSyms; i++ { + name := fmt.Sprintf("Noise_%d", i) + qualified := fmt.Sprintf("noise.%s", name) + fileID := fileIDs[i%len(fileIDs)] + if _, err := insertTestSymbol(ctx, s, repoID, fileID, name, qualified); err != nil { + b.Fatalf("insertTestSymbol(noise) error = %v", err) + } + } + + for i := 0; i < numEdges; i++ { + nameIdx := i % numNames + dstName := fmt.Sprintf("a_%d.b_%d.c_%d", nameIdx%50, nameIdx%25, nameIdx) + fileID := fileIDs[i%len(fileIDs)] + srcID := srcIDs[i%len(srcIDs)] + if _, err := insertTestEdge(ctx, s, repoID, fileID, srcID, dstName); err != nil { + b.Fatalf("insertTestEdge() error = %v", err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + b.Fatalf("BeginTx() error = %v", err) + } + b.StartTimer() + if _, err := s.resolveEdgesByDotSuffix(ctx, tx, repoID); err != nil { + b.Fatalf("resolveEdgesByDotSuffix() error = %v", err) + } + b.StopTimer() + _ = tx.Rollback() + } +} + // BenchmarkResolveEdgesBySlashSuffix_SlashOnlyLargeScale stresses the slash // branch with 20k slash-qualified targets + 20k noise symbols. The Go-scan + // hash-filter path is O(symbols), so it grows ~linearly with this fixture diff --git a/internal/store/schema/018_symbol_dot_tail3.sql b/internal/store/schema/018_symbol_dot_tail3.sql new file mode 100644 index 0000000..dc4b5aa --- /dev/null +++ b/internal/store/schema/018_symbol_dot_tail3.sql @@ -0,0 +1,60 @@ +-- Strategy 3c: indexed dot-suffix resolution (2-dot dst_names). +-- +-- `dot_tail3` stores the last three dot-separated segments of `afterSlash` +-- when `afterSlash` has at least three dots (i.e., at least four +-- segments). It mirrors the predicate +-- `qualified_name LIKE '%.' || dst_name` in `resolveEdgesByDotSuffix` +-- restricted to 2-dot dst_names: those need exactly the last three segments +-- of `afterSlash` to match, AND there must be a preceding segment in +-- `afterSlash` (otherwise the leading wildcard `%` cannot bind any chars +-- before the leading dot of the LIKE pattern). When `afterSlash` has only +-- 2 or fewer dots, no multi-dot dst_name can ever match it via this +-- path — `dot_tail3` stays at its DEFAULT '' and is excluded from the +-- partial index. +-- +-- Backfill walks two recursive CTEs. The first reuses migration 016's +-- `qualified_suffix` (or falls back to `qualified_name` for non-slash +-- names) so we don't restrip slashes. The second strips one +-- `.` prefix per step until the remaining tail has ≤1 dot +-- (i.e., we've stripped down to two segments); the terminal row's +-- `prev_rest` then holds the last three segments. Rows whose `afterSlash` +-- has fewer than three dots are filtered out by the seeding `multi_dot` +-- CTE so they never enter the recursion. +-- +-- 2-dot dst_names are by far the dominant multi-dot case in real codebases +-- (e.g., `pkg.Sub.Func`); ≥3-dot dst_names are rare and continue to fall +-- through to the original LIKE-based path inside `resolveEdgesByDotSuffix`. + +ALTER TABLE symbols ADD COLUMN dot_tail3 TEXT NOT NULL DEFAULT ''; + +WITH RECURSIVE + after_slash(id, s) AS ( + SELECT id, + CASE + WHEN qualified_suffix != '' THEN qualified_suffix + ELSE qualified_name + END + FROM symbols + ), + multi_dot(id, s) AS ( + SELECT id, s FROM after_slash + WHERE length(s) - length(replace(s, '.', '')) >= 3 + ), + strip_dot(id, rest, prev_rest) AS ( + SELECT id, s, '' FROM multi_dot + UNION ALL + SELECT id, substr(rest, instr(rest, '.') + 1), rest + FROM strip_dot + WHERE length(rest) - length(replace(rest, '.', '')) > 1 + ) +UPDATE symbols +SET dot_tail3 = ( + SELECT prev_rest FROM strip_dot + WHERE strip_dot.id = symbols.id + AND length(strip_dot.rest) - length(replace(strip_dot.rest, '.', '')) <= 1 +) +WHERE id IN (SELECT id FROM multi_dot); + +CREATE INDEX IF NOT EXISTS idx_symbols_repo_dot_tail3 + ON symbols(repo_id, dot_tail3) + WHERE dot_tail3 != ''; diff --git a/internal/store/store.go b/internal/store/store.go index 94ebcbd..ef4e2f8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -59,9 +59,9 @@ const ( // 128*7=896 variables, staying under sqliteDefaultMaxVariables. sqliteTestLinkValuesBatchRows = 128 - // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 17 parameters. - // 58*17=986 variables, staying under sqliteDefaultMaxVariables. - sqliteSymbolValuesBatchRows = 58 + // sqliteSymbolValuesBatchRows controls multi-row inserts into symbols where each row uses 18 parameters. + // 55*18=990 variables, staying under sqliteDefaultMaxVariables. + sqliteSymbolValuesBatchRows = 55 // sqliteSymbolFTSValuesBatchRows controls multi-row inserts into symbol_fts where each row uses 6 parameters. // 150*6=900 variables, staying under sqliteDefaultMaxVariables. sqliteSymbolFTSValuesBatchRows = 150 @@ -1591,7 +1591,7 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID return map[string]int64{}, nil } - args := make([]any, len(symbols)*17) + args := make([]any, len(symbols)*18) argIdx := 0 for _, sym := range symbols { args[argIdx+0] = repoID @@ -1611,7 +1611,8 @@ func insertSymbolsBatchReturning(ctx context.Context, tx *sql.Tx, repoID, fileID args[argIdx+14] = sym.StableKey args[argIdx+15] = qualifiedSuffix(sym.QualifiedName) args[argIdx+16] = dotTail2(sym.QualifiedName) - argIdx += 17 + args[argIdx+17] = dotTail3(sym.QualifiedName) + argIdx += 18 } rows, err := tx.QueryContext(ctx, symbolInsertSQL(len(symbols)), args...) @@ -1655,8 +1656,8 @@ func symbolInsertSQL(n int) string { if v, ok := symbolInsertSQLCache.Load(n); ok { return v.(string) } - const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key, qualified_suffix, dot_tail2) VALUES " - const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + const prefix = "INSERT INTO symbols(repo_id, file_id, language, kind, name, qualified_name, container_name, signature, visibility, start_line, start_col, end_line, end_col, doc_summary, stable_key, qualified_suffix, dot_tail2, dot_tail3) VALUES " + const row = "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" const suffix = " RETURNING id, stable_key" var b strings.Builder @@ -2307,6 +2308,37 @@ func qualifiedSuffix(qname string) string { return "" } +// dotTail3 returns the last three dot-separated segments of `qname`'s +// after-slash portion when that portion has at least three dots +// (i.e., at least four segments). Empty otherwise — including the case +// where `afterSlash` has exactly three segments, because such a qname +// can never match a `LIKE '%.' || dst_name` pattern (no preceding `.` +// before the leading segment). Persisted in `symbols.dot_tail3` so +// `resolveEdgesByDotSuffix`'s 2-dot dst_name path can do an indexed +// equality JOIN against `idx_symbols_repo_dot_tail3` instead of a +// repo-wide LIKE scan. +func dotTail3(qname string) string { + afterSlash := qualifiedSuffix(qname) + if afterSlash == "" { + afterSlash = qname + } + dots := strings.Count(afterSlash, ".") + if dots < 3 { + return "" + } + rest := afterSlash + // Strip leading "." (dots-2) times so `rest` becomes the last + // three segments. e.g. "a.b.c.d.e" with dots=4 strips twice → "c.d.e". + for i := 0; i < dots-2; i++ { + idx := strings.IndexByte(rest, '.') + if idx < 0 { + return "" + } + rest = rest[idx+1:] + } + return rest +} + // dotTail2 returns the last two dot-separated segments of `qname`'s // after-slash portion (matching the SQL CASE in migration 017's backfill). // Empty when there's no dot in afterSlash. Persisted in @@ -2330,6 +2362,22 @@ func dotTail2(qname string) string { } func (s *Store) resolveEdgesByDotSuffix(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { + totalResolved := 0 + + // Schema-backed prelude: 2-dot, no-slash dst_names match exactly the + // last three segments of `afterSlash`, which is persisted in + // `symbols.dot_tail3` (migration 018) + indexed by + // `idx_symbols_repo_dot_tail3`. An indexed equality JOIN replaces the + // per-distinct-dst-name LIKE scan for this dominant multi-dot case. + // dst_names with ≥3 dots fall through to the LIKE fallback below. + { + n, err := s.resolveEdgesByDotTail3(ctx, tx, repoID) + if err != nil { + return totalResolved, err + } + totalResolved += n + } + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_edge_dot_suffix(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { return 0, err } @@ -2385,6 +2433,92 @@ func (s *Store) resolveEdgesByDotSuffix(ctx context.Context, tx *sql.Tx, repoID if err != nil { return 0, err } + totalResolved += int(n) + return totalResolved, nil +} + +// resolveEdgesByDotTail3 handles the 2-dot, no-slash dst_name case via the +// schema-backed `symbols.dot_tail3` partial index (migration 018). The +// matching predicate is the same as the LIKE pattern +// `qualified_name LIKE '%.' || dst_name` for 2-dot dst_names, but bound +// to an equality JOIN that scales with the unique-needed-name count +// instead of total symbols. +func (s *Store) resolveEdgesByDotTail3(ctx context.Context, tx *sql.Tx, repoID int64) (int, error) { + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_resolver_needed_tail3(dst_name TEXT PRIMARY KEY)`); err != nil { + return 0, err + } + droppedNeeded := false + defer func() { + if droppedNeeded { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_resolver_needed_tail3`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_resolver_needed_tail3`); err != nil { + return 0, err + } + + // Seed with distinct unresolved dst_names that have exactly 2 dots and + // no slash. (Length-of-string minus length-of-dotless-string == 2 dots.) + if _, err := tx.ExecContext(ctx, ` + INSERT OR IGNORE INTO tmp_resolver_needed_tail3(dst_name) + SELECT DISTINCT dst_name + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL + AND instr(dst_name, '/') = 0 + AND length(dst_name) - length(replace(dst_name, '.', '')) = 2 + `, repoID); err != nil { + return 0, err + } + + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS tmp_symbol_dot_tail3(dst_name TEXT PRIMARY KEY, dst_symbol_id INTEGER NOT NULL)`); err != nil { + return 0, err + } + droppedDotTail3 := false + defer func() { + if droppedDotTail3 { + return + } + _, _ = tx.ExecContext(ctx, `DROP TABLE IF EXISTS tmp_symbol_dot_tail3`) + }() + if _, err := tx.ExecContext(ctx, `DELETE FROM tmp_symbol_dot_tail3`); err != nil { + return 0, err + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tmp_symbol_dot_tail3(dst_name, dst_symbol_id) + SELECT n.dst_name, MIN(s.id) + FROM tmp_resolver_needed_tail3 n + JOIN symbols s + ON s.repo_id = ? + AND s.dot_tail3 = n.dst_name + WHERE s.dot_tail3 != '' + GROUP BY n.dst_name + `, repoID); err != nil { + return 0, err + } + + updateRes, err := tx.ExecContext(ctx, ` + UPDATE edges + SET dst_symbol_id = (SELECT t.dst_symbol_id FROM tmp_symbol_dot_tail3 t WHERE t.dst_name = edges.dst_name) + WHERE edges.repo_id = ? AND edges.dst_symbol_id IS NULL + AND edges.dst_name IN (SELECT dst_name FROM tmp_symbol_dot_tail3) + `, repoID) + if err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_symbol_dot_tail3`); err != nil { + return 0, err + } + droppedDotTail3 = true + if _, err := tx.ExecContext(ctx, `DROP TABLE tmp_resolver_needed_tail3`); err != nil { + return 0, err + } + droppedNeeded = true + n, err := updateRes.RowsAffected() + if err != nil { + return 0, err + } return int(n), nil } From 7bb3751239762617410c1dfdcb0e69506e32152d Mon Sep 17 00:00:00 2001 From: isink <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:20:45 +0200 Subject: [PATCH 57/59] =?UTF-8?q?perf(store/indexer):=20defer=20ContentHas?= =?UTF-8?q?h=20to=20lazy=20lookup;=20trim=20ExistingF=E2=80=A6=20(#73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(store/indexer): defer ContentHash to lazy lookup; trim ExistingFiles map row size * fix(indexer): propagate hashLookup errors through processFileTask --- CHANGELOG.md | 2 + internal/indexer/indexer.go | 20 ++- internal/indexer/indexer_bench_test.go | 39 ++++++ internal/store/store.go | 46 +++++-- internal/store/store_bench_test.go | 165 +++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f4c387..fcb5de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. +- **store/indexer:** Defer `ContentHash` from the upfront existing-files map to a lazy per-path lookup. `ExistingFileMeta` further trimmed to `{SizeBytes, MtimeUnixNS}`; `ExistingFiles` / `ExistingFilesForPaths` SQL projection narrowed to `(path, size_bytes, mtime_unix_ns)`. New `Store.LookupFileContentHash(repoID, path)` is consulted only on a (size,mtime) mismatch via a `hashLookup func(rel) (string, bool)` callback threaded into `processFileTask`. The fast path of a no-op repo-wide update never allocates one hex string per file. New `BenchmarkIndexerNoOpUpdateRepoWide_FileScaling` sweeps 500/2000/5000 files. A/B at 2000 files (count=3 × benchtime=8x median): B/op 1854 KB → 1411 KB (~24% less); allocs/op 28,930 → 20,930 (~28% fewer); ns/op 23.0 → 22.7 ms (within noise — FS walk dominates). - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default). - **store/resolver:** Schema-backed dot-suffix path for the dominant 2-dot dst_name case (e.g. `pkg.Sub.Func`) via persisted `symbols.dot_tail3` (migration 018) + partial index `idx_symbols_repo_dot_tail3`. New `resolveEdgesByDotTail3` runs as a schema-backed prelude inside `resolveEdgesByDotSuffix` (indexed equality JOIN against `dot_tail3` for dst_names with exactly 2 dots and no slash), leaving only ≥3-dot dst_names to the original `LIKE '%.' || dst_name` fallback. Measured on new `BenchmarkResolveEdgesByDotSuffixLargeScale` (40k symbols, count=3 × benchtime=2x median, A/B): ns/op 73,216,905,800 (73.2 s) → 74,127,650 (74 ms) — **~988× faster**. Symbol insert SQL bumped to 18 columns (batch row cap 58→55). @@ -33,6 +34,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **benchmark:** Add `--sqlite-profile` and capture sqlite_profile/host context. (#44) - **store/bench:** Add chooser + callers/callees benchmarks; retry `applyPragmas` on `SQLITE_BUSY`. (#57) - **store/bench:** Add per-strategy resolver microbenchmarks (slash-suffix, dot-tail2, dot-suffix). (#59) +- **store/bench:** Add per-component write-path microbenchmarks: `BenchmarkStoreReplaceFileGraph_EdgeHeavy` (5000 edges, 1 src) isolates per-edge insert; `_TokenHeavy` (200 syms × ~25 tokens) isolates `execTokenTriplesInsert`; `_FileScaling` (16×40 syms vs 80×8 syms at constant total) surfaces per-file batch overhead. Measured: ~5.4 allocs/edge and ~6 allocs/token (interface{} boxing floor on `[]any` row args), ~100 allocs/file batch overhead. Hot loops verified clean (SQL cached, args reused, no per-row string ops); no localized hotspot above noise. - **store:** Speed up `FindDeadCode` with new `idx_refs_repo_symbol_id` / `idx_refs_repo_context_symbol_id` indexes. (#54) - **cli:** Add `index_smoke` runner with compact jsonl + median baseline for perf diffs. (#45) - **cli/config:** Default repo artifacts under `.codegraph/` (DB + bench gocache) with legacy DB fallback; harden repo DB path handling. (#49) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index a7fa29a..b01170b 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -292,9 +292,12 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err } return } + hashLookup := func(rel string) (string, bool, error) { + return i.store.LookupFileContentHash(ctxRun, repo.ID, rel) + } for task := range tasks { prev, hasPrev := existing[task.rel] - res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy) + res := processFileTask(ctxRun, task, prev, hasPrev, opts.Force, repoCfg.MaxFileSizeBytes, opts.Languages, repoCfg.ParseErrorPolicy, hashLookup) select { case results <- res: case <-ctxRun.Done(): @@ -694,7 +697,7 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err return summary, nil } -func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string) fileResult { +func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFileMeta, hasPrev bool, force bool, maxFileSizeBytes int64, allowedLanguages []string, parseErrorPolicy string, hashLookup func(rel string) (string, bool, error)) fileResult { result := fileResult{task: task} started := time.Now() defer func() { @@ -742,9 +745,16 @@ func processFileTask(ctx context.Context, task fileTask, prev store.ExistingFile result.hashDur = time.Since(hashStarted) } result.hash = hash - if hasPrev && !force && prev.ContentHash == hash { - result.action = "touch" - return result + if hasPrev && !force && hashLookup != nil { + prevHash, ok, err := hashLookup(task.rel) + if err != nil { + result.err = err + return result + } + if ok && prevHash == hash { + result.action = "touch" + return result + } } parsed := graph.ParsedFile{Language: task.language} diff --git a/internal/indexer/indexer_bench_test.go b/internal/indexer/indexer_bench_test.go index 260a41e..f2eb0c3 100644 --- a/internal/indexer/indexer_bench_test.go +++ b/internal/indexer/indexer_bench_test.go @@ -87,6 +87,45 @@ func BenchmarkIndexerNoOpUpdateRepoWide(b *testing.B) { } } +// BenchmarkIndexerNoOpUpdateRepoWide_FileScaling exposes the per-row cost of +// the existing-files map by sweeping fixture sizes. Surfaces wins or +// regressions in the change-detection floor as file count grows; the slim +// `ExistingFileMeta` (no `ContentHash`) keeps per-row alloc bounded so the +// curve stays roughly linear in N rather than amplified by per-file string +// allocs. +func BenchmarkIndexerNoOpUpdateRepoWide_FileScaling(b *testing.B) { + ctx := context.Background() + cases := []int{500, 2000, 5000} + for _, files := range cases { + b.Run(fmt.Sprintf("files=%d", files), func(b *testing.B) { + repoRoot := b.TempDir() + createGoFixtureRepo(b, repoRoot, files) + dbPath := filepath.Join(b.TempDir(), "bench-noop-update-scaling.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: sqliteBenchProfile()}) + if err != nil { + b.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + idx := New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, Options{RepoRoot: repoRoot}); err != nil { + b.Fatalf("Index() error = %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + summary, err := idx.Update(ctx, Options{RepoRoot: repoRoot}) + if err != nil { + b.Fatalf("Update() error = %v", err) + } + if summary.FilesChanged != 0 { + b.Fatalf("expected 0 files changed on no-op update, got %d", summary.FilesChanged) + } + } + }) + } +} + func BenchmarkIndexerUpdateOneFile(b *testing.B) { ctx := context.Background() repoRoot := b.TempDir() diff --git a/internal/store/store.go b/internal/store/store.go index ef4e2f8..14e368f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -749,25 +749,26 @@ func QueryDBPragmas(ctx context.Context, db *sql.DB) (DBPragmas, error) { } // ExistingFileMeta is the slim per-row value the change-detection path -// actually consumes. It intentionally omits the `Path` (held by the map -// key), `ID`, `Language`, and `IsDeleted` fields — none are read by the -// indexer worker, and dropping them shrinks the per-entry footprint of the -// repo-wide existing-files map. On a 2k/5k-file no-op the trim is the -// dominant remaining alloc on this path. +// actually consumes upfront. It carries only the (size, mtime) pair — +// the fields the indexer's fast path checks first. `content_sha256` is +// deliberately NOT in this struct: it's only consulted in the slow +// branch (size/mtime mismatch) and is fetched lazily via +// `LookupFileContentHash` so a repo-wide no-op load doesn't allocate +// one hex string per file. type ExistingFileMeta struct { SizeBytes int64 MtimeUnixNS int64 - ContentHash string } // ExistingFiles returns active (non-deleted) file records for the repo, // keyed by path with a slim `ExistingFileMeta` value. The projection is -// the change-detection minimum (`size_bytes`, `mtime_unix_ns`, -// `content_sha256`); `is_deleted = 0` is applied server-side so tombstone -// rows never reach the Go map. +// the fast-path minimum (`size_bytes`, `mtime_unix_ns`); `is_deleted = 0` +// is applied server-side so tombstone rows never reach the Go map. The +// content hash is fetched on demand via `LookupFileContentHash` only +// when (size, mtime) differs from disk. func (s *Store) ExistingFiles(ctx context.Context, repoID int64) (map[string]ExistingFileMeta, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT path, size_bytes, mtime_unix_ns, content_sha256 + SELECT path, size_bytes, mtime_unix_ns FROM files WHERE repo_id = ? AND is_deleted = 0 `, repoID) @@ -794,7 +795,7 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ chunk := paths[start:end] placeholders := strings.TrimRight(strings.Repeat("?,", len(chunk)), ",") query := ` - SELECT path, size_bytes, mtime_unix_ns, content_sha256 + SELECT path, size_bytes, mtime_unix_ns FROM files WHERE repo_id = ? AND is_deleted = 0 AND path IN (` + placeholders + `) ` @@ -814,6 +815,27 @@ func (s *Store) ExistingFilesForPaths(ctx context.Context, repoID int64, paths [ return out, nil } +// LookupFileContentHash returns the stored `content_sha256` for the given +// active file path, or ("", false, nil) if no live row exists. Used by the +// indexer's slow-path "touch vs. replace" decision after a (size, mtime) +// mismatch — folding this into a per-file query keeps the upfront +// `ExistingFiles` map free of per-row hex strings. +func (s *Store) LookupFileContentHash(ctx context.Context, repoID int64, path string) (string, bool, error) { + var hash string + err := s.db.QueryRowContext(ctx, ` + SELECT content_sha256 + FROM files + WHERE repo_id = ? AND is_deleted = 0 AND path = ? + `, repoID, path).Scan(&hash) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return hash, true, nil +} + // scanExistingFileMetasInto scans rows directly into `dst` so callers can // reuse a pre-allocated destination map across chunked queries instead of // paying for a per-chunk map alloc + merge-loop. @@ -822,7 +844,7 @@ func scanExistingFileMetasInto(rows *sql.Rows, dst map[string]ExistingFileMeta) for rows.Next() { var path string var meta ExistingFileMeta - if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS, &meta.ContentHash); err != nil { + if err := rows.Scan(&path, &meta.SizeBytes, &meta.MtimeUnixNS); err != nil { return err } dst[path] = meta diff --git a/internal/store/store_bench_test.go b/internal/store/store_bench_test.go index 3da6504..a681413 100644 --- a/internal/store/store_bench_test.go +++ b/internal/store/store_bench_test.go @@ -144,6 +144,171 @@ func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy(b *testing.B) { } } +// BenchmarkStoreReplaceFileGraph_EdgeHeavy isolates the per-edge insert +// cost path: 1 source symbol, many edges, no refs, no tokens. If a +// per-edge `dst_name`/args allocation hotspot exists, this surfaces it +// without the noise of the symbol-token / FTS / file-token paths. +func BenchmarkStoreReplaceFileGraph_EdgeHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-edges.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + const numEdges = 5000 + parsed := graph.ParsedFile{Language: "go"} + parsed.Symbols = []graph.Symbol{{ + Language: "go", + Kind: "function", + Name: "Src", + QualifiedName: "bench.Src", + Range: graph.Position{StartLine: 1, StartCol: 1, EndLine: 1, EndCol: 10}, + StableKey: "go:bench.Src:0", + }} + parsed.Edges = make([]graph.Edge, 0, numEdges) + for e := 0; e < numEdges; e++ { + parsed.Edges = append(parsed.Edges, graph.Edge{ + DstName: fmt.Sprintf("bench.Dst_%d", e), + Kind: "calls", + Evidence: "bench", + Line: e + 1, + }) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/edges.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil { + b.Fatalf("ReplaceFileGraph() error = %v", err) + } + } +} + +// BenchmarkStoreReplaceFileGraph_TokenHeavy isolates the per-token +// (`execTokenTriplesInsert`) path: small symbol count, large per-symbol +// token cardinality. Surfaces alloc/CPU cost in token batching loops. +func BenchmarkStoreReplaceFileGraph_TokenHeavy(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + b.Logf("sqlite_driver=%s", store.SQLiteDriverName()) + b.Logf("sqlite_profile=%s", profile) + + dbPath := filepath.Join(b.TempDir(), "store-tokens.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + // Long doc summary + signature drives the tokenizer to many tokens per + // symbol. 200 symbols × ~25 tokens/symbol ≈ 5k symbol_tokens rows. + docs := "this is a long descriptive doc summary covering multiple subjects and keywords for benchmark purposes including alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu" + parsed := graph.ParsedFile{Language: "go"} + const numSymbols = 200 + parsed.Symbols = make([]graph.Symbol, 0, numSymbols) + for i := 0; i < numSymbols; i++ { + name := fmt.Sprintf("TokenFn%d", i) + qname := fmt.Sprintf("bench.%s", name) + parsed.Symbols = append(parsed.Symbols, graph.Symbol{ + Language: "go", + Kind: "function", + Name: name, + QualifiedName: qname, + Signature: fmt.Sprintf("func %s(ctx context.Context, request *Request, options []Option) (*Response, error)", name), + DocSummary: docs, + Range: graph.Position{StartLine: i + 1, StartCol: 1, EndLine: i + 1, EndCol: 10}, + StableKey: fmt.Sprintf("go:%s:%d", qname, i), + }) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.ReplaceFileGraph(ctx, repo.ID, 1, "bench/tokens.go", "go", 1024, int64(i), strconv.Itoa(i), parsed); err != nil { + b.Fatalf("ReplaceFileGraph() error = %v", err) + } + } +} + +// BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling reveals +// per-batch / per-file overhead by holding total symbol count constant +// (~640 symbols) but varying file count (16 files × 40 syms vs +// 80 files × 8 syms). If per-file overhead (file upsert, deleteFileGraphs +// preamble, args slice setup) dominates over per-symbol cost, the +// many-files variant is meaningfully slower. +func BenchmarkStoreReplaceFileGraphsBatchWriteHeavy_FileScaling(b *testing.B) { + cases := []struct { + name string + numFiles int + perFile int + }{ + {"16files_40syms", 16, 40}, + {"80files_8syms", 80, 8}, + } + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + ctx := context.Background() + profile := sqliteBenchProfile() + dbPath := filepath.Join(b.TempDir(), "store-batch-scaling.sqlite") + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{PerformanceProfile: profile}) + if err != nil { + b.Fatalf("store.OpenWithOptions() error = %v", err) + } + b.Cleanup(func() { _ = s.Close() }) + + repoRoot := b.TempDir() + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + b.Fatalf("UpsertRepo() error = %v", err) + } + + parsed := heavyParsedFile(tc.perFile, 2, 2) + inputs := make([]store.ReplaceFileGraphInput, 0, tc.numFiles) + for j := 0; j < tc.numFiles; j++ { + inputs = append(inputs, store.ReplaceFileGraphInput{ + Path: fmt.Sprintf("bench/scale_%d.go", j), + Language: "go", + SizeBytes: 1024, + MtimeUnixNS: 0, + ContentHash: "", + Parsed: parsed, + }) + } + stats := &store.WriteStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + hash := strconv.Itoa(i) + for j := range inputs { + inputs[j].MtimeUnixNS = int64(i) + inputs[j].ContentHash = hash + } + if _, err := s.ReplaceFileGraphsBatchWithStats(ctx, repo.ID, 1, inputs, stats); err != nil { + b.Fatalf("ReplaceFileGraphsBatchWithStats() error = %v", err) + } + } + }) + } +} + func setupStoreBenchData(b *testing.B, ctx context.Context) (*store.Store, int64) { b.Helper() repoRoot := b.TempDir() From ce540b2d6f5228bc38cb7aa733bb73f525679512 Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:26:17 +0200 Subject: [PATCH 58/59] docs(changelog): clarify hashLookup callback signature + error propagation --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb5de1..23170e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Changes since `v1.0.9` (based on `git log v1.0.9..HEAD`). - **store/indexer:** Trim repo-wide change-detection floor cost by narrowing `ExistingFiles` / `ExistingFilesForPaths` projection and filtering tombstones server-side; add `BenchmarkIndexerNoOpUpdateRepoWide`. (#61) - **indexer:** Overlap `ExistingFiles` load with the FS walk on repo-wide scans (workers gate on `existingReady`; tasks chan buffer bumped to `workerCount*64`). 2k-file fixture ~17% faster, 5k-file ~13%. (#65) - **store/indexer:** Slim per-row value on the existing-files hot path: new `ExistingFileMeta{SizeBytes, MtimeUnixNS, ContentHash}` replaces `FileRecord` as the map value across `ExistingFiles` / `ExistingFilesForPaths`; the unused `Path` (already the map key), `Language`, `ID`, and `IsDeleted` fields are gone. `processFileTask` takes `(prev ExistingFileMeta, hasPrev bool, …)` so the "exists" signal is explicit. Measured on `BenchmarkIndexerNoOpUpdateRepoWide` (2000 files, count=5 × benchtime=10x median): B/op 2319 KB → 1850 KB (~20% less); ns/op within noise; allocs/op slightly up. -- **store/indexer:** Defer `ContentHash` from the upfront existing-files map to a lazy per-path lookup. `ExistingFileMeta` further trimmed to `{SizeBytes, MtimeUnixNS}`; `ExistingFiles` / `ExistingFilesForPaths` SQL projection narrowed to `(path, size_bytes, mtime_unix_ns)`. New `Store.LookupFileContentHash(repoID, path)` is consulted only on a (size,mtime) mismatch via a `hashLookup func(rel) (string, bool)` callback threaded into `processFileTask`. The fast path of a no-op repo-wide update never allocates one hex string per file. New `BenchmarkIndexerNoOpUpdateRepoWide_FileScaling` sweeps 500/2000/5000 files. A/B at 2000 files (count=3 × benchtime=8x median): B/op 1854 KB → 1411 KB (~24% less); allocs/op 28,930 → 20,930 (~28% fewer); ns/op 23.0 → 22.7 ms (within noise — FS walk dominates). +- **store/indexer:** Defer `ContentHash` from the upfront existing-files map to a lazy per-path lookup. `ExistingFileMeta` further trimmed to `{SizeBytes, MtimeUnixNS}`; `ExistingFiles` / `ExistingFilesForPaths` SQL projection narrowed to `(path, size_bytes, mtime_unix_ns)`. New `Store.LookupFileContentHash(repoID, path)` is consulted only on a (size,mtime) mismatch via a `hashLookup func(rel) (string, bool, error)` callback threaded into `processFileTask`; lookup errors propagate via `result.err` rather than being collapsed into a hash miss. The fast path of a no-op repo-wide update never allocates one hex string per file. New `BenchmarkIndexerNoOpUpdateRepoWide_FileScaling` sweeps 500/2000/5000 files. A/B at 2000 files (count=3 × benchtime=8x median): B/op 1854 KB → 1411 KB (~24% less); allocs/op 28,930 → 20,930 (~28% fewer); ns/op 23.0 → 22.7 ms (within noise — FS walk dominates). - **store/resolver:** Schema-backed slash-suffix path via persisted `symbols.qualified_suffix` (migration 016) + partial index `idx_symbols_repo_qsuffix`; `resolveEdgesBySlashSuffix` now does an indexed equality JOIN instead of a Go-side hash filter over a full symbols scan. Large-scale (40k symbols) ~10% faster, ~11.3× fewer allocs. (#66) - **store/resolver:** Schema-backed dot-tail2 path via persisted `symbols.dot_tail2` (migration 017) + partial index `idx_symbols_repo_dot_tail2`; `resolveEdgesBySlashSuffix`'s dot-tail2 sub-branch replaced its full repo `SELECT id, qualified_name FROM symbols` scan + Go-side string slicing with the same indexed equality JOIN pattern as the slash branch. Measured on new `BenchmarkResolveEdgesBySlashSuffix_DotTail2LargeScale` (40k symbols, count=5 × benchtime=3x median): ns/op 93.8 → 80.8 ms (~14% faster), B/op 8.0 MB → 2.44 MB (~69% less), allocs/op 303k → 32.6k (~89% fewer). Symbol insert SQL bumped to 17 columns (batch row cap 60→58 to stay under SQLite's 999-variable default). - **store/resolver:** Schema-backed dot-suffix path for the dominant 2-dot dst_name case (e.g. `pkg.Sub.Func`) via persisted `symbols.dot_tail3` (migration 018) + partial index `idx_symbols_repo_dot_tail3`. New `resolveEdgesByDotTail3` runs as a schema-backed prelude inside `resolveEdgesByDotSuffix` (indexed equality JOIN against `dot_tail3` for dst_names with exactly 2 dots and no slash), leaving only ≥3-dot dst_names to the original `LIKE '%.' || dst_name` fallback. Measured on new `BenchmarkResolveEdgesByDotSuffixLargeScale` (40k symbols, count=3 × benchtime=2x median, A/B): ns/op 73,216,905,800 (73.2 s) → 74,127,650 (74 ms) — **~988× faster**. Symbol insert SQL bumped to 18 columns (batch row cap 58→55). From 2626542735aca3cfe5d6d3ab0c28ffc7d865565e Mon Sep 17 00:00:00 2001 From: isink17 <39876158+isink17@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:58:16 +0200 Subject: [PATCH 59/59] fix(cli,watcher,indexer,export): MR review fixups before merge --- internal/cli/app.go | 75 +++++++++++++++++++++++++++++++++---- internal/export/export.go | 12 ++---- internal/indexer/indexer.go | 6 ++- internal/watcher/watcher.go | 26 +++++++++---- 4 files changed, 93 insertions(+), 26 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index 6d606cd..93876d1 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "sort" "strconv" @@ -104,6 +105,26 @@ func hasHelpFlag(args []string) bool { return false } +// isJSONEmpty mirrors `encoding/json`'s `omitempty` rules so reflection-based +// envelope builders produce the same output shape as a `json.Marshal` round-trip. +func isJSONEmpty(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Pointer: + return v.IsNil() + } + return false +} + func parseOptionalRepoRootArg(fs *flag.FlagSet, args []string, repoRootFlag *string, defaultRepoRoot string) (string, error) { repoRootArg := "" parseArgs := args @@ -114,6 +135,9 @@ func parseOptionalRepoRootArg(fs *flag.FlagSet, args []string, repoRootFlag *str if err := fs.Parse(parseArgs); err != nil { return "", err } + if repoRootArg == "" && fs.NArg() > 0 { + repoRootArg = fs.Arg(0) + } repoRoot := strings.TrimSpace(*repoRootFlag) if repoRoot == "" { repoRoot = strings.TrimSpace(repoRootArg) @@ -652,16 +676,51 @@ func runIndex(ctx context.Context, cfg config.Config, stdout io.Writer, cmdName } } stripEnvelopeKeys := func(v any, keys ...string) (map[string]any, error) { - b, err := json.Marshal(v) - if err != nil { - return nil, err + skip := make(map[string]struct{}, len(keys)) + for _, k := range keys { + skip[k] = struct{}{} } - var out map[string]any - if err := json.Unmarshal(b, &out); err != nil { - return nil, err + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return map[string]any{}, nil + } + rv = rv.Elem() } - for _, k := range keys { - delete(out, k) + if rv.Kind() != reflect.Struct { + return nil, fmt.Errorf("stripEnvelopeKeys: expected struct, got %s", rv.Kind()) + } + rt := rv.Type() + out := make(map[string]any, rt.NumField()) + for i := 0; i < rt.NumField(); i++ { + sf := rt.Field(i) + if !sf.IsExported() { + continue + } + name := sf.Name + omitempty := false + if tag := sf.Tag.Get("json"); tag != "" { + parts := strings.Split(tag, ",") + if parts[0] == "-" { + continue + } + if parts[0] != "" { + name = parts[0] + } + for _, opt := range parts[1:] { + if opt == "omitempty" { + omitempty = true + } + } + } + if _, drop := skip[name]; drop { + continue + } + fv := rv.Field(i) + if omitempty && isJSONEmpty(fv) { + continue + } + out[name] = fv.Interface() } return out, nil } diff --git a/internal/export/export.go b/internal/export/export.go index 3c7643f..0f0e56b 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -2,7 +2,6 @@ package export import ( "bufio" - "bytes" "context" "encoding/json" "fmt" @@ -117,14 +116,9 @@ func (s *Service) JSONStream(ctx context.Context, w io.Writer, repoID int64, pag if e != nil { return e } - // JSONStream writes the first-line indentation context manually - // before each call. Strip a leading prefix from MarshalIndent's - // output so the first line is never double-indented — defensive - // against the encoder's contract changing or a future swap to one - // that does prefix the first line. - if len(prefix) > 0 && bytes.HasPrefix(b, []byte(prefix)) { - b = b[len(prefix):] - } + // `encoding/json.MarshalIndent` does not emit `prefix` before the + // first line; JSONStream writes the call-site indentation manually + // before each call, so we can write the bytes through unchanged. _, e = bw.Write(b) return e } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index b01170b..a81c60e 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -184,7 +184,11 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err results := make(chan fileResult, workerCount*2) producerErr := make(chan error, 1) walkStart := time.Now() - missingCandidatePaths := make([]string, 0, 16) + missingCap := len(candidatePaths) + if missingCap > 128 { + missingCap = 128 + } + missingCandidatePaths := make([]string, 0, missingCap) go func() { defer close(tasks) diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index e054046..062af46 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -148,7 +148,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun eventsCh := fsw.Events errorsCh := fsw.Errors - addWatchTree := func(root string) error { + addWatchTreeTo := func(target *fsnotify.Watcher, root string) error { return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -163,7 +163,7 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun if filepath.Base(rel) == config.RepoArtifactsDir { // Keep watching repo-local config updates even though artifacts are // never indexable. - if addErr := fsw.Add(path); addErr != nil { + if addErr := target.Add(path); addErr != nil { return addErr } } @@ -171,24 +171,34 @@ func (w *Watcher) Run(ctx context.Context, repoRoot string, repoID int64, deboun } } if d.IsDir() { - return fsw.Add(path) + return target.Add(path) } return nil }) } + addWatchTree := func(root string) error { return addWatchTreeTo(fsw, root) } resetWatcher := func() error { // Rebuild the watcher so include/exclude changes take effect (especially - // for directories that were previously skipped and not watched). - _ = fsw.Close() + // for directories that were previously skipped and not watched). Build + // and fully populate the new watcher before swapping it in and closing + // the old one, so events on already-watched paths keep flowing through + // the old watcher during the rebuild instead of falling into a window + // where neither watcher is active. next, err := fsnotify.NewWatcher() if err != nil { return err } + if err := addWatchTreeTo(next, repoRoot); err != nil { + _ = next.Close() + return err + } + old := fsw fsw = next - eventsCh = fsw.Events - errorsCh = fsw.Errors - return addWatchTree(repoRoot) + eventsCh = next.Events + errorsCh = next.Errors + _ = old.Close() + return nil } if err := addWatchTree(repoRoot); err != nil {