From 1e86855f269f84d8ece70f09b63f328089347fca Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Fri, 4 Sep 2026 20:11:28 -0600 Subject: [PATCH 1/2] feat(explore): markdown index (#361) with a section-first doc tier Ports QingNagi/codegraph#361 (markdown extractor, heading nodes, name-matcher and resolution hooks) onto experimental and adds the section-first doc tier from feature/md-section-first: a doc-shaped query renders the best headed sections of the markdown file it names, ranked by idf-weighted line hits with path tokens weighted zero, capped at DOC_FILE_CAP per file. Markdown reaches an answer only through that tier, generated-file detection ignores markdown bodies, and the budget tiers count code files only so a README-heavy repo keeps its code answers. --- __tests__/extraction.test.ts | 213 ++++ __tests__/integration/full-pipeline.test.ts | 133 +++ __tests__/resolution.test.ts | 163 +++ __tests__/security.test.ts | 3 +- __tests__/watcher.test.ts | 4 +- src/extraction/generated-detection.ts | 4 + src/extraction/grammars.ts | 11 +- src/extraction/markdown-extractor.ts | 1003 +++++++++++++++++++ src/extraction/tree-sitter.ts | 185 +++- src/mcp/server-instructions.ts | 2 +- src/mcp/tools.ts | 299 +++++- src/resolution/index.ts | 13 +- src/resolution/name-matcher.ts | 132 ++- src/types.ts | 1 + 14 files changed, 2133 insertions(+), 33 deletions(-) create mode 100644 src/extraction/markdown-extractor.ts diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index ad0ba2374..597a26dcb 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -102,6 +102,12 @@ describe('Language Detection', () => { expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c'); }); + it('should detect Markdown files', () => { + expect(detectLanguage('README.md')).toBe('markdown'); + expect(detectLanguage('docs/guide.markdown')).toBe('markdown'); + expect(detectLanguage('docs/page.mdx')).toBe('markdown'); + }); + it('should detect Metal shader files as C++ (#1121)', () => { expect(detectLanguage('Shaders.metal')).toBe('cpp'); expect(isSourceFile('Renderer/Shaders.metal')).toBe(true); @@ -250,11 +256,218 @@ describe('Language Support', () => { expect(languages).toContain('swift'); expect(languages).toContain('kotlin'); expect(languages).toContain('dart'); + expect(languages).toContain('markdown'); expect(languages).toContain('solidity'); expect(languages).toContain('nix'); }); }); +describe('Markdown Extraction', () => { + it('should extract headings, links, and shell script references', () => { + const markdown = `# Project Guide + +See [Setup](docs/setup.md#install) and scripts/release.mjs. + +## Release + +\`\`\`bash +npm run build +node scripts/release.mjs +\`\`\` +`; + + const result = extractFromSource('README.md', markdown); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toMatchObject({ + name: 'README.md', + language: 'markdown', + }); + + const headings = result.nodes.filter((n) => n.kind === 'module'); + expect(headings.map((n) => n.name)).toContain('Project Guide'); + expect(headings.map((n) => n.name)).toContain('Release'); + + const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs'); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'docs/setup.md#install', + referenceKind: 'imports', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'scripts/release.mjs', + referenceKind: 'calls', + language: 'markdown', + }), + ]) + ); + }); + + it('should extract structured table rows and file-symbol references from Markdown', () => { + const markdown = `# Maintenance Guide + +## Phase 4 + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | + +- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('phases/phase4.md', markdown); + + const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row')); + expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2'])); + + const p4s1 = tableRows.find((n) => n.name === 'P4-S1'); + expect(p4s1?.signature).toContain('Template: P4-S1'); + expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4'); + + const commandNode = result.nodes.find((n) => + n.kind === 'function' && + n.language === 'markdown' && + n.signature?.includes('python "{script_path}" p4') + ); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::_p4_stage1', + referenceKind: 'references', + language: 'markdown', + }), + ]) + ); + }); + + it('should keep structured blocks after fences containing a different fence marker', () => { + const markdown = `# Runbook + +\`\`\`text +~~~~ +\`\`\` + +- POST-FENCE references \`src/auth.ts::login\`. + +| Key | Target | +| --- | --- | +| POST-TABLE | \`src/auth.ts::login\` | +`; + + const result = extractFromSource('docs/runbook.md', markdown); + const constants = result.nodes.filter((n) => n.kind === 'constant'); + + expect(constants).toEqual(expect.arrayContaining([ + expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }), + expect.objectContaining({ name: 'POST-TABLE' }), + ])); + }); + + it('indexes Setext (underline) headings and skips frontmatter / code fences', () => { + const markdown = `--- +title: Config Doc +--- + +Architecture Overview +===================== + +Intro paragraph for the overview. + +Routing Layer +------------- + +\`\`\`md +Not A Heading +============= +\`\`\` +`; + + const result = extractFromSource('docs/arch.md', markdown); + const headings = result.nodes.filter((n) => n.kind === 'module'); + const byName = new Map(headings.map((h) => [h.name, h])); + + // Setext H1 (===) and H2 (---) become module nodes. + expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview'); + expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer'); + // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a + // setext-looking line inside a code fence is ignored. + expect(byName.has('title: Config Doc')).toBe(false); + expect(byName.has('Not A Heading')).toBe(false); + }); + + it('builds a deterministic, compact file digest (intro + key references)', () => { + const markdown = `# Release Runbook + +This runbook explains how to cut a release. + +See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`. +It dispatches \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('RUNBOOK.md', markdown); + const fileNode = result.nodes.find((n) => n.kind === 'file'); + + expect(fileNode?.docstring).toBeDefined(); + const digest = fileNode!.docstring!; + // Intro is the first prose line, not the heading or a link blob. + expect(digest).toContain('This runbook explains how to cut a release.'); + // Key referenced files/symbols are surfaced, compacted to basenames. + expect(digest).toContain('refs:'); + expect(digest).toContain('setup.md#install'); + expect(digest).toContain('release.mjs'); + expect(digest).toContain('csv_search.py::run_p4'); + // Short enough to show in node details (the < 200 char detail gate). + expect(digest.length).toBeLessThan(200); + }); +}); + +describe('Code to Markdown Reference Extraction', () => { + it('should extract Markdown path references from code string literals', () => { + const code = ` +export const GUIDE = '../docs/guide.md'; + +export function loadDocs() { + return fs.readFileSync('../docs/guide.md#install', 'utf8'); +} +`; + + const result = extractFromSource('src/load-docs.ts', code); + const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs'); + const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE'); + + expect(loadDocs).toBeDefined(); + expect(guideConstant).toBeDefined(); + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fromNodeId: loadDocs!.id, + referenceName: 'docs/guide.md#install', + referenceKind: 'references', + language: 'typescript', + }), + expect.objectContaining({ + fromNodeId: guideConstant!.id, + referenceName: 'docs/guide.md', + referenceKind: 'references', + language: 'typescript', + }), + ]) + ); + }); +}); + describe('Nix Extraction', () => { it('should distinguish Nix variable and function bindings', () => { const code = ` diff --git a/__tests__/integration/full-pipeline.test.ts b/__tests__/integration/full-pipeline.test.ts index 5b551c136..fc3aab2e7 100644 --- a/__tests__/integration/full-pipeline.test.ts +++ b/__tests__/integration/full-pipeline.test.ts @@ -82,6 +82,97 @@ describe('Integration: full pipeline', () => { cleanupTempDir(tempDir); }); + it('indexes Markdown headings and resolves Markdown links to script files', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'README.md'), + `# Project Guide + +See [Setup](docs/setup.md#install). + +## Release + +\`\`\`bash +node scripts/release.mjs +\`\`\` +` + ); + fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n'); + fs.writeFileSync(path.join(tempDir, 'scripts', 'release.mjs'), 'export function release() { return true; }\n'); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const guide = cg.searchNodes('Project Guide').find((r) => r.node.language === 'markdown'); + expect(guide).toBeDefined(); + + const releaseCommand = cg + .searchNodes('release.mjs') + .find((r) => r.node.language === 'markdown' && r.node.kind === 'function'); + expect(releaseCommand).toBeDefined(); + + const guideEdges = cg.getOutgoingEdges(guide!.node.id).filter((e) => e.kind === 'imports'); + const guideTargets = guideEdges.map((e) => cg.getNode(e.target)); + const setupHeading = guideTargets.find((n) => n?.qualifiedName === 'docs/setup.md#install'); + expect(setupHeading).toMatchObject({ + kind: 'module', + name: 'Install', + filePath: 'docs/setup.md', + startLine: 1, + }); + + const commandEdges = cg.getOutgoingEdges(releaseCommand!.node.id).filter((e) => e.kind === 'calls'); + const commandTargets = commandEdges.map((e) => cg.getNode(e.target)?.filePath); + expect(commandTargets).toContain('scripts/release.mjs'); + } finally { + cg.destroy(); + } + }); + + it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => { + fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'phases', 'phase4.md'), + `# Phase 4 + +## Fixed Script Templates + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'csv_search.py'), + `def _p4_stage1(filepath, condition_spec, probe_cols_spec):\n return 's1'\n\n` + + `def _p4_stage2(filepath, stage1_rows, condition_spec, detail_cols_spec):\n return 's2'\n\n` + + `def run_p4(filepath, args):\n return _p4_stage1(filepath, '-', 'MPN')\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const p4s1Row = cg.searchNodes('P4-S1').find((r) => r.node.language === 'markdown'); + expect(p4s1Row?.node.kind).toBe('constant'); + + const edges = cg.getOutgoingEdges(p4s1Row!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'run_p4', filePath: 'scripts/csv_search.py' }), + expect.objectContaining({ name: '_p4_stage1', filePath: 'scripts/csv_search.py' }), + ]) + ); + } finally { + cg.destroy(); + } + }); + it('runs init → index → resolve → search → callers → context → sync', async () => { const MODULE_COUNT = 120; generateSyntheticProject(tempDir, MODULE_COUNT); @@ -269,4 +360,46 @@ describe('Integration: full pipeline', () => { cg.destroy(); } }, 30_000); + + it('resolves code string references to Markdown headings', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'docs', 'guide.md'), + `# Guide + +## Install + +Run the setup command. +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'load_docs.py'), + `GUIDE = "docs/guide.md"\n\n` + + `def load_docs():\n` + + ` return open("docs/guide.md#install", encoding="utf-8").read()\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const loadDocs = cg.searchNodes('load_docs').find((r) => r.node.language === 'python'); + expect(loadDocs).toBeDefined(); + + const edges = cg.getOutgoingEdges(loadDocs!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'module', + name: 'Install', + qualifiedName: 'docs/guide.md#install', + }), + ]) + ); + } finally { + cg.destroy(); + } + }); }); diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index decaadee5..794c8a1a0 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -358,6 +358,169 @@ describe('Resolution Module', () => { expect(result).not.toBeNull(); expect(result?.targetNodeId).toBe('method:user.ts:User.save:15'); }); + + it('should resolve Markdown file references by filename, path, and anchor suffix', () => { + const mockNodes: Node[] = [ + { + id: 'file:README.md', + kind: 'file', + name: 'README.md', + qualifiedName: 'README.md', + filePath: 'README.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'file:docs/setup.md', + kind: 'file', + name: 'setup.md', + qualifiedName: 'docs/setup.md', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:docs/setup.md:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'docs/setup.md#install', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + { + id: 'file:GUIDE.markdown', + kind: 'file', + name: 'GUIDE.markdown', + qualifiedName: 'GUIDE.markdown', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:GUIDE.markdown:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'GUIDE.markdown#install', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: () => mockNodes, + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['README.md', 'docs/setup.md', 'GUIDE.markdown'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const readmeRef = { + fromNodeId: 'module:docs/setup.md:install:1', + referenceName: 'README.md', + referenceKind: 'imports' as const, + line: 1, + column: 0, + filePath: 'docs/setup.md', + language: 'markdown' as const, + }; + const setupRef = { + ...readmeRef, + referenceName: 'docs/setup.md#install', + filePath: 'README.md', + }; + const markdownRef = { + ...readmeRef, + referenceName: 'GUIDE.markdown#install', + filePath: 'README.md', + }; + + expect(matchReference(readmeRef, context)?.targetNodeId).toBe('file:README.md'); + expect(matchReference(setupRef, context)?.targetNodeId).toBe('module:docs/setup.md:install:1'); + expect(matchReference(markdownRef, context)?.targetNodeId).toBe('module:GUIDE.markdown:install:1'); + }); + + it('should resolve Markdown file-symbol references to symbols in the referenced file', () => { + const mockNodes: Node[] = [ + { + id: 'file:scripts/csv_search.py', + kind: 'file', + name: 'csv_search.py', + qualifiedName: 'scripts/csv_search.py', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 1, + endLine: 100, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'function:scripts/csv_search.py:run_p4:40', + kind: 'function', + name: 'run_p4', + qualifiedName: 'scripts/csv_search.py::run_p4', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 40, + endLine: 55, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: (filePath) => mockNodes.filter((n) => n.filePath === filePath), + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['scripts/csv_search.py'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const ref = { + fromNodeId: 'constant:phases/phase4.md:P4-S1:10', + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references' as const, + line: 10, + column: 20, + filePath: 'phases/phase4.md', + language: 'markdown' as const, + }; + + expect(matchReference(ref, context)?.targetNodeId).toBe('function:scripts/csv_search.py:run_p4:40'); + }); }); describe('Ubiquitous-name ceiling (#999)', () => { diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 9aa3c9597..ccda3671b 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -513,11 +513,12 @@ describe('Source file detection (isSourceFile)', () => { expect(isSourceFile('src/component.tsx')).toBe(true); expect(isSourceFile('lib/util.js')).toBe(true); expect(isSourceFile('src/main.py')).toBe(true); + // Markdown documentation is indexed as a source language. + expect(isSourceFile('README.md')).toBe(true); }); it('rejects unsupported extensions and extensionless files', () => { expect(isSourceFile('src/component.css')).toBe(false); - expect(isSourceFile('README.md')).toBe(false); expect(isSourceFile('Makefile')).toBe(false); expect(isSourceFile('.gitignore')).toBe(false); }); diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index 942fd5bcd..067f70f22 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -442,8 +442,8 @@ describe('FileWatcher', () => { // gate must drop it before scheduling sync. (It must exist on disk: // a VANISHED non-source path is the deleted-directory shape, which // deliberately schedules a sync — #1285.) - fs.writeFileSync(path.join(testDir, 'src', 'readme.md'), '# docs\n'); - __emitWatchEventForTests(testDir, 'src/readme.md'); + fs.writeFileSync(path.join(testDir, 'src', 'styles.css'), 'body {}\n'); + __emitWatchEventForTests(testDir, 'src/styles.css'); // Wait a bit longer than debounce — sync should NOT trigger. await new Promise((r) => setTimeout(r, 400)); diff --git a/src/extraction/generated-detection.ts b/src/extraction/generated-detection.ts index b92ab860b..03116cfb5 100644 --- a/src/extraction/generated-detection.ts +++ b/src/extraction/generated-detection.ts @@ -249,5 +249,9 @@ export function hasGeneratedHeader(content: string): boolean { * indexer persists to `files.generated`. */ export function detectGeneratedFile(filePath: string, content: string): boolean { + // Markdown carries no banner: its `#` is a heading, not a comment leader, + // so a README that DESCRIBES generated code ("Code generated … DO NOT EDIT" + // under a heading) would flag itself. The path convention still applies. + if (/\.(?:md|markdown)$/i.test(filePath)) return isGeneratedFile(filePath); return isGeneratedFile(filePath) || hasGeneratedHeader(content); } diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index c7710f200..37ad0e0a4 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file @@ -100,6 +100,9 @@ export const EXTENSION_MAP: Record = { '.yaml': 'yaml', // Twig templates (file-level tracking only, no symbol extraction) '.twig': 'twig', + '.md': 'markdown', + '.mdx': 'markdown', + '.markdown': 'markdown', '.rb': 'ruby', '.rake': 'ruby', '.swift': 'swift', @@ -588,6 +591,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'markdown') return true; // custom documentation extractor if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -596,7 +600,7 @@ export function isLanguageSupported(language: Language): boolean { * Check if a grammar has been loaded and is ready for parsing. */ export function isGrammarLoaded(language: Language): boolean { - if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; + if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor' || language === 'markdown') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed return languageCache.has(language); @@ -619,7 +623,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'markdown']; } /** @@ -699,6 +703,7 @@ export function getLanguageDisplayName(language: Language): string { twig: 'Twig', xml: 'XML', properties: 'Java properties', + markdown: 'Markdown', cfml: 'CFML', cfscript: 'CFScript', cfquery: 'CFQuery (SQL)', diff --git a/src/extraction/markdown-extractor.ts b/src/extraction/markdown-extractor.ts new file mode 100644 index 000000000..4695ea4be --- /dev/null +++ b/src/extraction/markdown-extractor.ts @@ -0,0 +1,1003 @@ +import * as path from 'path'; +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, EdgeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +interface HeadingInfo { + id: string; + level: number; + title: string; + slug: string; + line: number; + endLine: number; + node: Node; +} + +interface FenceState { + marker: string; + language: string; + startLine: number; + lines: Array<{ text: string; line: number }>; +} + +/** + * Lightweight Markdown extractor. + * + * Markdown is indexed as documentation structure rather than code syntax: + * headings become searchable module nodes, links become import nodes, and + * shell-like fenced blocks create command nodes plus file-path references. + */ +export class MarkdownExtractor { + private filePath: string; + private lines: string[]; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private headings: HeadingInfo[] = []; + private referenceKeys = new Set(); + + constructor(filePath: string, source: string) { + this.filePath = normalizeRelativePath(filePath); + this.lines = source.split('\n'); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + + try { + const fileNode = this.createFileNode(); + this.extractHeadings(fileNode); + this.extractStructuredBlocks(fileNode); + this.extractLinksAndCommands(fileNode); + this.finalizeFileDigest(fileNode); + } catch (error) { + this.errors.push({ + message: `Markdown extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + private createFileNode(): Node { + const id = generateNodeId(this.filePath, 'file', this.filePath, 1); + const fileNode: Node = { + id, + kind: 'file', + name: path.posix.basename(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'markdown', + startLine: 1, + endLine: Math.max(1, this.lines.length), + startColumn: 0, + endColumn: this.lines[this.lines.length - 1]?.length || 0, + // docstring is filled in by finalizeFileDigest() once headings and + // references are known, so it becomes a triage-friendly digest. + docstring: undefined, + updatedAt: Date.now(), + }; + + this.nodes.push(fileNode); + return fileNode; + } + + private extractHeadings(fileNode: Node): void { + const rawHeadings: Array<{ level: number; title: string; line: number; column: number }> = []; + const frontmatterEnd = this.frontmatterEndIndex(); + let fenceMarker: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + if (i <= frontmatterEnd) continue; + const line = this.lines[i]!; + + // Track fenced code blocks so `# foo` comments and `===`/`---` lines + // inside a code sample are never mistaken for document headings. + const fence = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fence) { + const marker = fence[2]![0]!.repeat(fence[2]!.length); + if (fenceMarker === null) fenceMarker = marker; + else if (line.trimStart().startsWith(fenceMarker)) fenceMarker = null; + continue; + } + if (fenceMarker !== null) continue; + + // ATX heading: `## Title` + const atx = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); + if (atx) { + rawHeadings.push({ + level: atx[1]!.length, + title: stripInlineMarkdown(atx[2]!.trim()), + line: i + 1, + column: line.indexOf('#'), + }); + continue; + } + + // Setext heading: a paragraph line underlined by `===` (level 1) or + // `---` (level 2). CommonMark treats a `---` directly under a paragraph + // as a heading, not a thematic break — and the frontmatter skip above + // keeps a closing `---` from turning its last key into a heading. + const underline = /^(=+|-+)\s*$/.exec(line.trim()); + if (underline && i - 1 > frontmatterEnd && isSetextTextLine(this.lines[i - 1] ?? '')) { + rawHeadings.push({ + level: underline[1]![0] === '=' ? 1 : 2, + title: stripInlineMarkdown((this.lines[i - 1] ?? '').trim()), + line: i, // the text line carries the heading + column: 0, + }); + } + } + + const slugCounts = new Map(); + for (let i = 0; i < rawHeadings.length; i++) { + const heading = rawHeadings[i]!; + const baseSlug = slugifyHeading(heading.title); + const seen = slugCounts.get(baseSlug) ?? 0; + slugCounts.set(baseSlug, seen + 1); + const slug = seen === 0 ? baseSlug : `${baseSlug}-${seen}`; + + let endLine = this.lines.length; + for (let j = i + 1; j < rawHeadings.length; j++) { + if (rawHeadings[j]!.level <= heading.level) { + endLine = rawHeadings[j]!.line - 1; + break; + } + } + + const nodeId = generateNodeId(this.filePath, 'module', `${slug}:${heading.line}`, heading.line); + const node: Node = { + id: nodeId, + kind: 'module', + name: heading.title, + qualifiedName: `${this.filePath}#${slug}`, + filePath: this.filePath, + language: 'markdown', + signature: `${'#'.repeat(heading.level)} ${heading.title}`, + docstring: this.buildDocstring(heading.line + 1, endLine), + startLine: heading.line, + endLine, + startColumn: heading.column, + endColumn: this.lines[heading.line - 1]?.length || 0, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.headings.push({ + id: nodeId, + level: heading.level, + title: heading.title, + slug, + line: heading.line, + endLine, + node, + }); + } + + const stack: HeadingInfo[] = []; + for (const heading of this.headings) { + while (stack.length > 0 && stack[stack.length - 1]!.level >= heading.level) { + stack.pop(); + } + const parent = stack[stack.length - 1]; + this.edges.push({ + source: parent?.id ?? fileNode.id, + target: heading.id, + kind: 'contains', + provenance: 'heuristic', + }); + stack.push(heading); + } + } + + private extractLinksAndCommands(fileNode: Node): void { + let fence: FenceState | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const lineNumber = i + 1; + const fenceMatch = /^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+.-]*)/.exec(line); + + if (fence) { + if (line.trimStart().startsWith(fence.marker)) { + this.extractCommandsFromFence(fence, fileNode); + fence = null; + } else { + fence.lines.push({ text: line, line: lineNumber }); + } + continue; + } + + if (fenceMatch) { + fence = { + marker: fenceMatch[2]![0]!.repeat(fenceMatch[2]!.length), + language: (fenceMatch[3] ?? '').toLowerCase(), + startLine: lineNumber, + lines: [], + }; + continue; + } + + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + this.extractMarkdownLinks(line, lineNumber, owner); + if (!isTableRowLine(line)) { + this.extractFileSymbolMentions(line, lineNumber, owner); + this.extractPathMentions(line, lineNumber, owner); + } + } + + if (fence) { + this.extractCommandsFromFence(fence, fileNode); + } + } + + private extractStructuredBlocks(fileNode: Node): void { + this.extractTables(fileNode); + this.extractListItems(fileNode); + } + + private extractTables(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length - 1; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const separator = this.lines[i + 1]!; + if (!isTableHeaderLine(line) || !isTableSeparatorLine(separator)) continue; + + const headers = parseTableCells(line); + if (headers.length < 2) continue; + + let rowIndex = i + 2; + while (rowIndex < this.lines.length && isTableRowLine(this.lines[rowIndex]!)) { + const rowLine = this.lines[rowIndex]!; + const cells = parseTableCells(rowLine); + const lineNumber = rowIndex + 1; + if (shouldIndexTableRow(headers, cells)) { + this.createTableRowNode(headers, cells, rowLine, lineNumber, fileNode); + } + rowIndex++; + } + + i = rowIndex - 1; + } + } + + private extractListItems(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const match = /^\s*[-*+]\s+(?:\[[ xX]\]\s+)?(.+?)\s*$/.exec(line); + if (!match) continue; + + const text = stripInlineMarkdown(match[1]!.trim()); + if (!shouldIndexListItem(text)) continue; + + const lineNumber = i + 1; + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const name = stableIdentifier(text) ?? truncateForName(text, 80); + const nodeId = generateNodeId(this.filePath, 'constant', `list:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::list-item:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature: `list item: ${truncateForSignature(text, 180)}`, + docstring: text, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, line.indexOf(match[1]!)), + endColumn: line.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.extractStructuredReferences(text, lineNumber, node); + } + } + + private createTableRowNode( + headers: string[], + cells: string[], + rowLine: string, + lineNumber: number, + fileNode: Node + ): void { + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const rowText = cells.join(' | '); + const name = stableIdentifier(rowText) ?? truncateForName(firstNonEmptyCell(cells) || `row ${lineNumber}`, 80); + const signature = summarizeTableRow(headers, cells); + const nodeId = generateNodeId(this.filePath, 'constant', `table:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::table-row:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature, + docstring: signature, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, rowLine.indexOf(firstNonEmptyCell(cells) || '|')), + endColumn: rowLine.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + this.extractStructuredReferences(rowText, lineNumber, node); + this.extractCommandsFromStructuredCells(cells, rowLine, lineNumber, node); + } + + private extractCommandsFromStructuredCells(cells: string[], rowLine: string, lineNumber: number, owner: Node): void { + for (const cell of cells) { + const candidates = extractCodeSpans(cell); + if (candidates.length === 0) candidates.push(stripInlineMarkdown(cell)); + + for (const candidate of candidates) { + const command = extractCommandFromText(candidate); + if (!command) continue; + const column = Math.max(0, rowLine.indexOf(candidate)); + this.createCommandNode(command, lineNumber, column, owner); + } + } + } + + private createCommandNode(command: string, line: number, column: number, owner: Node): void { + const nodeId = generateNodeId(this.filePath, 'function', `command:${line}:${column}:${command}`, line); + const node: Node = { + id: nodeId, + kind: 'function', + name: commandName(command), + qualifiedName: `${this.filePath}::command:${line}:${column}:${command}`, + filePath: this.filePath, + language: 'markdown', + signature: command, + startLine: line, + endLine: line, + startColumn: column, + endColumn: column + command.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + const scriptPath = extractScriptPath(command); + if (scriptPath) { + const normalizedTarget = this.normalizeReference(scriptPath); + if (normalizedTarget) { + this.addReference(nodeId, normalizedTarget, 'calls', line, column); + } + } + } + + private extractMarkdownLinks(line: string, lineNumber: number, owner: Node): void { + const linkRegex = /!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g; + let match: RegExpExecArray | null; + + while ((match = linkRegex.exec(line)) !== null) { + const target = match[2]!; + if (!isLocalReference(target)) continue; + + const normalizedTarget = this.normalizeReference(target); + if (!normalizedTarget) continue; + + const column = match.index; + const label = stripInlineMarkdown(match[1] || target) || target; + const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: displayName, + qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, + filePath: this.filePath, + language: 'markdown', + signature: match[0], + docstring: label, + startLine: lineNumber, + endLine: lineNumber, + startColumn: column, + endColumn: column + match[0].length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.addReference(owner.id, normalizedTarget, 'imports', lineNumber, column); + } + } + + private extractPathMentions(line: string, lineNumber: number, owner: Node): void { + const pathRegex = /(?= best.level) best = heading; + } + } + return best?.node ?? null; + } + + private addReference( + fromNodeId: string, + referenceName: string, + referenceKind: EdgeKind, + line: number, + column: number + ): void { + const key = `${fromNodeId}:${referenceKind}:${referenceName}:${line}:${column}`; + if (this.referenceKeys.has(key)) return; + this.referenceKeys.add(key); + this.unresolvedReferences.push({ + fromNodeId, + referenceName, + referenceKind, + line, + column, + filePath: this.filePath, + language: 'markdown', + }); + } + + private normalizeReference(target: string): string | null { + const [pathAndSymbol, anchorPart] = splitAnchor(target.trim()); + const [pathPart, symbolPart] = splitFileSymbol(pathAndSymbol); + const cleanPath = decodePath(pathPart.split(/[?#]/)[0] ?? ''); + const anchor = anchorPart ? `#${slugifyHeading(anchorPart)}` : ''; + + if (!cleanPath && anchor) { + return `${this.filePath}${anchor}`; + } + if (!cleanPath) return null; + + const withoutLeadingSlash = cleanPath.startsWith('/') ? cleanPath.slice(1) : cleanPath; + const baseDir = path.posix.dirname(this.filePath); + const normalized = cleanPath.startsWith('/') || baseDir === '.' + ? path.posix.normalize(withoutLeadingSlash) + : path.posix.normalize(path.posix.join(baseDir, withoutLeadingSlash)); + + if (normalized.startsWith('../') || normalized === '..') return null; + return `${normalized}${symbolPart ? `::${symbolPart}` : ''}${anchor}`; + } + + private buildDocstring(startLine: number, endLine: number): string | undefined { + const text = this.lines + .slice(Math.max(0, startLine - 1), Math.max(0, endLine)) + .map((line) => stripInlineMarkdown(line.replace(/^#{1,6}\s+/, '').trim())) + .filter((line) => line && !/^(```|~~~)/.test(line)) + .join('\n') + .trim(); + + if (!text) return undefined; + return text.length > 600 ? `${text.slice(0, 600)}...` : text; + } + + /** + * Index of the closing `---` of a leading YAML frontmatter block, or -1 when + * the file has none. Lines at or before this index are metadata, not body — + * so they never produce headings and never seed the intro/digest. + */ + private frontmatterEndIndex(): number { + if ((this.lines[0] ?? '').trim() !== '---') return -1; + for (let j = 1; j < this.lines.length; j++) { + if (this.lines[j]!.trim() === '---') return j; + } + return -1; + } + + /** + * Replace the file node's docstring with a deterministic digest — a one-line + * "what is this doc about" intro plus the key files/symbols it references. + * It is derived purely from the already-extracted structure (no LLM), kept + * short enough to surface in node details, and — because docstrings are in + * the FTS index — makes a doc discoverable by the symbols it documents even + * when the query matches no heading or filename. See the reviewer thread on + * PR #361. + */ + private finalizeFileDigest(fileNode: Node): void { + const intro = this.buildIntro(); + const refs = this.collectKeyReferences(); + const parts: string[] = []; + if (intro) parts.push(intro); + if (refs.length > 0) parts.push(`refs: ${refs.join(', ')}`); + const digest = parts.join(' · ').trim(); + fileNode.docstring = digest || this.buildDocstring(1, Math.min(this.lines.length, 40)); + } + + /** + * First real prose line of the document — the closest deterministic stand-in + * for "what this file is about". Skips frontmatter, headings, fenced code, + * tables, badge/image-only lines, and raw HTML. + */ + private buildIntro(): string | null { + let fenceMarker: string | null = null; + for (let i = this.frontmatterEndIndex() + 1; i < this.lines.length; i++) { + const trimmed = this.lines[i]!.trim(); + const fence = /^(`{3,}|~{3,})/.exec(trimmed); + if (fence) { + const marker = fence[1]![0]!.repeat(fence[1]!.length); + fenceMarker = fenceMarker === null ? marker : (trimmed.startsWith(fenceMarker) ? null : fenceMarker); + continue; + } + if (fenceMarker !== null) continue; + if (!trimmed) continue; + if (/^#{1,6}\s/.test(trimmed)) continue; // ATX heading + if (/^(=+|-+|\*{3,}|_{3,})\s*$/.test(trimmed)) continue; // setext underline / hr + if (trimmed.startsWith('|')) continue; // table + if (/^!\[/.test(trimmed)) continue; // image / badge line + if (/^