diff --git a/CHANGELOG.md b/CHANGELOG.md index 1815c4150..a0f5f7008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- **Markdown is indexed, and a documentation question gets the section, not the graph.** Every `.md` file's headings, sections, tables and links are nodes (the extractor from #361), and a doc-shaped `codegraph_explore` query that names a markdown file now renders that file's best sections first and whole — the top three by idf-weighted line hits, a heading the query covers word for word counted as named, 8k characters per file — with the blast-radius, relationships and "additional files" blocks held back unless a code file rendered too. Measured on a 109-file docs corpus under headless Claude Code, 36 cells over three rounds: the right file and section in every call, median 1 tool call against 4 for Grep-then-Read, 36 of 36 correct. Code answers keep their shape: markdown nodes leave a subgraph the doc tier did not seed, a markdown body is never mistaken for a generated-file header, and the explore budget tiers count code files only, so a README-heavy repo does not cross a breakpoint. The server instructions say markdown is indexed, which the branch's own text still denied. (#361, #1439) - **A busy screen's picture is laid out by the parts of the screen.** A screen is a set of handlers with no order between them, so on a hub screen the old rows-by-distance collapsed into one enormous row — the main screen of one app put 89 boxes side by side on a canvas over 28,000px wide, every line a near-horizontal sweep across all of it. The Steps tab now groups a screen's picture by region — the component that owns each handler, named in a small caption over its boxes — with each region a column where a step sits above what it sets in motion, tiled in the screen's own source order. At rest the picture hides only two things: the screen's own fan-out — one line into each region stands in for it — and lines that point back up; every other line draws where it leads, between two regions included, and selecting a step brings out its whole story in the side panel, link by link. A box nothing points at is the screen's own doing — run on render or mount, or from a binding written inline — the key says so, and selecting it lights its line from the screen with what fires it. The same app's widest screen now lays out under 3,500px with every line local, and the whole picture fits on screen when it opens. Endpoints, handlers and the in-order reading are untouched, and nothing needs a re-index: the regions come from the same walk that draws the steps. - **Where the code chooses, the picture says so once.** A helper that ends `return (await hasSeenWelcome(id)) ? '/home/' : '/welcome/'` sends the app to one of two screens, but the Steps picture drew that as two separate arrows, each carrying the whole condition with one of them negated and both cut off at the same forty characters — and before you clicked anything, neither arrow was labelled at all, so nothing said it was a choice. Now sibling arrows out of one box that are the arms of one `if`, `switch` or ternary are drawn as the choice they are: the condition is written once under the box that decides it, and each arrow out says only which way it is — `yes`, `no`, or a case's own value. They are the only arrows labelled before you select anything, so the picture reads at a glance without becoming a wall of text. A one-sided guard — an early exit, an `if` with only one side drawn — still carries its condition on the arrow, and an arrow that is reached whether or not the condition holds never claims a side. Nothing needs a re-index: the decision is read from the source at request time. diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index ad0ba2374..597a26dcb 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -102,6 +102,12 @@ describe('Language Detection', () => { expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c'); }); + it('should detect Markdown files', () => { + expect(detectLanguage('README.md')).toBe('markdown'); + expect(detectLanguage('docs/guide.markdown')).toBe('markdown'); + expect(detectLanguage('docs/page.mdx')).toBe('markdown'); + }); + it('should detect Metal shader files as C++ (#1121)', () => { expect(detectLanguage('Shaders.metal')).toBe('cpp'); expect(isSourceFile('Renderer/Shaders.metal')).toBe(true); @@ -250,11 +256,218 @@ describe('Language Support', () => { expect(languages).toContain('swift'); expect(languages).toContain('kotlin'); expect(languages).toContain('dart'); + expect(languages).toContain('markdown'); expect(languages).toContain('solidity'); expect(languages).toContain('nix'); }); }); +describe('Markdown Extraction', () => { + it('should extract headings, links, and shell script references', () => { + const markdown = `# Project Guide + +See [Setup](docs/setup.md#install) and scripts/release.mjs. + +## Release + +\`\`\`bash +npm run build +node scripts/release.mjs +\`\`\` +`; + + const result = extractFromSource('README.md', markdown); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toMatchObject({ + name: 'README.md', + language: 'markdown', + }); + + const headings = result.nodes.filter((n) => n.kind === 'module'); + expect(headings.map((n) => n.name)).toContain('Project Guide'); + expect(headings.map((n) => n.name)).toContain('Release'); + + const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs'); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'docs/setup.md#install', + referenceKind: 'imports', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'scripts/release.mjs', + referenceKind: 'calls', + language: 'markdown', + }), + ]) + ); + }); + + it('should extract structured table rows and file-symbol references from Markdown', () => { + const markdown = `# Maintenance Guide + +## Phase 4 + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | + +- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('phases/phase4.md', markdown); + + const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row')); + expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2'])); + + const p4s1 = tableRows.find((n) => n.name === 'P4-S1'); + expect(p4s1?.signature).toContain('Template: P4-S1'); + expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4'); + + const commandNode = result.nodes.find((n) => + n.kind === 'function' && + n.language === 'markdown' && + n.signature?.includes('python "{script_path}" p4') + ); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::_p4_stage1', + referenceKind: 'references', + language: 'markdown', + }), + ]) + ); + }); + + it('should keep structured blocks after fences containing a different fence marker', () => { + const markdown = `# Runbook + +\`\`\`text +~~~~ +\`\`\` + +- POST-FENCE references \`src/auth.ts::login\`. + +| Key | Target | +| --- | --- | +| POST-TABLE | \`src/auth.ts::login\` | +`; + + const result = extractFromSource('docs/runbook.md', markdown); + const constants = result.nodes.filter((n) => n.kind === 'constant'); + + expect(constants).toEqual(expect.arrayContaining([ + expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }), + expect.objectContaining({ name: 'POST-TABLE' }), + ])); + }); + + it('indexes Setext (underline) headings and skips frontmatter / code fences', () => { + const markdown = `--- +title: Config Doc +--- + +Architecture Overview +===================== + +Intro paragraph for the overview. + +Routing Layer +------------- + +\`\`\`md +Not A Heading +============= +\`\`\` +`; + + const result = extractFromSource('docs/arch.md', markdown); + const headings = result.nodes.filter((n) => n.kind === 'module'); + const byName = new Map(headings.map((h) => [h.name, h])); + + // Setext H1 (===) and H2 (---) become module nodes. + expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview'); + expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer'); + // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a + // setext-looking line inside a code fence is ignored. + expect(byName.has('title: Config Doc')).toBe(false); + expect(byName.has('Not A Heading')).toBe(false); + }); + + it('builds a deterministic, compact file digest (intro + key references)', () => { + const markdown = `# Release Runbook + +This runbook explains how to cut a release. + +See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`. +It dispatches \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('RUNBOOK.md', markdown); + const fileNode = result.nodes.find((n) => n.kind === 'file'); + + expect(fileNode?.docstring).toBeDefined(); + const digest = fileNode!.docstring!; + // Intro is the first prose line, not the heading or a link blob. + expect(digest).toContain('This runbook explains how to cut a release.'); + // Key referenced files/symbols are surfaced, compacted to basenames. + expect(digest).toContain('refs:'); + expect(digest).toContain('setup.md#install'); + expect(digest).toContain('release.mjs'); + expect(digest).toContain('csv_search.py::run_p4'); + // Short enough to show in node details (the < 200 char detail gate). + expect(digest.length).toBeLessThan(200); + }); +}); + +describe('Code to Markdown Reference Extraction', () => { + it('should extract Markdown path references from code string literals', () => { + const code = ` +export const GUIDE = '../docs/guide.md'; + +export function loadDocs() { + return fs.readFileSync('../docs/guide.md#install', 'utf8'); +} +`; + + const result = extractFromSource('src/load-docs.ts', code); + const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs'); + const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE'); + + expect(loadDocs).toBeDefined(); + expect(guideConstant).toBeDefined(); + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fromNodeId: loadDocs!.id, + referenceName: 'docs/guide.md#install', + referenceKind: 'references', + language: 'typescript', + }), + expect.objectContaining({ + fromNodeId: guideConstant!.id, + referenceName: 'docs/guide.md', + referenceKind: 'references', + language: 'typescript', + }), + ]) + ); + }); +}); + describe('Nix Extraction', () => { it('should distinguish Nix variable and function bindings', () => { const code = ` diff --git a/__tests__/integration/full-pipeline.test.ts b/__tests__/integration/full-pipeline.test.ts index 5b551c136..fc3aab2e7 100644 --- a/__tests__/integration/full-pipeline.test.ts +++ b/__tests__/integration/full-pipeline.test.ts @@ -82,6 +82,97 @@ describe('Integration: full pipeline', () => { cleanupTempDir(tempDir); }); + it('indexes Markdown headings and resolves Markdown links to script files', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'README.md'), + `# Project Guide + +See [Setup](docs/setup.md#install). + +## Release + +\`\`\`bash +node scripts/release.mjs +\`\`\` +` + ); + fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n'); + fs.writeFileSync(path.join(tempDir, 'scripts', 'release.mjs'), 'export function release() { return true; }\n'); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const guide = cg.searchNodes('Project Guide').find((r) => r.node.language === 'markdown'); + expect(guide).toBeDefined(); + + const releaseCommand = cg + .searchNodes('release.mjs') + .find((r) => r.node.language === 'markdown' && r.node.kind === 'function'); + expect(releaseCommand).toBeDefined(); + + const guideEdges = cg.getOutgoingEdges(guide!.node.id).filter((e) => e.kind === 'imports'); + const guideTargets = guideEdges.map((e) => cg.getNode(e.target)); + const setupHeading = guideTargets.find((n) => n?.qualifiedName === 'docs/setup.md#install'); + expect(setupHeading).toMatchObject({ + kind: 'module', + name: 'Install', + filePath: 'docs/setup.md', + startLine: 1, + }); + + const commandEdges = cg.getOutgoingEdges(releaseCommand!.node.id).filter((e) => e.kind === 'calls'); + const commandTargets = commandEdges.map((e) => cg.getNode(e.target)?.filePath); + expect(commandTargets).toContain('scripts/release.mjs'); + } finally { + cg.destroy(); + } + }); + + it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => { + fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'phases', 'phase4.md'), + `# Phase 4 + +## Fixed Script Templates + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'csv_search.py'), + `def _p4_stage1(filepath, condition_spec, probe_cols_spec):\n return 's1'\n\n` + + `def _p4_stage2(filepath, stage1_rows, condition_spec, detail_cols_spec):\n return 's2'\n\n` + + `def run_p4(filepath, args):\n return _p4_stage1(filepath, '-', 'MPN')\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const p4s1Row = cg.searchNodes('P4-S1').find((r) => r.node.language === 'markdown'); + expect(p4s1Row?.node.kind).toBe('constant'); + + const edges = cg.getOutgoingEdges(p4s1Row!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'run_p4', filePath: 'scripts/csv_search.py' }), + expect.objectContaining({ name: '_p4_stage1', filePath: 'scripts/csv_search.py' }), + ]) + ); + } finally { + cg.destroy(); + } + }); + it('runs init → index → resolve → search → callers → context → sync', async () => { const MODULE_COUNT = 120; generateSyntheticProject(tempDir, MODULE_COUNT); @@ -269,4 +360,46 @@ describe('Integration: full pipeline', () => { cg.destroy(); } }, 30_000); + + it('resolves code string references to Markdown headings', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'docs', 'guide.md'), + `# Guide + +## Install + +Run the setup command. +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'load_docs.py'), + `GUIDE = "docs/guide.md"\n\n` + + `def load_docs():\n` + + ` return open("docs/guide.md#install", encoding="utf-8").read()\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const loadDocs = cg.searchNodes('load_docs').find((r) => r.node.language === 'python'); + expect(loadDocs).toBeDefined(); + + const edges = cg.getOutgoingEdges(loadDocs!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'module', + name: 'Install', + qualifiedName: 'docs/guide.md#install', + }), + ]) + ); + } finally { + cg.destroy(); + } + }); }); diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index decaadee5..794c8a1a0 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -358,6 +358,169 @@ describe('Resolution Module', () => { expect(result).not.toBeNull(); expect(result?.targetNodeId).toBe('method:user.ts:User.save:15'); }); + + it('should resolve Markdown file references by filename, path, and anchor suffix', () => { + const mockNodes: Node[] = [ + { + id: 'file:README.md', + kind: 'file', + name: 'README.md', + qualifiedName: 'README.md', + filePath: 'README.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'file:docs/setup.md', + kind: 'file', + name: 'setup.md', + qualifiedName: 'docs/setup.md', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:docs/setup.md:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'docs/setup.md#install', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + { + id: 'file:GUIDE.markdown', + kind: 'file', + name: 'GUIDE.markdown', + qualifiedName: 'GUIDE.markdown', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:GUIDE.markdown:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'GUIDE.markdown#install', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: () => mockNodes, + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['README.md', 'docs/setup.md', 'GUIDE.markdown'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const readmeRef = { + fromNodeId: 'module:docs/setup.md:install:1', + referenceName: 'README.md', + referenceKind: 'imports' as const, + line: 1, + column: 0, + filePath: 'docs/setup.md', + language: 'markdown' as const, + }; + const setupRef = { + ...readmeRef, + referenceName: 'docs/setup.md#install', + filePath: 'README.md', + }; + const markdownRef = { + ...readmeRef, + referenceName: 'GUIDE.markdown#install', + filePath: 'README.md', + }; + + expect(matchReference(readmeRef, context)?.targetNodeId).toBe('file:README.md'); + expect(matchReference(setupRef, context)?.targetNodeId).toBe('module:docs/setup.md:install:1'); + expect(matchReference(markdownRef, context)?.targetNodeId).toBe('module:GUIDE.markdown:install:1'); + }); + + it('should resolve Markdown file-symbol references to symbols in the referenced file', () => { + const mockNodes: Node[] = [ + { + id: 'file:scripts/csv_search.py', + kind: 'file', + name: 'csv_search.py', + qualifiedName: 'scripts/csv_search.py', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 1, + endLine: 100, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'function:scripts/csv_search.py:run_p4:40', + kind: 'function', + name: 'run_p4', + qualifiedName: 'scripts/csv_search.py::run_p4', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 40, + endLine: 55, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: (filePath) => mockNodes.filter((n) => n.filePath === filePath), + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['scripts/csv_search.py'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const ref = { + fromNodeId: 'constant:phases/phase4.md:P4-S1:10', + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references' as const, + line: 10, + column: 20, + filePath: 'phases/phase4.md', + language: 'markdown' as const, + }; + + expect(matchReference(ref, context)?.targetNodeId).toBe('function:scripts/csv_search.py:run_p4:40'); + }); }); describe('Ubiquitous-name ceiling (#999)', () => { diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 9aa3c9597..ccda3671b 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -513,11 +513,12 @@ describe('Source file detection (isSourceFile)', () => { expect(isSourceFile('src/component.tsx')).toBe(true); expect(isSourceFile('lib/util.js')).toBe(true); expect(isSourceFile('src/main.py')).toBe(true); + // Markdown documentation is indexed as a source language. + expect(isSourceFile('README.md')).toBe(true); }); it('rejects unsupported extensions and extensionless files', () => { expect(isSourceFile('src/component.css')).toBe(false); - expect(isSourceFile('README.md')).toBe(false); expect(isSourceFile('Makefile')).toBe(false); expect(isSourceFile('.gitignore')).toBe(false); }); diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index 942fd5bcd..067f70f22 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -442,8 +442,8 @@ describe('FileWatcher', () => { // gate must drop it before scheduling sync. (It must exist on disk: // a VANISHED non-source path is the deleted-directory shape, which // deliberately schedules a sync — #1285.) - fs.writeFileSync(path.join(testDir, 'src', 'readme.md'), '# docs\n'); - __emitWatchEventForTests(testDir, 'src/readme.md'); + fs.writeFileSync(path.join(testDir, 'src', 'styles.css'), 'body {}\n'); + __emitWatchEventForTests(testDir, 'src/styles.css'); // Wait a bit longer than debounce — sync should NOT trigger. await new Promise((r) => setTimeout(r, 400)); diff --git a/src/extraction/generated-detection.ts b/src/extraction/generated-detection.ts index b92ab860b..03116cfb5 100644 --- a/src/extraction/generated-detection.ts +++ b/src/extraction/generated-detection.ts @@ -249,5 +249,9 @@ export function hasGeneratedHeader(content: string): boolean { * indexer persists to `files.generated`. */ export function detectGeneratedFile(filePath: string, content: string): boolean { + // Markdown carries no banner: its `#` is a heading, not a comment leader, + // so a README that DESCRIBES generated code ("Code generated … DO NOT EDIT" + // under a heading) would flag itself. The path convention still applies. + if (/\.(?:md|markdown)$/i.test(filePath)) return isGeneratedFile(filePath); return isGeneratedFile(filePath) || hasGeneratedHeader(content); } diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index c7710f200..37ad0e0a4 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file @@ -100,6 +100,9 @@ export const EXTENSION_MAP: Record = { '.yaml': 'yaml', // Twig templates (file-level tracking only, no symbol extraction) '.twig': 'twig', + '.md': 'markdown', + '.mdx': 'markdown', + '.markdown': 'markdown', '.rb': 'ruby', '.rake': 'ruby', '.swift': 'swift', @@ -588,6 +591,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'markdown') return true; // custom documentation extractor if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -596,7 +600,7 @@ export function isLanguageSupported(language: Language): boolean { * Check if a grammar has been loaded and is ready for parsing. */ export function isGrammarLoaded(language: Language): boolean { - if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; + if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor' || language === 'markdown') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed return languageCache.has(language); @@ -619,7 +623,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'markdown']; } /** @@ -699,6 +703,7 @@ export function getLanguageDisplayName(language: Language): string { twig: 'Twig', xml: 'XML', properties: 'Java properties', + markdown: 'Markdown', cfml: 'CFML', cfscript: 'CFScript', cfquery: 'CFQuery (SQL)', diff --git a/src/extraction/markdown-extractor.ts b/src/extraction/markdown-extractor.ts new file mode 100644 index 000000000..4695ea4be --- /dev/null +++ b/src/extraction/markdown-extractor.ts @@ -0,0 +1,1003 @@ +import * as path from 'path'; +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, EdgeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +interface HeadingInfo { + id: string; + level: number; + title: string; + slug: string; + line: number; + endLine: number; + node: Node; +} + +interface FenceState { + marker: string; + language: string; + startLine: number; + lines: Array<{ text: string; line: number }>; +} + +/** + * Lightweight Markdown extractor. + * + * Markdown is indexed as documentation structure rather than code syntax: + * headings become searchable module nodes, links become import nodes, and + * shell-like fenced blocks create command nodes plus file-path references. + */ +export class MarkdownExtractor { + private filePath: string; + private lines: string[]; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private headings: HeadingInfo[] = []; + private referenceKeys = new Set(); + + constructor(filePath: string, source: string) { + this.filePath = normalizeRelativePath(filePath); + this.lines = source.split('\n'); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + + try { + const fileNode = this.createFileNode(); + this.extractHeadings(fileNode); + this.extractStructuredBlocks(fileNode); + this.extractLinksAndCommands(fileNode); + this.finalizeFileDigest(fileNode); + } catch (error) { + this.errors.push({ + message: `Markdown extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + private createFileNode(): Node { + const id = generateNodeId(this.filePath, 'file', this.filePath, 1); + const fileNode: Node = { + id, + kind: 'file', + name: path.posix.basename(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'markdown', + startLine: 1, + endLine: Math.max(1, this.lines.length), + startColumn: 0, + endColumn: this.lines[this.lines.length - 1]?.length || 0, + // docstring is filled in by finalizeFileDigest() once headings and + // references are known, so it becomes a triage-friendly digest. + docstring: undefined, + updatedAt: Date.now(), + }; + + this.nodes.push(fileNode); + return fileNode; + } + + private extractHeadings(fileNode: Node): void { + const rawHeadings: Array<{ level: number; title: string; line: number; column: number }> = []; + const frontmatterEnd = this.frontmatterEndIndex(); + let fenceMarker: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + if (i <= frontmatterEnd) continue; + const line = this.lines[i]!; + + // Track fenced code blocks so `# foo` comments and `===`/`---` lines + // inside a code sample are never mistaken for document headings. + const fence = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fence) { + const marker = fence[2]![0]!.repeat(fence[2]!.length); + if (fenceMarker === null) fenceMarker = marker; + else if (line.trimStart().startsWith(fenceMarker)) fenceMarker = null; + continue; + } + if (fenceMarker !== null) continue; + + // ATX heading: `## Title` + const atx = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); + if (atx) { + rawHeadings.push({ + level: atx[1]!.length, + title: stripInlineMarkdown(atx[2]!.trim()), + line: i + 1, + column: line.indexOf('#'), + }); + continue; + } + + // Setext heading: a paragraph line underlined by `===` (level 1) or + // `---` (level 2). CommonMark treats a `---` directly under a paragraph + // as a heading, not a thematic break — and the frontmatter skip above + // keeps a closing `---` from turning its last key into a heading. + const underline = /^(=+|-+)\s*$/.exec(line.trim()); + if (underline && i - 1 > frontmatterEnd && isSetextTextLine(this.lines[i - 1] ?? '')) { + rawHeadings.push({ + level: underline[1]![0] === '=' ? 1 : 2, + title: stripInlineMarkdown((this.lines[i - 1] ?? '').trim()), + line: i, // the text line carries the heading + column: 0, + }); + } + } + + const slugCounts = new Map(); + for (let i = 0; i < rawHeadings.length; i++) { + const heading = rawHeadings[i]!; + const baseSlug = slugifyHeading(heading.title); + const seen = slugCounts.get(baseSlug) ?? 0; + slugCounts.set(baseSlug, seen + 1); + const slug = seen === 0 ? baseSlug : `${baseSlug}-${seen}`; + + let endLine = this.lines.length; + for (let j = i + 1; j < rawHeadings.length; j++) { + if (rawHeadings[j]!.level <= heading.level) { + endLine = rawHeadings[j]!.line - 1; + break; + } + } + + const nodeId = generateNodeId(this.filePath, 'module', `${slug}:${heading.line}`, heading.line); + const node: Node = { + id: nodeId, + kind: 'module', + name: heading.title, + qualifiedName: `${this.filePath}#${slug}`, + filePath: this.filePath, + language: 'markdown', + signature: `${'#'.repeat(heading.level)} ${heading.title}`, + docstring: this.buildDocstring(heading.line + 1, endLine), + startLine: heading.line, + endLine, + startColumn: heading.column, + endColumn: this.lines[heading.line - 1]?.length || 0, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.headings.push({ + id: nodeId, + level: heading.level, + title: heading.title, + slug, + line: heading.line, + endLine, + node, + }); + } + + const stack: HeadingInfo[] = []; + for (const heading of this.headings) { + while (stack.length > 0 && stack[stack.length - 1]!.level >= heading.level) { + stack.pop(); + } + const parent = stack[stack.length - 1]; + this.edges.push({ + source: parent?.id ?? fileNode.id, + target: heading.id, + kind: 'contains', + provenance: 'heuristic', + }); + stack.push(heading); + } + } + + private extractLinksAndCommands(fileNode: Node): void { + let fence: FenceState | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const lineNumber = i + 1; + const fenceMatch = /^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+.-]*)/.exec(line); + + if (fence) { + if (line.trimStart().startsWith(fence.marker)) { + this.extractCommandsFromFence(fence, fileNode); + fence = null; + } else { + fence.lines.push({ text: line, line: lineNumber }); + } + continue; + } + + if (fenceMatch) { + fence = { + marker: fenceMatch[2]![0]!.repeat(fenceMatch[2]!.length), + language: (fenceMatch[3] ?? '').toLowerCase(), + startLine: lineNumber, + lines: [], + }; + continue; + } + + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + this.extractMarkdownLinks(line, lineNumber, owner); + if (!isTableRowLine(line)) { + this.extractFileSymbolMentions(line, lineNumber, owner); + this.extractPathMentions(line, lineNumber, owner); + } + } + + if (fence) { + this.extractCommandsFromFence(fence, fileNode); + } + } + + private extractStructuredBlocks(fileNode: Node): void { + this.extractTables(fileNode); + this.extractListItems(fileNode); + } + + private extractTables(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length - 1; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const separator = this.lines[i + 1]!; + if (!isTableHeaderLine(line) || !isTableSeparatorLine(separator)) continue; + + const headers = parseTableCells(line); + if (headers.length < 2) continue; + + let rowIndex = i + 2; + while (rowIndex < this.lines.length && isTableRowLine(this.lines[rowIndex]!)) { + const rowLine = this.lines[rowIndex]!; + const cells = parseTableCells(rowLine); + const lineNumber = rowIndex + 1; + if (shouldIndexTableRow(headers, cells)) { + this.createTableRowNode(headers, cells, rowLine, lineNumber, fileNode); + } + rowIndex++; + } + + i = rowIndex - 1; + } + } + + private extractListItems(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const match = /^\s*[-*+]\s+(?:\[[ xX]\]\s+)?(.+?)\s*$/.exec(line); + if (!match) continue; + + const text = stripInlineMarkdown(match[1]!.trim()); + if (!shouldIndexListItem(text)) continue; + + const lineNumber = i + 1; + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const name = stableIdentifier(text) ?? truncateForName(text, 80); + const nodeId = generateNodeId(this.filePath, 'constant', `list:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::list-item:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature: `list item: ${truncateForSignature(text, 180)}`, + docstring: text, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, line.indexOf(match[1]!)), + endColumn: line.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.extractStructuredReferences(text, lineNumber, node); + } + } + + private createTableRowNode( + headers: string[], + cells: string[], + rowLine: string, + lineNumber: number, + fileNode: Node + ): void { + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const rowText = cells.join(' | '); + const name = stableIdentifier(rowText) ?? truncateForName(firstNonEmptyCell(cells) || `row ${lineNumber}`, 80); + const signature = summarizeTableRow(headers, cells); + const nodeId = generateNodeId(this.filePath, 'constant', `table:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::table-row:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature, + docstring: signature, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, rowLine.indexOf(firstNonEmptyCell(cells) || '|')), + endColumn: rowLine.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + this.extractStructuredReferences(rowText, lineNumber, node); + this.extractCommandsFromStructuredCells(cells, rowLine, lineNumber, node); + } + + private extractCommandsFromStructuredCells(cells: string[], rowLine: string, lineNumber: number, owner: Node): void { + for (const cell of cells) { + const candidates = extractCodeSpans(cell); + if (candidates.length === 0) candidates.push(stripInlineMarkdown(cell)); + + for (const candidate of candidates) { + const command = extractCommandFromText(candidate); + if (!command) continue; + const column = Math.max(0, rowLine.indexOf(candidate)); + this.createCommandNode(command, lineNumber, column, owner); + } + } + } + + private createCommandNode(command: string, line: number, column: number, owner: Node): void { + const nodeId = generateNodeId(this.filePath, 'function', `command:${line}:${column}:${command}`, line); + const node: Node = { + id: nodeId, + kind: 'function', + name: commandName(command), + qualifiedName: `${this.filePath}::command:${line}:${column}:${command}`, + filePath: this.filePath, + language: 'markdown', + signature: command, + startLine: line, + endLine: line, + startColumn: column, + endColumn: column + command.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + const scriptPath = extractScriptPath(command); + if (scriptPath) { + const normalizedTarget = this.normalizeReference(scriptPath); + if (normalizedTarget) { + this.addReference(nodeId, normalizedTarget, 'calls', line, column); + } + } + } + + private extractMarkdownLinks(line: string, lineNumber: number, owner: Node): void { + const linkRegex = /!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g; + let match: RegExpExecArray | null; + + while ((match = linkRegex.exec(line)) !== null) { + const target = match[2]!; + if (!isLocalReference(target)) continue; + + const normalizedTarget = this.normalizeReference(target); + if (!normalizedTarget) continue; + + const column = match.index; + const label = stripInlineMarkdown(match[1] || target) || target; + const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: displayName, + qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, + filePath: this.filePath, + language: 'markdown', + signature: match[0], + docstring: label, + startLine: lineNumber, + endLine: lineNumber, + startColumn: column, + endColumn: column + match[0].length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.addReference(owner.id, normalizedTarget, 'imports', lineNumber, column); + } + } + + private extractPathMentions(line: string, lineNumber: number, owner: Node): void { + const pathRegex = /(?= best.level) best = heading; + } + } + return best?.node ?? null; + } + + private addReference( + fromNodeId: string, + referenceName: string, + referenceKind: EdgeKind, + line: number, + column: number + ): void { + const key = `${fromNodeId}:${referenceKind}:${referenceName}:${line}:${column}`; + if (this.referenceKeys.has(key)) return; + this.referenceKeys.add(key); + this.unresolvedReferences.push({ + fromNodeId, + referenceName, + referenceKind, + line, + column, + filePath: this.filePath, + language: 'markdown', + }); + } + + private normalizeReference(target: string): string | null { + const [pathAndSymbol, anchorPart] = splitAnchor(target.trim()); + const [pathPart, symbolPart] = splitFileSymbol(pathAndSymbol); + const cleanPath = decodePath(pathPart.split(/[?#]/)[0] ?? ''); + const anchor = anchorPart ? `#${slugifyHeading(anchorPart)}` : ''; + + if (!cleanPath && anchor) { + return `${this.filePath}${anchor}`; + } + if (!cleanPath) return null; + + const withoutLeadingSlash = cleanPath.startsWith('/') ? cleanPath.slice(1) : cleanPath; + const baseDir = path.posix.dirname(this.filePath); + const normalized = cleanPath.startsWith('/') || baseDir === '.' + ? path.posix.normalize(withoutLeadingSlash) + : path.posix.normalize(path.posix.join(baseDir, withoutLeadingSlash)); + + if (normalized.startsWith('../') || normalized === '..') return null; + return `${normalized}${symbolPart ? `::${symbolPart}` : ''}${anchor}`; + } + + private buildDocstring(startLine: number, endLine: number): string | undefined { + const text = this.lines + .slice(Math.max(0, startLine - 1), Math.max(0, endLine)) + .map((line) => stripInlineMarkdown(line.replace(/^#{1,6}\s+/, '').trim())) + .filter((line) => line && !/^(```|~~~)/.test(line)) + .join('\n') + .trim(); + + if (!text) return undefined; + return text.length > 600 ? `${text.slice(0, 600)}...` : text; + } + + /** + * Index of the closing `---` of a leading YAML frontmatter block, or -1 when + * the file has none. Lines at or before this index are metadata, not body — + * so they never produce headings and never seed the intro/digest. + */ + private frontmatterEndIndex(): number { + if ((this.lines[0] ?? '').trim() !== '---') return -1; + for (let j = 1; j < this.lines.length; j++) { + if (this.lines[j]!.trim() === '---') return j; + } + return -1; + } + + /** + * Replace the file node's docstring with a deterministic digest — a one-line + * "what is this doc about" intro plus the key files/symbols it references. + * It is derived purely from the already-extracted structure (no LLM), kept + * short enough to surface in node details, and — because docstrings are in + * the FTS index — makes a doc discoverable by the symbols it documents even + * when the query matches no heading or filename. See the reviewer thread on + * PR #361. + */ + private finalizeFileDigest(fileNode: Node): void { + const intro = this.buildIntro(); + const refs = this.collectKeyReferences(); + const parts: string[] = []; + if (intro) parts.push(intro); + if (refs.length > 0) parts.push(`refs: ${refs.join(', ')}`); + const digest = parts.join(' · ').trim(); + fileNode.docstring = digest || this.buildDocstring(1, Math.min(this.lines.length, 40)); + } + + /** + * First real prose line of the document — the closest deterministic stand-in + * for "what this file is about". Skips frontmatter, headings, fenced code, + * tables, badge/image-only lines, and raw HTML. + */ + private buildIntro(): string | null { + let fenceMarker: string | null = null; + for (let i = this.frontmatterEndIndex() + 1; i < this.lines.length; i++) { + const trimmed = this.lines[i]!.trim(); + const fence = /^(`{3,}|~{3,})/.exec(trimmed); + if (fence) { + const marker = fence[1]![0]!.repeat(fence[1]!.length); + fenceMarker = fenceMarker === null ? marker : (trimmed.startsWith(fenceMarker) ? null : fenceMarker); + continue; + } + if (fenceMarker !== null) continue; + if (!trimmed) continue; + if (/^#{1,6}\s/.test(trimmed)) continue; // ATX heading + if (/^(=+|-+|\*{3,}|_{3,})\s*$/.test(trimmed)) continue; // setext underline / hr + if (trimmed.startsWith('|')) continue; // table + if (/^!\[/.test(trimmed)) continue; // image / badge line + if (/^