From e729a6f69b2e1527ea0471985c65f1ca0db38d69 Mon Sep 17 00:00:00 2001 From: Mehdi Baneshi Date: Sat, 21 Mar 2026 23:22:55 +0330 Subject: [PATCH] feat: Schematic Explorer, Theme System, E2E Tests (v0.2.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: performance optimizations, schematic API, and branching setup Backend: - Fix N+1 queries in communities, processes, ask context, skip chapter - Add 6 missing DB indexes (migration 0010) - Add pagination (limit/offset) to files, symbols, narratives repos and API routes - Expand moka cache capacity 100→500, add caching to files/symbols endpoints - Add HTTP Cache-Control middleware for GET routes - Add schematic API endpoint with lazy-load tree + expand + detail Frontend: - Add client-side fetch cache (cachedGet with 5min TTL) for read-heavy endpoints - Rewrite schematic pages: remove ELK.js (1.5MB), inline pure-TS layout engine (159 LOC) - Add vite-plugin-compression2 (brotli), manual chunk splitting (three, shiki, 3d-graph) - Lazy-load Shiki languages per file instead of all upfront CI/Infra: - Update CI to trigger on dev branch - Enable delete-branch-on-merge, adopt feature branch workflow Docs: - Add docs/OPTIMIZATIONS.md (12-item performance plan) - Add docs/SCHEMATIC_DESIGN.md (unified explorer design) - Add docs/adr/0002-unified-schematic-explorer.md Co-Authored-By: Claude Opus 4.6 (1M context) * docs: add git workflow rules and performance patterns to CLAUDE.md Ensures all coding agents (Claude, Cursor, etc.) follow the feature branch workflow (feat/xxx → dev → main) and are aware of established performance patterns (pagination, caching, N+1 prevention). Co-Authored-By: Claude Opus 4.6 (1M context) * feat: unified schematic explorer with full interaction system Backend: - Community color enrichment for directory nodes (dominant community) - Grid layout for community overview, layered layout for symbol graphs - Schematic expand returns subdirectories correctly Frontend (6 new components, ~730 lines): - SchematicTooltip: hover tooltips with node metadata - SchematicDetailTabs: 5-tab detail panel (overview, source, relations, learn, notes) - SchematicSourcePopup: floating source viewer on double-click - SchematicContextMenu: right-click actions (view source, ask AI, annotations, etc.) - SchematicKeyboardOverlay: keyboard shortcut reference (?) - SchematicMinimap: viewport overview with click-to-pan Unified page (/explore/schematic): - Tree + Graph mode toggle with shared state - Hover highlighting (connected edges + ghost nodes) - Single click expand/select, double-click deep dive - Right-click context menu per node type - Keyboard shortcuts (1/2 modes, F fit, ? help, Esc close) - Lazy loading via /api/v1/schematic endpoints - Community sidebar with progress bars in graph mode - Search highlighting across all nodes Also: - Strip ("name") prefix from module_summary narratives - Grid layout for community nodes (was single row) - computeFitToView utility in layout.ts Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add legend panel, drop-shadow on hover, improved node borders - Legend toggle in toolbar with edge colors, node types, community swatches - SVG drop-shadow filter on hovered nodes - Rounded corners rx=8, fix minimap type error Co-Authored-By: Claude Opus 4.6 (1M context) * fix: polish schematic - root label, tooltip hints, narrative fallback - Root node shows "codeilus" instead of "." - Tooltip shows "click to expand" hint for directories - Detail panel shows helpful message when no narrative available - Strip ("name") prefix from narratives in detail panel - Fix minimap type error (community_color cast) Co-Authored-By: Claude Opus 4.6 (1M context) * fix: community-filtered symbols/edges, source tab loading, root label Backend: - Filter symbols by community_id when set (was loading ALL symbols) - Filter edges to intra-community only (JOIN on community_members) - Community 2: 241 symbols + 304 edges (was 5773 + 14252) Frontend: - Fix Source tab lazy loading (track sourceNodeId to detect node changes) - Root node shows "codeilus" instead of "." - Narrative fallback message when no AI explanation Co-Authored-By: Claude Opus 4.6 (1M context) * feat: E2E tests for schematic (31 tests, all green) + source code fix Tests (31 passing): - 6 API tests: schematic, expand, detail, community filter - 7 tree mode tests: rendering, expand, detail panel, search, minimap - 3 graph mode tests: mode switch, sidebar, drill-down - 8 interaction tests: hover, right-click, keyboard, fit, legend, source tab - 7 TDD behavior tests: source code, progress, learn tab, copy, focus, expand Bug fixes: - Source code endpoint: handle absolute file paths (was "path traversal" error) - Schematic expand: use common-prefix normalization for any codebase path format - Clipboard copy: wrap in try/catch for headless environments Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 4 +- CLAUDE.md | 58 +- crates/codeilus-api/src/routes/ask.rs | 43 +- crates/codeilus-api/src/routes/files.rs | 53 +- crates/codeilus-api/src/routes/graph.rs | 141 ++-- crates/codeilus-api/src/routes/learning.rs | 7 +- crates/codeilus-api/src/routes/mod.rs | 37 + crates/codeilus-api/src/routes/schematic.rs | 105 +++ crates/codeilus-api/src/routes/symbols.rs | 73 +- crates/codeilus-api/src/state.rs | 2 +- crates/codeilus-db/src/lib.rs | 2 +- crates/codeilus-db/src/migrations.rs | 10 + crates/codeilus-db/src/repos/file_repo.rs | 56 ++ crates/codeilus-db/src/repos/mod.rs | 2 + .../codeilus-db/src/repos/narrative_repo.rs | 28 + .../codeilus-db/src/repos/schematic_repo.rs | 752 ++++++++++++++++++ crates/codeilus-db/src/repos/symbol_repo.rs | 60 ++ docs/OPTIMIZATIONS.md | 181 +++++ docs/SCHEMATIC_DESIGN.md | 664 ++++++++++++++++ docs/adr/0002-unified-schematic-explorer.md | 77 ++ frontend/e2e/schematic.spec.ts | 488 ++++++++++++ frontend/package.json | 11 +- frontend/pnpm-lock.yaml | 48 +- frontend/src/lib/api.ts | 50 +- .../src/lib/schematic/SchematicCanvas.svelte | 92 --- .../lib/schematic/SchematicContextMenu.svelte | 91 +++ .../lib/schematic/SchematicDetailTabs.svelte | 241 ++++++ .../schematic/SchematicKeyboardOverlay.svelte | 41 + .../src/lib/schematic/SchematicMinimap.svelte | 77 ++ .../src/lib/schematic/SchematicModal.svelte | 49 -- .../src/lib/schematic/SchematicSearch.svelte | 78 -- .../lib/schematic/SchematicSourcePopup.svelte | 73 ++ .../src/lib/schematic/SchematicTooltip.svelte | 76 ++ frontend/src/lib/schematic/edge-path.ts | 42 - frontend/src/lib/schematic/elk-layout.ts | 140 ---- frontend/src/lib/schematic/layout.ts | 216 +++++ frontend/src/lib/schematic/types.ts | 26 - frontend/src/lib/types.ts | 62 ++ frontend/src/routes/explore/+page.svelte | 16 +- .../src/routes/explore/schematic/+page.svelte | 610 ++++++++++++++ .../explore/schematic/graph/+page.svelte | 502 +++++------- .../explore/schematic/tree/+page.svelte | 414 +++++----- frontend/src/routes/explore/tree/+page.svelte | 4 +- frontend/src/routes/learn/+page.svelte | 2 +- frontend/src/routes/learn/[id]/+page.svelte | 2 +- frontend/vite.config.ts | 9 +- migrations/0010_add_indexes.sql | 10 + 47 files changed, 4707 insertions(+), 1118 deletions(-) create mode 100644 crates/codeilus-api/src/routes/schematic.rs create mode 100644 crates/codeilus-db/src/repos/schematic_repo.rs create mode 100644 docs/OPTIMIZATIONS.md create mode 100644 docs/SCHEMATIC_DESIGN.md create mode 100644 docs/adr/0002-unified-schematic-explorer.md create mode 100644 frontend/e2e/schematic.spec.ts delete mode 100644 frontend/src/lib/schematic/SchematicCanvas.svelte create mode 100644 frontend/src/lib/schematic/SchematicContextMenu.svelte create mode 100644 frontend/src/lib/schematic/SchematicDetailTabs.svelte create mode 100644 frontend/src/lib/schematic/SchematicKeyboardOverlay.svelte create mode 100644 frontend/src/lib/schematic/SchematicMinimap.svelte delete mode 100644 frontend/src/lib/schematic/SchematicModal.svelte delete mode 100644 frontend/src/lib/schematic/SchematicSearch.svelte create mode 100644 frontend/src/lib/schematic/SchematicSourcePopup.svelte create mode 100644 frontend/src/lib/schematic/SchematicTooltip.svelte delete mode 100644 frontend/src/lib/schematic/edge-path.ts delete mode 100644 frontend/src/lib/schematic/elk-layout.ts create mode 100644 frontend/src/lib/schematic/layout.ts delete mode 100644 frontend/src/lib/schematic/types.ts create mode 100644 frontend/src/routes/explore/schematic/+page.svelte create mode 100644 migrations/0010_add_indexes.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d251050..3bef7eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main] + branches: [main, dev] pull_request: - branches: [main] + branches: [main, dev] env: CARGO_TERM_COLOR: always diff --git a/CLAUDE.md b/CLAUDE.md index 7a1d1e5..90f4ed2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,8 +6,45 @@ Codeilus is a single Rust binary that analyzes any codebase and transforms it in ## Key Docs - `NORTH_STAR.md` — Purpose, architecture, data flow, sprints, acceptance criteria - `docs/AGENT_PROMPTS.md` — Copy-paste prompts for parallel Cursor agents (6 waves) +- `docs/OPTIMIZATIONS.md` — Performance optimization plan (12 items, 6 phases) +- `docs/SCHEMATIC_DESIGN.md` — Unified schematic explorer design +- `docs/adr/0002-unified-schematic-explorer.md` — ADR for schematic unification - `CLAUDE.md` — This file (read by all agents) +## Git Workflow (MUST FOLLOW) + +**Branching model:** +``` +main (production — protected, receives PRs from dev only) + └── dev (integration — receives PRs from feature branches) + └── feat/xxx, fix/xxx, docs/xxx (short-lived feature branches) +``` + +**Rules for all agents:** +1. **NEVER commit directly to `main` or `dev`.** Always create a feature branch. +2. Create branches from `dev`: `git checkout -b feat/my-feature dev` +3. Push and open a PR targeting `dev` (not `main`). +4. PRs require CI to pass: `cargo build`, `cargo clippy` (zero warnings), `cargo test`. +5. Use squash merge when merging to `dev`. Feature branches auto-delete after merge. +6. Only `dev → main` PRs are used for releases. + +**Branch naming conventions:** +- `feat/short-description` — new features +- `fix/short-description` — bug fixes +- `docs/short-description` — documentation only +- `refactor/short-description` — code restructuring +- `perf/short-description` — performance improvements + +**Commit message format:** +``` +type: concise description + +Optional body with details. + +Co-Authored-By: +``` +Types: `feat`, `fix`, `docs`, `refactor`, `perf`, `test`, `ci`, `chore` + ## Build & Test ```bash cd /Users/bm/codeilus/codeilus @@ -15,6 +52,7 @@ cargo build # build all crates cargo test # run all tests cargo clippy # must be zero warnings cargo test -p codeilus-parse # test single crate +cd frontend && pnpm build # frontend build ``` ## Architecture Rules @@ -33,17 +71,27 @@ cargo test -p codeilus-parse # test single crate - Events flow through EventBus (tokio broadcast) — never direct state mutation - Tests use in-memory SQLite (`DbPool::in_memory()`) +## Performance Patterns (established) +- **API pagination**: All list endpoints accept `?limit=` and `?offset=` (default 50, max 200) +- **Moka cache**: 500-entry, 10min TTL in `AppCache`. Cache reads before DB queries. +- **HTTP caching**: `Cache-Control: public, max-age=300` on all GET routes via middleware +- **No N+1 queries**: Batch-load with JOINs or `WHERE IN (...)`, group in Rust with HashMap +- **Frontend fetch cache**: `cachedGet()` in `api.ts` with 5min TTL for read-heavy endpoints +- **Vite**: Brotli compression, manual chunk splitting for three/shiki/3d-force-graph +- **Shiki**: Languages loaded per-file on demand, not all upfront + ## Current State -Waves 1-4 complete. All 16 crates functional with 268 tests passing, zero clippy warnings. +Waves 1-4 complete. All 16 crates functional with 220+ tests passing, zero clippy warnings. - Parse: Tree-sitter for 12 languages, incremental parsing - Graph: Call graph, Louvain communities, 3-level zoom visualization - Metrics: SLOC, complexity, fan-in/out, modularity - Narrate: 8 narrative types via Claude Code CLI - Learn: Curriculum, quizzes, XP/badges/streaks -- API: 50+ REST endpoints, SSE streaming Q&A -- Frontend: SvelteKit 5 with graph explorer, learning path, Ask AI -- Infrastructure: r2d2 pool, moka cache, pipeline checkpoints, structured logging -Next: Wave 5-6 polish, release pipeline, documentation refresh. +- API: 50+ REST endpoints, SSE streaming Q&A, schematic lazy-load API +- Frontend: SvelteKit 5 with graph explorer, learning path, Ask AI, schematic views +- Infrastructure: r2d2 pool, moka cache (500 entries), pipeline checkpoints, structured logging +- Performance: Pagination on all list endpoints, N+1 fixes, client+server caching, brotli compression +Next: Unified schematic explorer (ADR-0002), Wave 5-6 polish, release pipeline. ## Parallel Agent Waves - **Wave 1** (3 agents): codeilus-parse, codeilus-db repos, frontend skeleton diff --git a/crates/codeilus-api/src/routes/ask.rs b/crates/codeilus-api/src/routes/ask.rs index 2981279..3a8d9bb 100644 --- a/crates/codeilus-api/src/routes/ask.rs +++ b/crates/codeilus-api/src/routes/ask.rs @@ -17,8 +17,6 @@ use tracing::info; use crate::state::AppState; -type SymbolRow = (String, String, String, i64, i64, Option); - #[derive(Deserialize)] struct AskRequest { question: String, @@ -72,24 +70,35 @@ async fn ask_stream( }).into_response(); } - // Build context from selected symbols + // Build context from selected symbols (batch query) let mut context_parts = Vec::new(); if !body.context_symbol_ids.is_empty() { let conn = state.db.connection(); - for sid in &body.context_symbol_ids { - let result: Result = conn.query_row( - "SELECT s.name, s.kind, f.path, s.start_line, s.end_line, s.signature - FROM symbols s JOIN files f ON s.file_id = f.id WHERE s.id = ?1", - [sid], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?, row.get(5)?)), - ); - if let Ok((name, kind, path, start, end, sig)) = result { - context_parts.push(format!( - "- {} `{}` in `{}` (lines {}-{}){}", - kind, name, path, start, end, - sig.map(|s| format!("\n Signature: {}", s)).unwrap_or_default() - )); - } + let placeholders: Vec = body.context_symbol_ids.iter().enumerate().map(|(i, _)| format!("?{}", i + 1)).collect(); + let sql = format!( + "SELECT s.name, s.kind, f.path, s.start_line, s.end_line, s.signature \ + FROM symbols s JOIN files f ON s.file_id = f.id \ + WHERE s.id IN ({})", + placeholders.join(", ") + ); + let mut stmt = conn.prepare(&sql).unwrap(); + let params: Vec<&dyn rusqlite::types::ToSql> = body.context_symbol_ids.iter().map(|id| id as &dyn rusqlite::types::ToSql).collect(); + let rows = stmt.query_map(params.as_slice(), |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Option>(5)?, + )) + }).unwrap(); + for (name, kind, path, start, end, sig) in rows.flatten() { + context_parts.push(format!( + "- {} `{}` in `{}` (lines {}-{}){}", + kind, name, path, start, end, + sig.map(|s| format!("\n Signature: {}", s)).unwrap_or_default() + )); } } diff --git a/crates/codeilus-api/src/routes/files.rs b/crates/codeilus-api/src/routes/files.rs index bc2d2eb..eb9261b 100644 --- a/crates/codeilus-api/src/routes/files.rs +++ b/crates/codeilus-api/src/routes/files.rs @@ -17,16 +17,29 @@ use crate::state::AppState; #[derive(Deserialize)] pub struct FileListQuery { pub language: Option, + pub limit: Option, + pub offset: Option, } -/// GET /api/v1/files — List all files, optional ?language= filter +/// GET /api/v1/files — List files with pagination, optional ?language= filter async fn list_files( State(state): State, Query(query): Query, -) -> Result>, ApiError> { +) -> Result, ApiError> { + let limit = query.limit.unwrap_or(50).clamp(1, 200); + let offset = query.offset.unwrap_or(0).max(0); + let cache_key = format!("files:l={:?}:l={}:o={}", query.language, limit, offset); + + if let Some(cached) = state.cache.json.get(&cache_key) { + return Ok(Json(cached)); + } + let repo = FileRepo::new(Arc::clone(&state.db)); - let files = repo.list(query.language.as_deref())?; - Ok(Json(files)) + let files = repo.list_paginated(query.language.as_deref(), limit, offset)?; + let value = serde_json::to_value(&files) + .map_err(|e| ApiError::from(codeilus_core::error::CodeilusError::Internal(e.to_string())))?; + state.cache.json.insert(cache_key, value.clone()); + Ok(Json(value)) } /// GET /api/v1/files/:id — Get a single file by ID @@ -83,24 +96,32 @@ async fn get_file_source( message: "No repository has been analyzed".to_string(), })?; - // Resolve the file path relative to repo root + // Resolve the file path — handle both relative and absolute paths let clean_path = file.path.strip_prefix("./").unwrap_or(&file.path); - let full_path = repo_root.join(clean_path); + let full_path = if std::path::Path::new(clean_path).is_absolute() { + std::path::PathBuf::from(clean_path) + } else { + repo_root.join(clean_path) + }; - // Canonicalize and verify the path stays within repo root (prevent path traversal) + // Canonicalize and verify the path exists and is safe let canonical = full_path.canonicalize().map_err(|e| ApiError { status: StatusCode::NOT_FOUND, message: format!("Could not resolve file path: {}", e), })?; - let canonical_root = repo_root.canonicalize().map_err(|e| ApiError { - status: StatusCode::INTERNAL_SERVER_ERROR, - message: format!("Could not resolve repo root: {}", e), - })?; - if !canonical.starts_with(&canonical_root) { - return Err(ApiError { - status: StatusCode::FORBIDDEN, - message: "Path traversal detected".to_string(), - }); + + // For relative paths, verify within repo root (prevent traversal) + if !std::path::Path::new(clean_path).is_absolute() { + let canonical_root = repo_root.canonicalize().map_err(|e| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: format!("Could not resolve repo root: {}", e), + })?; + if !canonical.starts_with(&canonical_root) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "Path traversal detected".to_string(), + }); + } } let content = std::fs::read_to_string(&canonical).map_err(|e| { diff --git a/crates/codeilus-api/src/routes/graph.rs b/crates/codeilus-api/src/routes/graph.rs index da0e999..773aff6 100644 --- a/crates/codeilus-api/src/routes/graph.rs +++ b/crates/codeilus-api/src/routes/graph.rs @@ -299,12 +299,12 @@ async fn list_communities( State(state): State, ) -> Result>, ApiError> { let conn = state.db.connection(); - let mut communities = Vec::new(); - let mut stmt = conn + // Load all communities + let mut comm_stmt = conn .prepare("SELECT id, name, cohesion_score FROM communities") .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let rows = stmt + let comm_rows = comm_stmt .query_map([], |row| { Ok(( row.get::<_, i64>(0)?, @@ -314,29 +314,43 @@ async fn list_communities( }) .map_err(|e| CodeilusError::Database(Box::new(e)))?; - for row in rows { - let (id, name, cohesion) = - row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + let mut community_info: Vec<(i64, String, f64)> = Vec::new(); + for row in comm_rows { + let (id, name, cohesion) = row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + community_info.push((id, name.unwrap_or_default(), cohesion.unwrap_or(0.0))); + } - // Fetch members for this community - let mut member_stmt = conn - .prepare("SELECT symbol_id FROM community_members WHERE community_id = ?1") - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let members: Vec = member_stmt - .query_map(params![id], |row| row.get(0)) - .map_err(|e| CodeilusError::Database(Box::new(e)))? - .collect::, _>>() - .map_err(|e| CodeilusError::Database(Box::new(e)))?; + // Batch-load all members in one query, group by community_id + let mut member_stmt = conn + .prepare("SELECT community_id, symbol_id FROM community_members") + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let member_rows = member_stmt + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; - communities.push(CommunityResponse { - id, - label: name.unwrap_or_default(), - cohesion: cohesion.unwrap_or(0.0), - member_count: members.len(), - members, - }); + let mut members_by_community: HashMap> = HashMap::new(); + for row in member_rows { + let (community_id, symbol_id) = row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + members_by_community + .entry(community_id) + .or_default() + .push(symbol_id); } + let communities: Vec = community_info + .into_iter() + .map(|(id, label, cohesion)| { + let members = members_by_community.remove(&id).unwrap_or_default(); + CommunityResponse { + id, + label, + cohesion, + member_count: members.len(), + members, + } + }) + .collect(); + Ok(Json(communities)) } @@ -345,12 +359,12 @@ async fn list_processes( State(state): State, ) -> Result>, ApiError> { let conn = state.db.connection(); - let mut processes = Vec::new(); - let mut stmt = conn + // Load all processes + let mut proc_stmt = conn .prepare("SELECT id, name, entry_symbol_id FROM processes") .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let rows = stmt + let proc_rows = proc_stmt .query_map([], |row| { Ok(( row.get::<_, i64>(0)?, @@ -360,42 +374,55 @@ async fn list_processes( }) .map_err(|e| CodeilusError::Database(Box::new(e)))?; - for row in rows { - let (id, name, entry_symbol_id) = - row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + let mut process_info: Vec<(i64, String, i64)> = Vec::new(); + for row in proc_rows { + let (id, name, entry_symbol_id) = row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + process_info.push((id, name.unwrap_or_default(), entry_symbol_id.unwrap_or(0))); + } - // Fetch steps with symbol names - let mut step_stmt = conn - .prepare( - "SELECT ps.step_order, ps.symbol_id, COALESCE(s.name, ''), COALESCE(p.description, '') \ - FROM process_steps ps \ - LEFT JOIN symbols s ON s.id = ps.symbol_id \ - LEFT JOIN processes p ON p.id = ps.process_id \ - WHERE ps.process_id = ?1 \ - ORDER BY ps.step_order", - ) - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let steps: Vec = step_stmt - .query_map(params![id], |row| { - Ok(ProcessStepResponse { - order: row.get(0)?, - symbol_id: row.get(1)?, - symbol_name: row.get(2)?, - description: row.get(3)?, - }) - }) - .map_err(|e| CodeilusError::Database(Box::new(e)))? - .collect::, _>>() - .map_err(|e| CodeilusError::Database(Box::new(e)))?; + // Batch-load all steps with symbol names in one query + let mut step_stmt = conn + .prepare( + "SELECT ps.process_id, ps.step_order, ps.symbol_id, COALESCE(s.name, ''), COALESCE(p.description, '') \ + FROM process_steps ps \ + LEFT JOIN symbols s ON s.id = ps.symbol_id \ + LEFT JOIN processes p ON p.id = ps.process_id \ + ORDER BY ps.process_id, ps.step_order", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let step_rows = step_stmt + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + ProcessStepResponse { + order: row.get(1)?, + symbol_id: row.get(2)?, + symbol_name: row.get(3)?, + description: row.get(4)?, + }, + )) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; - processes.push(ProcessResponse { - id, - name: name.unwrap_or_default(), - entry_symbol_id: entry_symbol_id.unwrap_or(0), - steps, - }); + let mut steps_by_process: HashMap> = HashMap::new(); + for row in step_rows { + let (process_id, step) = row.map_err(|e| CodeilusError::Database(Box::new(e)))?; + steps_by_process.entry(process_id).or_default().push(step); } + let processes: Vec = process_info + .into_iter() + .map(|(id, name, entry_symbol_id)| { + let steps = steps_by_process.remove(&id).unwrap_or_default(); + ProcessResponse { + id, + name, + entry_symbol_id, + steps, + } + }) + .collect(); + Ok(Json(processes)) } diff --git a/crates/codeilus-api/src/routes/learning.rs b/crates/codeilus-api/src/routes/learning.rs index 19f432c..adc37d2 100644 --- a/crates/codeilus-api/src/routes/learning.rs +++ b/crates/codeilus-api/src/routes/learning.rs @@ -206,10 +206,13 @@ async fn skip_chapter( )).into()); } + // Batch: get already-completed sections in one query, then only insert missing + let completed = progress_repo.list_completed_sections(chapter_id)?; + let completed_set: std::collections::HashSet<&str> = completed.iter().map(|s| s.as_str()).collect(); + let mut skipped = 0; for (_section_id, content_type) in §ions { - // Only mark incomplete sections - if !progress_repo.is_section_complete(chapter_id, content_type)? { + if !completed_set.contains(content_type.as_str()) { progress_repo.record_section(chapter_id, content_type)?; skipped += 1; } diff --git a/crates/codeilus-api/src/routes/mod.rs b/crates/codeilus-api/src/routes/mod.rs index 055ebda..852f5a4 100644 --- a/crates/codeilus-api/src/routes/mod.rs +++ b/crates/codeilus-api/src/routes/mod.rs @@ -8,13 +8,48 @@ pub mod graph; pub mod health; pub mod learning; pub mod narratives; +pub mod schematic; pub mod search; pub mod symbols; pub mod ws; use axum::Router; +use axum::middleware::{self, Next}; +use axum::extract::Request; +use axum::response::Response; +use http::Method; +use serde::Deserialize; use crate::state::AppState; +/// Shared pagination query parameters for list endpoints. +#[derive(Deserialize)] +pub struct PaginationQuery { + pub limit: Option, + pub offset: Option, +} + +impl PaginationQuery { + /// Returns (limit, offset) clamped to safe defaults. + pub fn resolve(&self) -> (i64, i64) { + let limit = self.limit.unwrap_or(50).clamp(1, 200); + let offset = self.offset.unwrap_or(0).max(0); + (limit, offset) + } +} + +/// Middleware that adds Cache-Control headers to GET responses. +async fn cache_control_middleware(request: Request, next: Next) -> Response { + let is_get = request.method() == Method::GET; + let mut response = next.run(request).await; + if is_get { + response.headers_mut().insert( + http::header::CACHE_CONTROL, + http::HeaderValue::from_static("public, max-age=300, stale-while-revalidate=60"), + ); + } + response +} + pub fn router() -> Router { Router::new() .merge(health::routes()) @@ -27,5 +62,7 @@ pub fn router() -> Router { .merge(narratives::router()) .merge(chapters::router()) .merge(annotations::router()) + .merge(schematic::router()) .merge(learning::router()) + .layer(middleware::from_fn(cache_control_middleware)) } diff --git a/crates/codeilus-api/src/routes/schematic.rs b/crates/codeilus-api/src/routes/schematic.rs new file mode 100644 index 0000000..628ae3b --- /dev/null +++ b/crates/codeilus-api/src/routes/schematic.rs @@ -0,0 +1,105 @@ +//! Schematic API: unified tree + graph + learning endpoint. + +use std::sync::Arc; + +use axum::extract::{Query, State}; +use axum::routing::get; +use axum::{Json, Router}; +use serde::Deserialize; + +use codeilus_db::SchematicRepo; + +use crate::error::ApiError; +use crate::state::AppState; + +#[derive(Deserialize)] +pub struct SchematicQuery { + pub depth: Option, + pub community_id: Option, + pub include_symbols: Option, + pub include_edges: Option, +} + +#[derive(Deserialize)] +pub struct ExpandQuery { + pub node_id: String, + pub include_symbols: Option, + pub include_edges: Option, +} + +#[derive(Deserialize)] +pub struct DetailQuery { + pub node_id: String, + pub include_source: Option, +} + +async fn get_schematic( + State(state): State, + Query(query): Query, +) -> Result, ApiError> { + let depth = query.depth.unwrap_or(2).min(10); + let include_symbols = query.include_symbols.unwrap_or(false); + let include_edges = query.include_edges.unwrap_or(false); + + let cache_key = format!( + "schematic:d={}:c={:?}:s={}:e={}", + depth, query.community_id, include_symbols, include_edges + ); + + if let Some(cached) = state.cache.json.get(&cache_key) { + return Ok(Json(cached)); + } + + let repo = SchematicRepo::new(Arc::clone(&state.db)); + let result = repo.get_schematic(depth, query.community_id, include_symbols, include_edges)?; + + let value = serde_json::to_value(&result) + .map_err(|e| ApiError::from(codeilus_core::CodeilusError::Internal(format!("serialize: {}", e))))?; + state.cache.json.insert(cache_key, value.clone()); + + Ok(Json(value)) +} + +async fn expand_node( + State(state): State, + Query(query): Query, +) -> Result, ApiError> { + let include_symbols = query.include_symbols.unwrap_or(true); + let include_edges = query.include_edges.unwrap_or(true); + + let repo = SchematicRepo::new(Arc::clone(&state.db)); + let result = repo.expand_node(&query.node_id, include_symbols, include_edges)?; + + let value = serde_json::to_value(&result) + .map_err(|e| ApiError::from(codeilus_core::CodeilusError::Internal(format!("serialize: {}", e))))?; + + Ok(Json(value)) +} + +async fn get_detail( + State(state): State, + Query(query): Query, +) -> Result, ApiError> { + let include_source = query.include_source.unwrap_or(true); + + let cache_key = format!("schematic_detail:{}:s={}", query.node_id, include_source); + if let Some(cached) = state.cache.json.get(&cache_key) { + return Ok(Json(cached)); + } + + let repo = SchematicRepo::new(Arc::clone(&state.db)); + let result = repo.get_detail(&query.node_id, include_source)?; + + let value = serde_json::to_value(&result) + .map_err(|e| ApiError::from(codeilus_core::CodeilusError::Internal(format!("serialize: {}", e))))?; + state.cache.json.insert(cache_key, value.clone()); + + Ok(Json(value)) +} + +pub fn router() -> Router { + Router::new() + .route("/schematic", get(get_schematic)) + .route("/schematic/expand", get(expand_node)) + .route("/schematic/detail", get(get_detail)) +} diff --git a/crates/codeilus-api/src/routes/symbols.rs b/crates/codeilus-api/src/routes/symbols.rs index 55acd3b..f35866b 100644 --- a/crates/codeilus-api/src/routes/symbols.rs +++ b/crates/codeilus-api/src/routes/symbols.rs @@ -7,10 +7,8 @@ use serde::Deserialize; use std::sync::Arc; -use codeilus_core::error::CodeilusError; -use codeilus_core::ids::{FileId, SymbolId}; +use codeilus_core::ids::SymbolId; use codeilus_db::{SymbolRepo, SymbolRow}; -use rusqlite::params; use crate::error::ApiError; use crate::state::AppState; @@ -19,64 +17,29 @@ use crate::state::AppState; pub struct SymbolSearchQuery { pub q: Option, pub kind: Option, + pub limit: Option, + pub offset: Option, } -/// GET /api/v1/symbols — List all symbols, optional ?kind= filter +/// GET /api/v1/symbols — List symbols with pagination, optional ?kind= filter async fn list_symbols( State(state): State, Query(query): Query, -) -> Result>, ApiError> { - let conn = state.db.connection(); - let mut result = Vec::new(); - match query.kind { - Some(kind) => { - let mut stmt = conn - .prepare( - "SELECT id, file_id, name, kind, start_line, end_line, signature FROM symbols WHERE kind = ?1", - ) - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let rows = stmt - .query_map(params![kind], |row| { - Ok(SymbolRow { - id: SymbolId(row.get(0)?), - file_id: FileId(row.get(1)?), - name: row.get(2)?, - kind: row.get(3)?, - start_line: row.get(4)?, - end_line: row.get(5)?, - signature: row.get(6)?, - }) - }) - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - for row in rows { - result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); - } - } - None => { - let mut stmt = conn - .prepare( - "SELECT id, file_id, name, kind, start_line, end_line, signature FROM symbols", - ) - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - let rows = stmt - .query_map([], |row| { - Ok(SymbolRow { - id: SymbolId(row.get(0)?), - file_id: FileId(row.get(1)?), - name: row.get(2)?, - kind: row.get(3)?, - start_line: row.get(4)?, - end_line: row.get(5)?, - signature: row.get(6)?, - }) - }) - .map_err(|e| CodeilusError::Database(Box::new(e)))?; - for row in rows { - result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); - } - } +) -> Result, ApiError> { + let limit = query.limit.unwrap_or(50).clamp(1, 200); + let offset = query.offset.unwrap_or(0).max(0); + let cache_key = format!("symbols:k={:?}:l={}:o={}", query.kind, limit, offset); + + if let Some(cached) = state.cache.json.get(&cache_key) { + return Ok(Json(cached)); } - Ok(Json(result)) + + let repo = SymbolRepo::new(Arc::clone(&state.db)); + let results = repo.list_paginated(query.kind.as_deref(), limit, offset)?; + let value = serde_json::to_value(&results) + .map_err(|e| ApiError::from(codeilus_core::error::CodeilusError::Internal(e.to_string())))?; + state.cache.json.insert(cache_key, value.clone()); + Ok(Json(value)) } /// GET /api/v1/symbols/:id — Get a single symbol by ID diff --git a/crates/codeilus-api/src/state.rs b/crates/codeilus-api/src/state.rs index 6a1a5da..33a2af8 100644 --- a/crates/codeilus-api/src/state.rs +++ b/crates/codeilus-api/src/state.rs @@ -24,7 +24,7 @@ impl AppCache { pub fn new() -> Self { Self { json: Cache::builder() - .max_capacity(100) + .max_capacity(500) .time_to_live(Duration::from_secs(600)) // 10 min TTL .build(), } diff --git a/crates/codeilus-db/src/lib.rs b/crates/codeilus-db/src/lib.rs index 7f12573..3ac63c8 100644 --- a/crates/codeilus-db/src/lib.rs +++ b/crates/codeilus-db/src/lib.rs @@ -13,7 +13,7 @@ pub use repos::{ CommunityRow, EdgeRepo, EdgeRow, FileMetricsRepo, FileMetricsRow, FileRepo, FileRow, HarvestRepoRepo, HarvestRepoRow, LearnerStatsRow, NarrativeRepo, NarrativeRow, PatternRepo, PatternRow, ProcessRepo, ProcessRow, ProcessStepRow, ProgressRepo, ProgressRow, - PipelineRepo, QuizQuestionRow, QuizRepo, SymbolRepo, SymbolRow, + PipelineRepo, QuizQuestionRow, QuizRepo, SchematicRepo, SymbolRepo, SymbolRow, }; use std::collections::HashMap; diff --git a/crates/codeilus-db/src/migrations.rs b/crates/codeilus-db/src/migrations.rs index 380a58a..4b6d19b 100644 --- a/crates/codeilus-db/src/migrations.rs +++ b/crates/codeilus-db/src/migrations.rs @@ -13,6 +13,7 @@ const MIGRATION_006: &str = include_str!("../../../migrations/0006_seed_badges.s const MIGRATION_007: &str = include_str!("../../../migrations/0007_narrative_placeholder.sql"); const MIGRATION_008: &str = include_str!("../../../migrations/0008_content_hash.sql"); const MIGRATION_009: &str = include_str!("../../../migrations/0009_pipeline_runs.sql"); +const MIGRATION_010: &str = include_str!("../../../migrations/0010_add_indexes.sql"); pub struct Migrator<'a> { conn: &'a Connection, @@ -172,6 +173,15 @@ impl<'a> Migrator<'a> { info!("migration 0009 applied, now at version 9"); } + if current < 10 { + info!("applying migration 0010_add_indexes.sql"); + self.conn + .execute_batch(MIGRATION_010) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + applied += 1; + info!("migration 0010 applied, now at version 10"); + } + if applied == 0 { info!(version = current, "schema already at latest version"); } diff --git a/crates/codeilus-db/src/repos/file_repo.rs b/crates/codeilus-db/src/repos/file_repo.rs index 836c503..36c8685 100644 --- a/crates/codeilus-db/src/repos/file_repo.rs +++ b/crates/codeilus-db/src/repos/file_repo.rs @@ -163,6 +163,62 @@ impl FileRepo { Ok(result) } + /// List files with pagination. Optional language filter. + pub fn list_paginated( + &self, + language: Option<&str>, + limit: i64, + offset: i64, + ) -> CodeilusResult> { + let conn = self.db.connection(); + let mut result = Vec::new(); + match language { + Some(lang) => { + let mut stmt = conn + .prepare( + "SELECT id, path, language, COALESCE(sloc, 0), last_modified FROM files WHERE language = ?1 LIMIT ?2 OFFSET ?3", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let rows = stmt + .query_map(params![lang, limit, offset], |row| { + Ok(FileRow { + id: FileId(row.get(0)?), + path: row.get(1)?, + language: row.get(2)?, + sloc: row.get(3)?, + last_modified: row.get(4)?, + }) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + for row in rows { + result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); + } + } + None => { + let mut stmt = conn + .prepare( + "SELECT id, path, language, COALESCE(sloc, 0), last_modified FROM files LIMIT ?1 OFFSET ?2", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let rows = stmt + .query_map(params![limit, offset], |row| { + Ok(FileRow { + id: FileId(row.get(0)?), + path: row.get(1)?, + language: row.get(2)?, + sloc: row.get(3)?, + last_modified: row.get(4)?, + }) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + for row in rows { + result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); + } + } + } + Ok(result) + } + /// Count total files. pub fn count(&self) -> CodeilusResult { let conn = self.db.connection(); diff --git a/crates/codeilus-db/src/repos/mod.rs b/crates/codeilus-db/src/repos/mod.rs index 906b23b..895464a 100644 --- a/crates/codeilus-db/src/repos/mod.rs +++ b/crates/codeilus-db/src/repos/mod.rs @@ -33,3 +33,5 @@ pub mod annotation_repo; pub use annotation_repo::{AnnotationRepo, AnnotationRow}; pub mod pipeline_repo; pub use pipeline_repo::PipelineRepo; +pub mod schematic_repo; +pub use schematic_repo::SchematicRepo; diff --git a/crates/codeilus-db/src/repos/narrative_repo.rs b/crates/codeilus-db/src/repos/narrative_repo.rs index 966bc5a..7b97814 100644 --- a/crates/codeilus-db/src/repos/narrative_repo.rs +++ b/crates/codeilus-db/src/repos/narrative_repo.rs @@ -159,6 +159,34 @@ impl NarrativeRepo { Ok(result) } + /// List narratives with pagination. + pub fn list_paginated(&self, limit: i64, offset: i64) -> CodeilusResult> { + let conn = self.db.connection(); + let mut stmt = conn + .prepare( + "SELECT id, kind, target_id, language, content, generated_at, is_placeholder FROM narratives LIMIT ?1 OFFSET ?2", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let rows = stmt + .query_map(params![limit, offset], |row| { + Ok(NarrativeRow { + id: row.get(0)?, + kind: row.get(1)?, + target_id: row.get(2)?, + language: row.get(3)?, + content: row.get(4)?, + generated_at: row.get(5)?, + is_placeholder: row.get(6)?, + }) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let mut result = Vec::new(); + for row in rows { + result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); + } + Ok(result) + } + /// List narratives by kind. pub fn list_by_kind(&self, kind: &str) -> CodeilusResult> { let conn = self.db.connection(); diff --git a/crates/codeilus-db/src/repos/schematic_repo.rs b/crates/codeilus-db/src/repos/schematic_repo.rs new file mode 100644 index 0000000..3d1d44a --- /dev/null +++ b/crates/codeilus-db/src/repos/schematic_repo.rs @@ -0,0 +1,752 @@ +//! Schematic repository: depth-limited tree with community + learning enrichment. + +use codeilus_core::error::{CodeilusError, CodeilusResult}; +use rusqlite::params; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::pool::DbPool; + +#[derive(Debug, Clone, Serialize)] +pub struct SchematicNode { + pub id: String, + #[serde(rename = "type")] + pub node_type: String, + pub label: String, + pub parent_id: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub file_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub symbol_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sloc: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub community_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub community_label: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub community_color: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub chapter_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chapter_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub difficulty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, + + pub has_children: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub child_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub symbol_count: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ProgressInfo { + pub completed: i64, + pub total: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SchematicEdge { + pub id: String, + pub source: String, + pub target: String, + #[serde(rename = "type")] + pub edge_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CommunityInfo { + pub id: i64, + pub label: String, + pub color: String, + pub cohesion: f64, + pub member_count: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub chapter_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chapter_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub difficulty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SchematicMeta { + pub total_files: i64, + pub total_symbols: i64, + pub total_communities: i64, + pub depth_returned: u32, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SchematicResponse { + pub nodes: Vec, + pub edges: Vec, + pub communities: Vec, + pub meta: SchematicMeta, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SchematicDetail { + pub node_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub narrative: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub narrative_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + pub callers: Vec, + pub callees: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub chapter: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SourceInfo { + pub path: String, + pub language: Option, + pub lines: Vec, + pub total_lines: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SourceLine { + pub number: i64, + pub content: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RelatedSymbol { + pub id: String, + pub name: String, + pub kind: String, + pub file_path: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChapterInfo { + pub id: i64, + pub title: String, + pub difficulty: String, + pub progress: ProgressInfo, +} + +const COMMUNITY_COLORS: &[&str] = &[ + "#6366f1", "#ec4899", "#14b8a6", "#f59e0b", "#8b5cf6", + "#06b6d4", "#f97316", "#84cc16", "#ef4444", "#a855f7", +]; + +fn community_color(id: i64) -> String { + COMMUNITY_COLORS[(id as usize) % COMMUNITY_COLORS.len()].to_string() +} + +pub struct SchematicRepo { + db: Arc, +} + +impl SchematicRepo { + pub fn new(db: Arc) -> Self { + Self { db } + } + + /// Build the schematic tree up to `depth` levels. + pub fn get_schematic( + &self, + depth: u32, + community_filter: Option, + include_symbols: bool, + include_edges: bool, + ) -> CodeilusResult { + let conn = self.db.connection(); + + // 1. Load files (with optional community filter) + let files: Vec<(i64, String, Option, i64)> = if let Some(cid) = community_filter { + let mut stmt = conn.prepare( + "SELECT DISTINCT f.id, f.path, f.language, f.sloc + FROM files f + JOIN symbols s ON s.file_id = f.id + JOIN community_members cm ON cm.symbol_id = s.id + WHERE cm.community_id = ?1 + ORDER BY f.path" + ).map_err(db_err)?; + let rows = stmt.query_map(params![cid], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + } else { + let mut stmt = conn.prepare( + "SELECT id, path, language, sloc FROM files ORDER BY path" + ).map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 2. Load symbol → community mapping + let sym_community: HashMap = { + let mut stmt = conn.prepare("SELECT symbol_id, community_id FROM community_members").map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 3. Load community info + let communities_raw: Vec<(i64, Option, f64)> = { + let mut stmt = conn.prepare("SELECT id, name, cohesion_score FROM communities").map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get::<_, f64>(2).unwrap_or(0.0)))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 4. Community member counts + let community_member_counts: HashMap = { + let mut stmt = conn.prepare("SELECT community_id, COUNT(*) FROM community_members GROUP BY community_id").map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 5. Chapters linked to communities + let community_chapters: HashMap = { + let mut stmt = conn.prepare("SELECT community_id, id, title, difficulty FROM chapters WHERE community_id IS NOT NULL").map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, (r.get(1)?, r.get(2)?, r.get(3)?)))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 6. Chapter progress + let chapter_progress: HashMap = { + let mut stmt = conn.prepare( + "SELECT cs.chapter_id, + COUNT(cs.id) as total, + SUM(CASE WHEN p.completed = 1 THEN 1 ELSE 0 END) as done + FROM chapter_sections cs + LEFT JOIN progress p ON p.chapter_id = cs.chapter_id AND p.section_id = cs.id + GROUP BY cs.chapter_id" + ).map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, (r.get::<_, i64>(1)?, r.get::<_, i64>(2)?)))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // 7. File → dominant community (most symbols in same community) + let file_dominant_community: HashMap = { + let mut stmt = conn.prepare( + "SELECT s.file_id, cm.community_id, COUNT(*) as cnt + FROM symbols s + JOIN community_members cm ON cm.symbol_id = s.id + GROUP BY s.file_id, cm.community_id + ORDER BY s.file_id, cnt DESC" + ).map_err(db_err)?; + let rows: Vec<(i64, i64, i64)> = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + let mut result = HashMap::new(); + for (file_id, comm_id, _cnt) in rows { + result.entry(file_id).or_insert(comm_id); + } + result + }; + + // 8. Symbol counts per file + let file_symbol_counts: HashMap = { + let mut stmt = conn.prepare("SELECT file_id, COUNT(*) FROM symbols GROUP BY file_id").map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, usize>(1)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + + // Build community info list + let community_map: HashMap = communities_raw.iter() + .map(|(id, name, _)| (*id, name.clone().unwrap_or_else(|| format!("community_{}", id)))) + .collect(); + + let communities: Vec = communities_raw.iter().map(|(id, name, cohesion)| { + let label = name.clone().unwrap_or_else(|| format!("community_{}", id)); + let (chap_id, chap_title, diff) = community_chapters.get(id) + .map(|(cid, title, d)| (Some(*cid), Some(title.clone()), Some(d.clone()))) + .unwrap_or((None, None, None)); + let progress = chap_id.and_then(|cid| chapter_progress.get(&cid)) + .map(|(total, completed)| ProgressInfo { completed: *completed, total: *total }); + CommunityInfo { + id: *id, label, color: community_color(*id), cohesion: *cohesion, + member_count: *community_member_counts.get(id).unwrap_or(&0), + chapter_id: chap_id, chapter_title: chap_title, difficulty: diff, progress, + } + }).collect(); + + // Build directory tree nodes + let mut nodes: Vec = Vec::new(); + let mut dir_children: HashMap> = HashMap::new(); // dir_id → child dir_ids + + // Root node + nodes.push(SchematicNode { + id: "dir:.".into(), node_type: "directory".into(), label: ".".into(), + parent_id: None, has_children: true, child_count: None, symbol_count: None, + file_id: None, symbol_id: None, language: None, sloc: None, + kind: None, signature: None, + community_id: None, community_label: None, community_color: None, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + + // Find common prefix for path normalization + let common_prefix = if files.len() > 1 { + let paths: Vec<&str> = files.iter().map(|(_, p, _, _)| p.as_str()).collect(); + find_common_dir_prefix(&paths) + } else { + String::new() + }; + + // Process files into directory tree + let mut all_dirs: HashMap = HashMap::new(); + all_dirs.insert(".".to_string(), true); + + for (fid, path, lang, sloc) in &files { + let clean = path.strip_prefix(&common_prefix).unwrap_or(path); + let clean = clean.strip_prefix('/').unwrap_or(clean); + let clean = clean.strip_prefix("./").unwrap_or(clean); + let parts: Vec<&str> = clean.split('/').collect(); + + // Create intermediate directory nodes + let mut current_dir = ".".to_string(); + for part in &parts[..parts.len() - 1] { + let dir_path = if current_dir == "." { + (*part).to_string() + } else { + format!("{}/{}", current_dir, part) + }; + let dir_id = format!("dir:{}", dir_path); + let parent_id = format!("dir:{}", current_dir); + + let depth_of_dir = dir_path.matches('/').count() as u32 + 1; + if depth_of_dir <= depth && !all_dirs.contains_key(&dir_path) { + all_dirs.insert(dir_path.clone(), true); + nodes.push(SchematicNode { + id: dir_id.clone(), node_type: "directory".into(), + label: (*part).to_string(), parent_id: Some(parent_id.clone()), + has_children: true, child_count: None, symbol_count: None, + file_id: None, symbol_id: None, language: None, sloc: None, + kind: None, signature: None, + community_id: None, community_label: None, community_color: None, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + dir_children.entry(parent_id).or_default().push(dir_id); + } + current_dir = dir_path; + } + + // Add file node if within depth + let file_depth = parts.len() as u32 - 1; + if file_depth <= depth { + let file_id_str = format!("file:{}", fid); + let parent_id = format!("dir:{}", current_dir); + let comm_id = file_dominant_community.get(fid); + let comm_label = comm_id.and_then(|c| community_map.get(c)).cloned(); + let comm_color = comm_id.map(|c| community_color(*c)); + + nodes.push(SchematicNode { + id: file_id_str, node_type: "file".into(), + label: parts.last().unwrap_or(&"").to_string(), + parent_id: Some(parent_id), + has_children: file_symbol_counts.get(fid).copied().unwrap_or(0) > 0, + child_count: None, + symbol_count: file_symbol_counts.get(fid).copied(), + file_id: Some(*fid), symbol_id: None, + language: lang.clone(), sloc: Some(*sloc), + kind: None, signature: None, + community_id: comm_id.copied(), + community_label: comm_label, + community_color: comm_color, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + } + } + + // Optionally load symbols (filtered by community if set) + if include_symbols { + #[allow(clippy::type_complexity)] + let syms: Vec<(i64, i64, String, String, Option, Option, Option)> = if let Some(cid) = community_filter { + let mut stmt = conn.prepare( + "SELECT s.id, s.file_id, s.name, s.kind, s.start_line, s.end_line, s.signature + FROM symbols s + JOIN community_members cm ON cm.symbol_id = s.id + WHERE cm.community_id = ?1 + ORDER BY s.file_id, s.start_line" + ).map_err(db_err)?; + let rows = stmt.query_map(params![cid], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + } else { + let mut stmt = conn.prepare( + "SELECT id, file_id, name, kind, start_line, end_line, signature FROM symbols ORDER BY file_id, start_line" + ).map_err(db_err)?; + let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + }; + for (sid, fid, name, kind, _start, _end, sig) in syms { + let comm_id = sym_community.get(&sid); + let comm_label = comm_id.and_then(|c| community_map.get(c)).cloned(); + let comm_color = comm_id.map(|c| community_color(*c)); + nodes.push(SchematicNode { + id: format!("sym:{}", sid), node_type: "symbol".into(), + label: name, parent_id: Some(format!("file:{}", fid)), + has_children: false, child_count: None, symbol_count: None, + file_id: Some(fid), symbol_id: Some(sid), + language: None, sloc: None, + kind: Some(kind), signature: sig, + community_id: comm_id.copied(), + community_label: comm_label, + community_color: comm_color, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + } + + } + + // Enrich directory nodes with dominant community using ALL files + { + let mut dir_community_counts: HashMap> = HashMap::new(); + // Use all files (not just returned nodes) to compute dir communities + for (fid, path, _lang, _sloc) in &files { + if let Some(&comm_id) = file_dominant_community.get(fid) { + let clean = path.strip_prefix("./").unwrap_or(path); + let parts: Vec<&str> = clean.split('/').collect(); + // Attribute to every ancestor directory + let mut dir_path = ".".to_string(); + for i in 0..parts.len() - 1 { + dir_path = if dir_path == "." { parts[i].to_string() } else { format!("{}/{}", dir_path, parts[i]) }; + let dir_id = format!("dir:{}", dir_path); + *dir_community_counts.entry(dir_id).or_default().entry(comm_id).or_default() += 1; + } + } + } + for node in &mut nodes { + if node.node_type == "directory" && node.community_id.is_none() { + if let Some(counts) = dir_community_counts.get(&node.id) { + if let Some((&best_id, _)) = counts.iter().max_by_key(|(_, c)| *c) { + node.community_id = Some(best_id); + node.community_label = community_map.get(&best_id).cloned(); + node.community_color = Some(community_color(best_id)); + } + } + } + } + } + + // Optionally load edges (filtered by community if set) + let edges = if include_edges { + if let Some(cid) = community_filter { + // Only edges where both endpoints are in this community + let mut stmt = conn.prepare( + "SELECT e.id, e.source_id, e.target_id, e.kind, e.confidence + FROM edges e + JOIN community_members cm1 ON cm1.symbol_id = e.source_id AND cm1.community_id = ?1 + JOIN community_members cm2 ON cm2.symbol_id = e.target_id AND cm2.community_id = ?1" + ).map_err(db_err)?; + let rows = stmt.query_map(params![cid], |r| { + Ok(SchematicEdge { + id: format!("e:{}", r.get::<_, i64>(0)?), + source: format!("sym:{}", r.get::<_, i64>(1)?), + target: format!("sym:{}", r.get::<_, i64>(2)?), + edge_type: r.get(3)?, + confidence: r.get(4).ok(), + }) + }).map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + } else { + let mut stmt = conn.prepare("SELECT id, source_id, target_id, kind, confidence FROM edges").map_err(db_err)?; + let rows = stmt.query_map([], |r| { + Ok(SchematicEdge { + id: format!("e:{}", r.get::<_, i64>(0)?), + source: format!("sym:{}", r.get::<_, i64>(1)?), + target: format!("sym:{}", r.get::<_, i64>(2)?), + edge_type: r.get(3)?, + confidence: r.get(4).ok(), + }) + }).map_err(db_err)?.collect::, _>>().map_err(db_err)?; + rows + } + } else { + Vec::new() + }; + + // Counts + let total_files: i64 = conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)).unwrap_or(0); + let total_symbols: i64 = conn.query_row("SELECT COUNT(*) FROM symbols", [], |r| r.get(0)).unwrap_or(0); + let total_communities: i64 = conn.query_row("SELECT COUNT(*) FROM communities", [], |r| r.get(0)).unwrap_or(0); + + Ok(SchematicResponse { + nodes, + edges, + communities, + meta: SchematicMeta { total_files, total_symbols, total_communities, depth_returned: depth }, + }) + } + + /// Expand children of a specific node. + pub fn expand_node( + &self, + node_id: &str, + include_symbols: bool, + include_edges: bool, + ) -> CodeilusResult { + let conn = self.db.connection(); + + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + + if let Some(dir_path) = node_id.strip_prefix("dir:") { + // Load ALL files, find common prefix, then filter to this directory's children + let mut stmt = conn.prepare("SELECT id, path, language, sloc FROM files ORDER BY path") + .map_err(db_err)?; + let all_files: Vec<(i64, String, Option, i64)> = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + + // Find common prefix to normalize paths (handles both "./crates/..." and "/abs/path/..." ) + let common_prefix = if all_files.len() > 1 { + let paths: Vec<&str> = all_files.iter().map(|(_, p, _, _)| p.as_str()).collect(); + find_common_dir_prefix(&paths) + } else { + String::new() + }; + + // Extract direct children: subdirs and files + let mut seen_dirs: HashMap = HashMap::new(); + for (fid, path, lang, sloc) in &all_files { + let clean = path.strip_prefix(&common_prefix).unwrap_or(path); + let clean = clean.strip_prefix('/').unwrap_or(clean); + let clean = clean.strip_prefix("./").unwrap_or(clean); + + let relative = if dir_path == "." { + clean.to_string() + } else { + match clean.strip_prefix(&format!("{}/", dir_path)) { + Some(r) => r.to_string(), + None => continue, + } + }; + + let parts: Vec<&str> = relative.split('/').collect(); + if parts.len() == 1 && !parts[0].is_empty() { + nodes.push(SchematicNode { + id: format!("file:{}", fid), node_type: "file".into(), + label: parts[0].to_string(), parent_id: Some(node_id.to_string()), + has_children: false, child_count: None, symbol_count: None, + file_id: Some(*fid), symbol_id: None, + language: lang.clone(), sloc: Some(*sloc), + kind: None, signature: None, + community_id: None, community_label: None, community_color: None, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + } else if parts.len() > 1 { + let sub_dir = parts[0]; + let sub_dir_path = if dir_path == "." { sub_dir.to_string() } else { format!("{}/{}", dir_path, sub_dir) }; + if !seen_dirs.contains_key(&sub_dir_path) { + seen_dirs.insert(sub_dir_path.clone(), true); + nodes.push(SchematicNode { + id: format!("dir:{}", sub_dir_path), node_type: "directory".into(), + label: sub_dir.to_string(), parent_id: Some(node_id.to_string()), + has_children: true, child_count: None, symbol_count: None, + file_id: None, symbol_id: None, language: None, sloc: None, + kind: None, signature: None, + community_id: None, community_label: None, community_color: None, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + } + } + } + } else if let Some(file_id_str) = node_id.strip_prefix("file:") { + let file_id: i64 = file_id_str.parse().map_err(|_| CodeilusError::Validation("invalid file id".into()))?; + if include_symbols { + let mut stmt = conn.prepare( + "SELECT id, name, kind, start_line, end_line, signature FROM symbols WHERE file_id = ?1 ORDER BY start_line" + ).map_err(db_err)?; + #[allow(clippy::type_complexity)] + let syms: Vec<(i64, String, String, Option, Option, Option)> = stmt + .query_map(params![file_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?))) + .map_err(db_err)?.collect::, _>>().map_err(db_err)?; + + let sym_ids: Vec = syms.iter().map(|(id, ..)| *id).collect(); + + for (sid, name, kind, _start, _end, sig) in syms { + nodes.push(SchematicNode { + id: format!("sym:{}", sid), node_type: "symbol".into(), + label: name, parent_id: Some(node_id.to_string()), + has_children: false, child_count: None, symbol_count: None, + file_id: Some(file_id), symbol_id: Some(sid), + language: None, sloc: None, + kind: Some(kind), signature: sig, + community_id: None, community_label: None, community_color: None, + chapter_id: None, chapter_title: None, difficulty: None, progress: None, + }); + } + + if include_edges && !sym_ids.is_empty() { + let placeholders: String = sym_ids.iter().map(|_| "?").collect::>().join(","); + let sql = format!( + "SELECT id, source_id, target_id, kind, confidence FROM edges WHERE source_id IN ({0}) OR target_id IN ({0})", + placeholders + ); + let mut stmt = conn.prepare(&sql).map_err(db_err)?; + let params: Vec> = sym_ids.iter().map(|id| Box::new(*id) as Box).collect(); + let double_params: Vec<&dyn rusqlite::types::ToSql> = params.iter().chain(params.iter()).map(|b| b.as_ref()).collect(); + edges = stmt.query_map(double_params.as_slice(), |r| { + Ok(SchematicEdge { + id: format!("e:{}", r.get::<_, i64>(0)?), + source: format!("sym:{}", r.get::<_, i64>(1)?), + target: format!("sym:{}", r.get::<_, i64>(2)?), + edge_type: r.get(3)?, + confidence: r.get(4).ok(), + }) + }).map_err(db_err)?.collect::, _>>().map_err(db_err)?; + } + } + } + + Ok(SchematicResponse { + nodes, + edges, + communities: Vec::new(), + meta: SchematicMeta { total_files: 0, total_symbols: 0, total_communities: 0, depth_returned: 0 }, + }) + } + + /// Get detailed info for a node (narrative, source, callers/callees, chapter). + pub fn get_detail(&self, node_id: &str, include_source: bool) -> CodeilusResult { + let conn = self.db.connection(); + + if let Some(sym_id_str) = node_id.strip_prefix("sym:") { + let sym_id: i64 = sym_id_str.parse().map_err(|_| CodeilusError::Validation("invalid symbol id".into()))?; + + // Get symbol info + let (file_id, _name, _kind, _start_line, _end_line): (i64, String, String, Option, Option) = conn.query_row( + "SELECT file_id, name, kind, start_line, end_line FROM symbols WHERE id = ?1", + params![sym_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)) + ).map_err(db_err)?; + + // Narrative + let narrative: Option<(String, String)> = conn.query_row( + "SELECT content, kind FROM narratives WHERE target_id = ?1 AND kind = 'symbol_explanation'", + params![sym_id], |r| Ok((r.get(0)?, r.get(1)?)) + ).ok(); + + // Source + let source = if include_source { + let (path, lang): (String, Option) = conn.query_row( + "SELECT path, language FROM files WHERE id = ?1", params![file_id], |r| Ok((r.get(0)?, r.get(1)?)) + ).map_err(db_err)?; + // Read source file if repo_root available — for now return empty + Some(SourceInfo { path, language: lang, lines: Vec::new(), total_lines: 0 }) + } else { None }; + + // Callers + let callers = self.get_related_symbols(&conn, sym_id, true)?; + let callees = self.get_related_symbols(&conn, sym_id, false)?; + + // Chapter (via community) + let chapter = self.get_symbol_chapter(&conn, sym_id)?; + + Ok(SchematicDetail { + node_id: node_id.to_string(), + narrative: narrative.as_ref().map(|(c, _)| c.clone()), + narrative_kind: narrative.map(|(_, k)| k), + source, callers, callees, chapter, + }) + } else if let Some(file_id_str) = node_id.strip_prefix("file:") { + let file_id: i64 = file_id_str.parse().map_err(|_| CodeilusError::Validation("invalid file id".into()))?; + + let narrative: Option<(String, String)> = conn.query_row( + "SELECT content, kind FROM narratives WHERE target_id = ?1 AND kind = 'file_overview'", + params![file_id], |r| Ok((r.get(0)?, r.get(1)?)) + ).ok(); + + Ok(SchematicDetail { + node_id: node_id.to_string(), + narrative: narrative.as_ref().map(|(c, _)| c.clone()), + narrative_kind: narrative.map(|(_, k)| k), + source: None, callers: Vec::new(), callees: Vec::new(), chapter: None, + }) + } else { + Ok(SchematicDetail { + node_id: node_id.to_string(), + narrative: None, narrative_kind: None, source: None, + callers: Vec::new(), callees: Vec::new(), chapter: None, + }) + } + } + + fn get_related_symbols(&self, conn: &rusqlite::Connection, sym_id: i64, callers: bool) -> CodeilusResult> { + let sql = if callers { + "SELECT s.id, s.name, s.kind, f.path FROM edges e JOIN symbols s ON s.id = e.source_id JOIN files f ON f.id = s.file_id WHERE e.target_id = ?1 LIMIT 20" + } else { + "SELECT s.id, s.name, s.kind, f.path FROM edges e JOIN symbols s ON s.id = e.target_id JOIN files f ON f.id = s.file_id WHERE e.source_id = ?1 LIMIT 20" + }; + let mut stmt = conn.prepare(sql).map_err(db_err)?; + let rows = stmt.query_map(params![sym_id], |r| Ok(RelatedSymbol { + id: format!("sym:{}", r.get::<_, i64>(0)?), + name: r.get(1)?, kind: r.get(2)?, file_path: r.get(3)?, + })).map_err(db_err)?.collect::, _>>().map_err(db_err)?; + Ok(rows) + } + + fn get_symbol_chapter(&self, conn: &rusqlite::Connection, sym_id: i64) -> CodeilusResult> { + let comm_id: Option = conn.query_row( + "SELECT community_id FROM community_members WHERE symbol_id = ?1 LIMIT 1", + params![sym_id], |r| r.get(0) + ).ok(); + + if let Some(cid) = comm_id { + let chapter: Option<(i64, String, String)> = conn.query_row( + "SELECT id, title, difficulty FROM chapters WHERE community_id = ?1", + params![cid], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)) + ).ok(); + + if let Some((ch_id, title, diff)) = chapter { + let (total, completed): (i64, i64) = conn.query_row( + "SELECT COUNT(cs.id), SUM(CASE WHEN p.completed = 1 THEN 1 ELSE 0 END) + FROM chapter_sections cs + LEFT JOIN progress p ON p.chapter_id = cs.chapter_id AND p.section_id = cs.id + WHERE cs.chapter_id = ?1", + params![ch_id], |r| Ok((r.get(0)?, r.get::<_, i64>(1).unwrap_or(0))) + ).unwrap_or((0, 0)); + + return Ok(Some(ChapterInfo { id: ch_id, title, difficulty: diff, progress: ProgressInfo { completed, total } })); + } + } + Ok(None) + } +} + +fn db_err(e: rusqlite::Error) -> CodeilusError { + CodeilusError::Database(Box::new(e)) +} + +/// Find the longest common directory prefix among a set of file paths. +fn find_common_dir_prefix(paths: &[&str]) -> String { + if paths.is_empty() { return String::new(); } + let first = paths[0]; + let mut prefix_len = 0; + for (i, ch) in first.char_indices() { + if paths.iter().all(|p| p.as_bytes().get(i) == Some(&(ch as u8))) { + if ch == '/' { prefix_len = i + 1; } + } else { + break; + } + } + first[..prefix_len].to_string() +} diff --git a/crates/codeilus-db/src/repos/symbol_repo.rs b/crates/codeilus-db/src/repos/symbol_repo.rs index 9f9e774..3050d4e 100644 --- a/crates/codeilus-db/src/repos/symbol_repo.rs +++ b/crates/codeilus-db/src/repos/symbol_repo.rs @@ -186,6 +186,66 @@ impl SymbolRepo { Ok(result) } + /// List all symbols with pagination. Optional kind filter. + pub fn list_paginated( + &self, + kind: Option<&str>, + limit: i64, + offset: i64, + ) -> CodeilusResult> { + let conn = self.db.connection(); + let mut result = Vec::new(); + match kind { + Some(k) => { + let mut stmt = conn + .prepare( + "SELECT id, file_id, name, kind, start_line, end_line, signature FROM symbols WHERE kind = ?1 LIMIT ?2 OFFSET ?3", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let rows = stmt + .query_map(params![k, limit, offset], |row| { + Ok(SymbolRow { + id: SymbolId(row.get(0)?), + file_id: FileId(row.get(1)?), + name: row.get(2)?, + kind: row.get(3)?, + start_line: row.get(4)?, + end_line: row.get(5)?, + signature: row.get(6)?, + }) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + for row in rows { + result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); + } + } + None => { + let mut stmt = conn + .prepare( + "SELECT id, file_id, name, kind, start_line, end_line, signature FROM symbols LIMIT ?1 OFFSET ?2", + ) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + let rows = stmt + .query_map(params![limit, offset], |row| { + Ok(SymbolRow { + id: SymbolId(row.get(0)?), + file_id: FileId(row.get(1)?), + name: row.get(2)?, + kind: row.get(3)?, + start_line: row.get(4)?, + end_line: row.get(5)?, + signature: row.get(6)?, + }) + }) + .map_err(|e| CodeilusError::Database(Box::new(e)))?; + for row in rows { + result.push(row.map_err(|e| CodeilusError::Database(Box::new(e)))?); + } + } + } + Ok(result) + } + /// Count total symbols. pub fn count(&self) -> CodeilusResult { let conn = self.db.connection(); diff --git a/docs/OPTIMIZATIONS.md b/docs/OPTIMIZATIONS.md new file mode 100644 index 0000000..b8530d1 --- /dev/null +++ b/docs/OPTIMIZATIONS.md @@ -0,0 +1,181 @@ +# Codeilus — Optimization & Performance Plan + +> Audit date: 2026-03-18 + +## Backend (Rust/Axum) + +### 1. N+1 Query Problems — HIGH PRIORITY + +| Location | Issue | Fix | +|----------|-------|-----| +| `crates/codeilus-api/src/routes/graph.rs` — `list_communities()` | 1 query for communities + N queries for members (one per community) | Single query with JOIN or batch `IN (...)` clause | +| `crates/codeilus-api/src/routes/graph.rs` — `list_processes()` | 1 query for processes + N queries for steps | Same: JOIN or batch load | +| `crates/codeilus-api/src/routes/ask.rs` — `ask_stream()` | Loads context symbols one-by-one | Batch with `WHERE id IN (...)` | +| `crates/codeilus-api/src/routes/learning.rs` — `get_learner_stats()` | 3 separate queries (stats, completed chapters, badges) | Single query with JOINs | +| `crates/codeilus-api/src/routes/learning.rs` — `skip_chapter()` | Nested loop: fetch sections, check each, record each | Batch insert + aggregate completion check | + +### 2. Missing Database Indexes — HIGH PRIORITY + +Add to a new migration: + +```sql +CREATE INDEX IF NOT EXISTS idx_community_members_community ON community_members(community_id); +CREATE INDEX IF NOT EXISTS idx_community_members_symbol ON community_members(symbol_id); +CREATE INDEX IF NOT EXISTS idx_process_steps_process ON process_steps(process_id); +CREATE INDEX IF NOT EXISTS idx_chapters_community ON chapters(community_id); +CREATE INDEX IF NOT EXISTS idx_chapter_sections_chapter ON chapter_sections(chapter_id); +CREATE INDEX IF NOT EXISTS idx_progress_chapter ON progress(chapter_id); +``` + +### 3. Missing Pagination — HIGH PRIORITY + +These endpoints return all rows with no limit. Add `limit` (default 50, max 200) and `offset` query params: + +- `GET /api/v1/symbols` +- `GET /api/v1/files` +- `GET /api/v1/narratives` +- `GET /api/v1/chapters` +- `GET /api/v1/communities` +- `GET /api/v1/processes` +- `GET /api/v1/annotations` + +Already paginated (no change needed): `/api/v1/graph`, `/api/v1/search`. + +### 4. Expand Moka Cache — MEDIUM PRIORITY + +Current state: moka cache with 100-entry capacity and 10-minute TTL covers only graph, narratives, and chapters. + +**Add caching to:** + +| Endpoint | Cache Key | Suggested TTL | +|----------|-----------|---------------| +| `GET /api/v1/symbols` | `"symbols:l={limit}:o={offset}"` | 10 min | +| `GET /api/v1/files` | `"files:l={limit}:o={offset}"` | 10 min | +| `GET /api/v1/search?q=...` | `"search:q={query}:l={limit}"` | 5 min | +| `GET /api/v1/learner/stats` | `"learner:stats"` | 2 min | +| `GET /api/v1/files/:id/source` | `"file:source:{id}"` | 10 min | +| `GET /api/v1/chapters/:id/quiz` | `"quiz:{id}"` | 10 min | + +Also increase capacity from 100 to 500 entries. + +### 5. HTTP Caching Headers — MEDIUM PRIORITY + +Add middleware or per-route headers for read-only endpoints: + +``` +Cache-Control: public, max-age=300, stale-while-revalidate=60 +``` + +For mutable endpoints (progress, quiz answers): + +``` +Cache-Control: no-store +``` + +Consider adding `ETag` based on last-modified timestamp from the pipeline run. + +### 6. Cache Invalidation + +Current `invalidate_all()` is too coarse. Add key-prefix invalidation so that a pipeline re-run only clears stale keys (e.g., all keys starting with `"graph:"` or `"symbols:"`). + +--- + +## Frontend (SvelteKit) + +### 7. Client-Side API Cache — MEDIUM PRIORITY + +No caching exists in `frontend/src/lib/api.ts`. Every page visit re-fetches all data. + +**Approach:** Add a simple in-memory cache with TTL in `api.ts`: + +```typescript +const cache = new Map(); +const CACHE_TTL = 5 * 60 * 1000; // 5 minutes + +async function cachedFetch(url: string): Promise { + const entry = cache.get(url); + if (entry && entry.expires > Date.now()) return entry.data as T; + const data = await apiFetch(url); + cache.set(url, { data, expires: Date.now() + CACHE_TTL }); + return data; +} +``` + +Use for: files, symbols, communities, narratives, chapters (read-heavy, rarely change). + +### 8. Lazy-Load Shiki Languages — MEDIUM PRIORITY + +Currently loads all language packs upfront in the tree page. Instead, load only the language needed for the selected file: + +```typescript +const highlighter = await getHighlighter({ + themes: ['github-dark'], + langs: [detectedLanguage], // not all languages +}); +``` + +### 9. Vite Compression — MEDIUM PRIORITY + +Add brotli/gzip compression for production builds: + +```bash +pnpm add -D vite-plugin-compression +``` + +```typescript +// vite.config.ts +import compression from 'vite-plugin-compression'; +export default defineConfig({ + plugins: [compression({ algorithm: 'brotli' })], +}); +``` + +### 10. Virtual Scrolling for Large Lists — MEDIUM PRIORITY + +File trees, symbol lists, and metric tables render all items in the DOM. For codebases with 1000+ files this will degrade. + +**Candidates:** +- File tree in `/routes/explore/tree/+page.svelte` +- Symbol search results in layout sidebar +- Metrics tables in `/routes/explore/metrics/+page.svelte` + +Use `svelte-virtual-list` or a simple windowed renderer. + +### 11. Bundle Size Reduction — LOW PRIORITY + +| Asset | Size | Action | +|-------|------|--------| +| `elk.bundled.js` | 1.5 MB | Serve with brotli compression (~300 KB), add loading indicator | +| `3d-force-graph` + `force-graph` | Both in deps | Confirm both are needed; remove unused one | +| Shiki language packs | ~1 MB+ | Lazy-load per language (see item 8) | + +### 12. Manual Chunk Splitting — LOW PRIORITY + +Add to `vite.config.ts` to isolate heavy deps into separate chunks: + +```typescript +build: { + rollupOptions: { + output: { + manualChunks: { + 'three': ['three'], + '3d-graph': ['3d-force-graph'], + 'shiki': ['shiki'], + }, + }, + }, +}, +``` + +--- + +## Implementation Order + +| Phase | Items | Impact | Effort | +|-------|-------|--------|--------| +| **Phase 1** | N+1 queries (#1), DB indexes (#2) | High | Low | +| **Phase 2** | Pagination (#3), expand moka cache (#4) | High | Medium | +| **Phase 3** | HTTP headers (#5), client-side cache (#7) | Medium | Low | +| **Phase 4** | Shiki lazy-load (#8), vite compression (#9) | Medium | Low | +| **Phase 5** | Virtual scrolling (#10), bundle splitting (#11, #12) | Medium | Medium | +| **Phase 6** | Cache invalidation (#6) | Low | Medium | diff --git a/docs/SCHEMATIC_DESIGN.md b/docs/SCHEMATIC_DESIGN.md new file mode 100644 index 0000000..3e74418 --- /dev/null +++ b/docs/SCHEMATIC_DESIGN.md @@ -0,0 +1,664 @@ +# Schematic Explorer — Design Document + +> Unified, lazy-loaded, interactive schematic view that merges file tree, community graph, and learning curriculum into one explorable canvas. + +**Status:** Draft +**Date:** 2026-03-19 +**Relates to:** NORTH_STAR.md (Mode 1: Interactive Local Server) + +--- + +## 1. Problem Statement + +The current implementation has **three disconnected exploration surfaces**: + +| View | Route | Shows | Missing | +|------|-------|-------|---------| +| File tree | `/explore/tree` | Directory hierarchy + source | No communities, no learning links | +| Tree schematic | `/explore/schematic/tree` | File tree as SVG diagram | No symbols, no communities | +| Symbol graph | `/explore/schematic/graph` | Communities → symbols (layered) | No file context, no learning links | + +**Pain points:** + +1. **No unified model.** Each page fetches independently, builds its own layout, shares no state. Navigating between them loses all context. +2. **Eager loading.** Both schematic pages load ALL data upfront (`fetchGraph()` = up to 2000 nodes, `fetchFiles()` = everything). Slow on large repos. +3. **Modal is a dead-end.** Click a node → spinner → text. No way to navigate to related symbols, no "Start learning this" link, no outgoing paths. +4. **Search is inline-only.** Highlights matching nodes but doesn't provide a result list, can't navigate/zoom to a result, no keyboard navigation. +5. **Learning is disconnected.** Chapters live at `/learn`, mapped to communities via `community_id`, but the schematic has zero awareness of learning progress or chapter links. +6. **Layout quality.** Heuristic node sizing (`label.length * 7.5 + 32`), straight-line edges, no edge bundling, no community grouping in tree view. + +--- + +## 2. Design Goals + +1. **One page, multiple lenses.** Tree, graph, and mixed modes on a single route with shared state. +2. **Lazy everything.** Initial load is lightweight (top dirs + communities). Children, symbols, narratives, and source load on expand/click. +3. **Community-aware tree.** Files and directories are color-coded by dominant community. Community badges on nodes. +4. **Learning-integrated.** Every community node shows chapter progress. Detail panel links directly to the relevant chapter. +5. **Real search.** Cmd+K modal with results across files, symbols, and communities. Click result → navigate + zoom to node. +6. **Smooth interactions.** Bezier edges, animated expand/collapse, breadcrumb navigation, keyboard shortcuts. + +--- + +## 3. Data Model + +### 3.1 SchematicNode + +The unified node type that represents directories, files, symbols, and communities in a single tree: + +```typescript +interface SchematicNode { + id: string; // "dir:src/lib", "file:42", "sym:108", "comm:5" + type: "directory" | "file" | "symbol" | "community"; + label: string; // Display name + parent_id: string | null; // Tree parent (null for root) + + // File/symbol enrichment + file_id?: number; // For file and symbol nodes + symbol_id?: number; // For symbol nodes + language?: string; // For file nodes + sloc?: number; // For file nodes + kind?: string; // For symbol nodes: function, class, struct, etc. + signature?: string; // For symbol nodes + + // Community linkage + community_id?: number; // Which community this belongs to + community_label?: string; // Human-readable community name + community_color?: string; // Hex color for visual grouping + + // Learning linkage + chapter_id?: number; // Linked learning chapter + chapter_title?: string; // Chapter display name + difficulty?: string; // beginner, intermediate, advanced + progress?: { // Learning completion + completed_sections: number; + total_sections: number; + }; + + // Layout state (client-side only, not persisted) + collapsed?: boolean; // Whether children are hidden + x?: number; // Computed by layout engine + y?: number; + width?: number; + height?: number; +} +``` + +### 3.2 SchematicEdge + +```typescript +interface SchematicEdge { + id: string; // "e:108-209" + source: string; // SchematicNode.id + target: string; // SchematicNode.id + type: "contains" | "calls" | "imports" | "extends" | "implements"; + confidence?: number; // 0.0-1.0 + label?: string; // Optional display label +} +``` + +### 3.3 SchematicGraph (the full client-side model) + +```typescript +interface SchematicGraph { + nodes: Map; // Fast lookup by id + edges: SchematicEdge[]; // All visible edges + root_ids: string[]; // Top-level node ids + communities: CommunityInfo[]; // Community metadata +} + +interface CommunityInfo { + id: number; + label: string; + color: string; // Generated from HSL space + cohesion: number; + member_count: number; + chapter_id?: number; + chapter_title?: string; + progress?: { completed: number; total: number }; +} +``` + +### 3.4 How Existing Data Maps to This Model + +``` +Database → SchematicNode +───────────────────────────────────────────── +files.path (split by /) → type:"directory" (intermediate segments) +files row → type:"file", file_id, language, sloc +symbols row → type:"symbol", symbol_id, kind, signature +communities row → type:"community", community_id, cohesion + +community_members → symbol.community_id, file.community_id (dominant) +chapters → community.chapter_id, chapter_title, difficulty +progress → community.progress.{completed,total} +edges → SchematicEdge with type mapping +``` + +--- + +## 4. Backend API + +### 4.1 New Endpoint: `GET /api/v1/schematic` + +Single endpoint that returns a depth-limited, expandable schematic tree with community and learning enrichment. + +**Query parameters:** + +| Param | Type | Default | Description | +|-------|------|---------|-------------| +| `depth` | `u32` | `2` | How many directory levels deep to return | +| `expand` | `string` | `""` | Comma-separated node IDs to expand beyond depth limit | +| `community_id` | `i64` | — | Filter: only show files/symbols in this community | +| `include_symbols` | `bool` | `false` | Whether to include symbol nodes inside files | +| `include_edges` | `bool` | `false` | Whether to include cross-reference edges | + +**Response:** + +```json +{ + "nodes": [ + { + "id": "dir:src", + "type": "directory", + "label": "src", + "parent_id": "dir:.", + "has_children": true, + "child_count": 12, + "dominant_community_id": 3, + "dominant_community_color": "#6366f1" + }, + { + "id": "file:42", + "type": "file", + "label": "parser.rs", + "parent_id": "dir:src", + "file_id": 42, + "language": "rust", + "sloc": 340, + "community_id": 3, + "community_label": "parser_module", + "community_color": "#6366f1", + "symbol_count": 15, + "has_children": true + }, + { + "id": "sym:108", + "type": "symbol", + "label": "parse_file", + "parent_id": "file:42", + "symbol_id": 108, + "file_id": 42, + "kind": "function", + "signature": "pub fn parse_file(path: &Path) -> Result", + "community_id": 3, + "community_color": "#6366f1", + "chapter_id": 3, + "difficulty": "intermediate" + } + ], + "edges": [ + { + "id": "e:108-209", + "source": "sym:108", + "target": "sym:209", + "type": "calls", + "confidence": 0.95 + } + ], + "communities": [ + { + "id": 3, + "label": "parser_module", + "color": "#6366f1", + "cohesion": 0.87, + "member_count": 15, + "chapter_id": 3, + "chapter_title": "Chapter 3: The Parser", + "difficulty": "intermediate", + "progress": { "completed": 2, "total": 5 } + } + ], + "meta": { + "total_files": 342, + "total_symbols": 2847, + "total_communities": 12, + "depth_returned": 2 + } +} +``` + +### 4.2 Lazy Expansion: `GET /api/v1/schematic/expand` + +Fetch children of a specific node (used when user clicks to expand). + +**Query parameters:** + +| Param | Type | Description | +|-------|------|-------------| +| `node_id` | `string` | The node to expand (e.g., `dir:src/lib`, `file:42`) | +| `include_symbols` | `bool` | For directory expansion: also load symbols | +| `include_edges` | `bool` | Include edges between returned nodes | + +**Response:** Same shape as `/schematic` but only the children + their edges. + +### 4.3 Detail Fetch: `GET /api/v1/schematic/detail` + +Fetch rich detail for the detail panel (narrative, source, callers/callees, learning link). + +**Query parameters:** + +| Param | Type | Description | +|-------|------|-------------| +| `node_id` | `string` | e.g., `file:42` or `sym:108` | +| `include_source` | `bool` | Include source code lines | +| `source_start` | `u32` | Start line for source | +| `source_end` | `u32` | End line for source | + +**Response:** + +```json +{ + "node_id": "sym:108", + "narrative": "This function is the main entry point for parsing...", + "narrative_kind": "symbol_explanation", + "source": { + "path": "src/parser.rs", + "language": "rust", + "lines": [{ "number": 42, "content": "pub fn parse_file..." }], + "total_lines": 340 + }, + "callers": [ + { "id": "sym:55", "name": "run_pipeline", "kind": "function", "file_path": "src/main.rs" } + ], + "callees": [ + { "id": "sym:209", "name": "extract_symbols", "kind": "function", "file_path": "src/extract.rs" } + ], + "chapter": { + "id": 3, + "title": "Chapter 3: The Parser", + "difficulty": "intermediate", + "progress": { "completed": 2, "total": 5 }, + "next_section_id": 12 + }, + "annotations": [ + { "id": 1, "content": "Key entry point — understand this first", "flagged": true } + ] +} +``` + +### 4.4 Existing Endpoints (Unchanged) + +These remain for backward compatibility and use by other pages: + +- `GET /api/v1/graph` — Raw symbol graph (used by `/explore/graph`) +- `GET /api/v1/graph/communities` — Community-level graph +- `GET /api/v1/communities` — Community list +- `GET /api/v1/files`, `/files/:id/source`, `/files/:id/symbols` — File data +- `GET /api/v1/symbols/search?q=...` — Symbol search +- `GET /api/v1/chapters`, `/chapters/:id` — Learning chapters + +--- + +## 5. Frontend Architecture + +### 5.1 Route Structure + +``` +/explore/schematic/ → Unified Schematic Explorer (NEW) +/explore/schematic/tree/ → DEPRECATED (redirect to /explore/schematic?mode=tree) +/explore/schematic/graph/ → DEPRECATED (redirect to /explore/schematic?mode=graph) +``` + +### 5.2 Component Hierarchy + +``` +SchematicExplorer (+page.svelte) +├── SchematicToolbar +│ ├── Breadcrumb trail +│ ├── Mode toggle: [Tree] [Graph] [Mixed] +│ ├── Edge type filters: [Calls] [Imports] [Extends] [Implements] +│ ├── Node count + community count +│ └── Search trigger (Cmd+K) +│ +├── SchematicSidebar +│ ├── CommunityList +│ │ ├── Community card (color, label, member count, cohesion bar) +│ │ ├── Chapter progress bar +│ │ └── Click → filter canvas to this community +│ ├── LearnerProgress (XP, streak, badges) +│ └── CollapsibleTreeNav (mini file tree for quick nav) +│ +├── SchematicCanvas (main area) +│ ├── SVG layer: edges (bezier curves, colored by type) +│ ├── SVG layer: nodes (positioned by layout engine) +│ │ ├── DirectoryNode — folder icon, expand/collapse chevron +│ │ ├── FileNode — language color dot, sloc badge, community stripe +│ │ ├── SymbolNode — kind badge (FUN/CLS/STR), community color border +│ │ └── CommunityNode — large card with cohesion, member count, progress +│ ├── Pan/zoom/drag handler +│ └── Viewport culling (only render visible nodes) +│ +├── SchematicDetail (slide-in panel, right side) +│ ├── Header: node name, type badge, file path +│ ├── Tab: Explain — narrative content +│ ├── Tab: Source — syntax-highlighted code (lazy-loaded, line range) +│ ├── Tab: Relations — callers, callees, imports (clickable → navigate) +│ ├── Tab: Learn — chapter link, section list, "Start Learning" button +│ └── Annotations section (add/edit/flag) +│ +└── SchematicSearch (Cmd+K modal overlay) + ├── Input with debounced search (calls /symbols/search + local filter) + ├── Result groups: [Files] [Symbols] [Communities] + ├── Keyboard navigation (↑↓ to select, Enter to go) + └── Action: navigate canvas to node + open detail panel +``` + +### 5.3 State Management + +Single Svelte `$state` store, shared across all components: + +```typescript +// schematic-store.svelte.ts + +interface SchematicStore { + // Data + graph: SchematicGraph; // All loaded nodes + edges + communities: CommunityInfo[]; // Community metadata + + // View state + mode: "tree" | "graph" | "mixed"; // Layout mode + selectedNodeId: string | null; // Currently selected node + expandedNodes: Set; // Which nodes are expanded + visibleEdgeTypes: Set; // Which edge types to show + filteredCommunityId: number | null; // Community filter (null = all) + + // Layout output + layoutNodes: LayoutNode[]; // Positioned nodes (from layout engine) + layoutEdges: LayoutEdge[]; // Positioned edges + canvasWidth: number; + canvasHeight: number; + + // Viewport + panX: number; + panY: number; + scale: number; + + // Detail panel + detailOpen: boolean; + detailData: DetailData | null; + detailLoading: boolean; + + // Search + searchOpen: boolean; + searchQuery: string; + searchResults: SearchResult[]; + + // Loading + initialLoading: boolean; + expandLoading: Set; // Nodes currently being expanded +} +``` + +### 5.4 Layout Modes + +#### Tree Mode +- Right-flowing tree layout (like current `layoutTree`) +- Directories → files → (symbols if expanded) +- Community color as left border stripe on each node +- `contains` edges shown as tree lines (bezier curves) +- Cross-reference edges (calls/imports) shown as dashed arcs + +#### Graph Mode +- Community-centric layout +- Top level: community nodes (large cards) +- Drill into community: symbols laid out with topological layering +- All edge types visible, colored by type +- No file/directory structure shown + +#### Mixed Mode (the key innovation) +- Tree layout as the backbone (directory → file → symbol) +- Community grouping overlaid: files in the same community get a shared background region (convex hull) +- Cross-reference edges drawn as arcs between symbols across communities +- Best of both worlds: you see WHERE things are (tree) AND how they relate (graph) + +### 5.5 Lazy Loading Flow + +``` +Page load + │ + ├─ GET /api/v1/schematic?depth=2 + │ Returns: top dirs, files at depth ≤ 2, communities, chapter progress + │ Cost: ~50-100 nodes (lightweight) + │ + ├─ Run layout engine → render initial canvas + │ + User clicks directory node "src/lib" (expand) + │ + ├─ GET /api/v1/schematic/expand?node_id=dir:src/lib + │ Returns: children of src/lib (files + subdirs) + │ Merge into graph store → re-layout → animate expansion + │ + User clicks file node "parser.rs" (expand) + │ + ├─ GET /api/v1/schematic/expand?node_id=file:42&include_symbols=true&include_edges=true + │ Returns: symbols inside parser.rs + edges between them + │ Merge into graph store → re-layout → animate expansion + │ + User clicks symbol node "parse_file" (select) + │ + ├─ GET /api/v1/schematic/detail?node_id=sym:108&include_source=true + │ Returns: narrative, source, callers, callees, chapter link + │ Open detail panel → show tabs + │ + User clicks caller "run_pipeline" in detail panel + │ + ├─ Ensure sym:55 exists in graph (expand its file if needed) + ├─ Pan/zoom canvas to sym:55 + └─ Open detail for sym:55 +``` + +### 5.6 Search Flow + +``` +User presses Cmd+K + │ + ├─ Open search modal with focus on input + │ + User types "parse" + │ + ├─ Debounce 200ms + ├─ Local filter: search loaded nodes in graph store + ├─ Remote: GET /api/v1/symbols/search?q=parse (if local results < 5) + ├─ Combine + deduplicate → show grouped results + │ + User selects "parse_file" with ↓ + Enter + │ + ├─ Close search modal + ├─ If node not in graph → expand its parent chain (lazy load) + ├─ Pan/zoom canvas to center on node + ├─ Highlight node with pulse animation + └─ Open detail panel for node +``` + +--- + +## 6. Visual Design + +### 6.1 Node Styles + +``` +Directory Node: +┌─────────────────┐ +│ 📂 src/lib │ ← muted border, expandable chevron +│ 12 files │ +└─────────────────┘ + +File Node: +┌─────────────────┐ +│● parser.rs │ ← ● = language color dot +│ rust · 340 loc │ left border = community color +│ ■■■■■□□ 5/7 │ ← optional: chapter progress mini-bar +└─────────────────┘ + +Symbol Node: +┌─────────────────┐ +│ FUN parse_file │ ← kind badge + name +│ L42-L98 │ border color = community +└─────────────────┘ + +Community Node (graph mode): +┌───────────────────────┐ +│ ● parser_module │ ← ● = community color +│ 15 symbols │ +│ cohesion: 0.87 │ +│ ■■□□□ 2/5 learned │ ← chapter progress +│ [→ Start Chapter 3] │ ← learning link +└───────────────────────┘ +``` + +### 6.2 Edge Styles + +| Type | Color | Style | Arrowhead | +|------|-------|-------|-----------| +| contains | `var(--c-border)` | Solid, thin | None | +| calls | `#6366f1` (indigo) | Solid | Triangle | +| imports | `#14b8a6` (teal) | Dashed | Triangle | +| extends | `#f59e0b` (amber) | Solid, thick | Diamond | +| implements | `#ec4899` (pink) | Dotted | Circle | + +### 6.3 Community Colors + +Generated from HSL space to ensure distinctness: + +```typescript +function communityColor(id: number, total: number): string { + const hue = (id * 360 / Math.max(total, 1)) % 360; + return `hsl(${hue}, 65%, 55%)`; +} +``` + +### 6.4 Keyboard Shortcuts + +| Key | Action | +|-----|--------| +| `Cmd+K` / `Ctrl+K` | Open search | +| `Escape` | Close search / detail panel | +| `1` / `2` / `3` | Switch mode: tree / graph / mixed | +| `+` / `-` | Zoom in / out | +| `0` | Fit all nodes in viewport | +| `←` / `→` | Collapse / expand selected node | +| `Tab` | Cycle through detail panel tabs | +| `Enter` | Open detail for selected node | + +--- + +## 7. Implementation Plan + +### Phase 1: Backend — Unified Data Endpoint + +**Files to create/modify:** + +| File | Action | +|------|--------| +| `crates/codeilus-api/src/routes/schematic.rs` | NEW — `/schematic`, `/schematic/expand`, `/schematic/detail` | +| `crates/codeilus-api/src/routes/mod.rs` | Add schematic routes | +| `crates/codeilus-db/src/repos/schematic_repo.rs` | NEW — Queries joining files, symbols, communities, chapters, progress | +| `crates/codeilus-db/src/repos/mod.rs` | Add schematic repo | +| `crates/codeilus-core/src/types.rs` | Add `SchematicNode`, `SchematicEdge`, `SchematicResponse` types | + +**Key implementation details:** + +- Directory tree built by splitting file paths on `/` and grouping — same as frontend `buildDirTree()` but server-side +- `depth` parameter controls how many levels to return — BFS with depth counter +- `dominant_community_id` for directories: most common community among child files' symbols +- Community colors assigned once, returned in response so all clients see the same colors +- Chapter progress computed by joining `chapters` → `chapter_sections` → `progress` + +**Estimated scope:** ~400 lines Rust + +### Phase 2: Frontend — Core Components + +**Files to create/modify:** + +| File | Action | +|------|--------| +| `frontend/src/lib/schematic/types.ts` | NEW — SchematicNode, SchematicEdge, SchematicGraph interfaces | +| `frontend/src/lib/schematic/store.svelte.ts` | NEW — Centralized state with lazy-load actions | +| `frontend/src/lib/schematic/layout.ts` | REWRITE — Add mixed mode, community grouping, bezier edges | +| `frontend/src/lib/schematic/SchematicCanvas.svelte` | NEW — SVG canvas with pan/zoom, viewport culling | +| `frontend/src/lib/schematic/SchematicToolbar.svelte` | NEW — Mode toggle, edge filters, breadcrumb | +| `frontend/src/lib/schematic/SchematicSidebar.svelte` | NEW — Community list, progress, mini-tree | +| `frontend/src/lib/schematic/SchematicDetail.svelte` | NEW — Tabbed detail panel (explain, source, relations, learn) | +| `frontend/src/lib/schematic/SchematicSearch.svelte` | NEW — Cmd+K search modal | +| `frontend/src/lib/schematic/nodes/DirectoryNode.svelte` | NEW — Directory node renderer | +| `frontend/src/lib/schematic/nodes/FileNode.svelte` | NEW — File node renderer | +| `frontend/src/lib/schematic/nodes/SymbolNode.svelte` | NEW — Symbol node renderer | +| `frontend/src/lib/schematic/nodes/CommunityNode.svelte` | NEW — Community node renderer | +| `frontend/src/routes/explore/schematic/+page.svelte` | NEW — Unified explorer page | +| `frontend/src/lib/api.ts` | ADD — `fetchSchematic()`, `fetchSchematicExpand()`, `fetchSchematicDetail()` | +| `frontend/src/lib/types.ts` | ADD — Schematic-specific types | + +**Estimated scope:** ~1200 lines Svelte/TS + +### Phase 3: Polish + +| Feature | Details | +|---------|---------| +| Animated expand/collapse | CSS transitions on node position changes | +| Viewport culling | Only render nodes within visible SVG bounds | +| Edge bundling | Group parallel edges between same communities | +| Breadcrumb trail | Track navigation path with clickable segments | +| Keyboard navigation | Full keyboard support per section 6.4 | +| URL state sync | `?mode=mixed&community=3&selected=sym:108` in URL | +| Responsive layout | Sidebar collapses on narrow screens | +| Loading skeletons | Placeholder nodes while expanding | + +--- + +## 8. Migration Strategy + +1. Build the new `/explore/schematic` page alongside existing pages +2. Add redirect from `/explore/schematic/tree` and `/explore/schematic/graph` to new page +3. Remove old pages and dead `layout.ts` exports once stable +4. Keep existing `/explore/graph` (force-directed 3D graph) unchanged — different use case + +--- + +## 9. Performance Considerations + +### Backend +- **Depth-limited queries** prevent full-table scans +- **Community assignment** cached in moka (already exists) +- **Progress aggregation** via SQL `GROUP BY` instead of N+1 +- **Response size**: depth=2 on a 342-file repo returns ~80 nodes (~5KB JSON) + +### Frontend +- **Viewport culling**: Only render SVG elements within the visible bounds + 200px margin +- **Layout caching**: Re-layout only when nodes are added/removed, not on pan/zoom +- **Debounced search**: 200ms debounce on keystroke, local-first then remote +- **Edge rendering**: Skip edges where both endpoints are outside viewport +- **Node pooling**: Reuse SVG `` elements during re-layout instead of destroying/recreating + +### Scaling Targets +| Metric | Target | +|--------|--------| +| Initial load (depth=2) | < 200ms | +| Expand a directory | < 100ms (fetch + layout + render) | +| Open detail panel | < 300ms (narrative + source fetch) | +| Pan/zoom at 500 nodes | 60fps | +| Search results | < 150ms | + +--- + +## 10. Open Questions + +1. **Canvas vs SVG for large graphs?** SVG is simpler but degrades past ~1000 nodes. Could use Canvas for rendering + invisible SVG for hit testing. Decide after measuring real-world repos. + +2. **Community convex hull in mixed mode?** Drawing a shaded region around files in the same community looks great but is computationally expensive (convex hull + SVG path). Could simplify to a colored background band instead. + +3. **How to handle files in multiple communities?** A file may contain symbols from different communities. Options: (a) assign file to dominant community, (b) show multiple color stripes, (c) split file node into symbol-level nodes. + +4. **Offline narrative generation?** Detail panel calls `/narratives/:kind/:target_id` which may return a placeholder if LLM hasn't generated content yet. Should we show a "Generate now" button or queue it automatically? + +5. **Process flows integration?** The `processes` table has execution flows (BFS from entry points). Could overlay these as "guided paths" on the schematic — "Follow the request lifecycle: route → handler → service → db". Defer to Phase 3+. diff --git a/docs/adr/0002-unified-schematic-explorer.md b/docs/adr/0002-unified-schematic-explorer.md new file mode 100644 index 0000000..69b5da2 --- /dev/null +++ b/docs/adr/0002-unified-schematic-explorer.md @@ -0,0 +1,77 @@ +# ADR-0002: Unified Schematic Explorer (Replace Split Tree/Graph Views) + +**Status:** Proposed +**Date:** 2026-03-19 +**Decider:** Human (pending) + +## Context + +The schematic exploration feature currently has two separate pages: + +- `/explore/schematic/tree` — Renders the file system as a right-flowing SVG tree diagram. Clicking a file opens a modal with symbols, narrative, and source preview. No community awareness, no learning links. +- `/explore/schematic/graph` — Renders communities as layered nodes, drills into symbols. No file context, no learning links, no connection to the tree. + +Both pages share a layout library (`layout.ts`) but otherwise have no shared state, no shared components, and no shared data model. They were recently simplified by removing ELK.js and inlining canvas/modal/search components, leaving 159 lines of layout code and ~300 lines per page. + +**Problems this creates:** + +1. Users must choose between "where is it?" (tree) and "how does it connect?" (graph) — they can't see both. +2. Both pages load all data upfront. On large repos (2000+ symbols), initial load is slow and most data is never viewed. +3. The modal is a dead-end: no navigation to related symbols, no links to learning chapters, no way to follow call chains. +4. Search highlights nodes but can't navigate to them or show a result list. +5. Learning chapters (mapped to communities via `community_id`) are completely invisible in the schematic — a user exploring the graph has no idea that a curated learning path exists for the cluster they're looking at. + +## Decision + +Replace both schematic pages with a single **Unified Schematic Explorer** at `/explore/schematic/` that: + +1. **Merges tree and graph** via three switchable modes (tree, graph, mixed) sharing one data store. +2. **Lazy-loads** via a new backend endpoint (`GET /api/v1/schematic`) that returns depth-limited results with on-demand expansion. +3. **Enriches every node** with community assignment, learning chapter link, and progress. +4. **Provides a rich detail panel** with tabs (explain, source, relations, learn) instead of a basic modal. +5. **Adds Cmd+K search** with navigation to results on the canvas. + +Full design: `docs/SCHEMATIC_DESIGN.md` + +## Alternatives Considered + +### 1. Improve the two pages independently +Add community colors to the tree page, add file context to the graph page, add search to both. + +**Rejected.** Duplicates effort, still no shared state. User still must choose one view. Lazy loading would need to be implemented twice. + +### 2. Use a third-party graph library (Cytoscape, D3-force, vis.js) +Replace custom SVG rendering with a mature graph visualization library. + +**Rejected for now.** These libraries are general-purpose and heavy (100-300KB). Our nodes have very specific rendering needs (community stripes, progress bars, kind badges). Custom SVG gives us full control. Could reconsider for Canvas rendering if SVG degrades past 1000 nodes. + +### 3. Embed the graph inside the existing file tree page +Add an optional graph overlay to `/explore/tree` (the sidebar + code viewer page). + +**Rejected.** The tree page is a code browser (sidebar + source). The schematic is a canvas-based diagram. Different interaction paradigms — merging them would compromise both. + +### 4. Keep ELK.js for layout +Re-add ELK.js with lazy loading to get sophisticated layout quality. + +**Deferred.** The custom layout algorithms work well for the current node count. ELK.js adds 1.5MB to the bundle. Can lazy-import it later if layout quality becomes a bottleneck on complex repos. + +## Consequences + +**Positive:** +- One page to learn instead of two — simpler mental model for users +- Lazy loading means fast initial paint on any repo size +- Community → learning path link makes the schematic a gateway to the learning experience (core product value) +- Shared components (canvas, detail panel, search) reduce total frontend code +- Server-side tree+community joining enables future features (community heatmap, progress map) without new endpoints + +**Negative:** +- Requires a new backend endpoint and repository (schematic_repo) +- More complex frontend state management (single store with lazy-load actions) +- Mixed mode layout (tree + community grouping) is non-trivial to implement well +- Two old pages need redirects during migration period +- Risk of over-engineering if the mixed mode proves confusing to users + +**Risks & Mitigations:** +- **Mixed mode too complex?** Ship tree and graph modes first. Add mixed mode behind a toggle. User-test before making it default. +- **Backend query performance?** The schematic endpoint joins 5 tables. Mitigate with depth limits, indexes (already added in `0010_add_indexes.sql`), and moka caching. +- **SVG performance at scale?** Add viewport culling (only render visible nodes). If insufficient, migrate rendering to Canvas in a follow-up. diff --git a/frontend/e2e/schematic.spec.ts b/frontend/e2e/schematic.spec.ts new file mode 100644 index 0000000..fa2fc86 --- /dev/null +++ b/frontend/e2e/schematic.spec.ts @@ -0,0 +1,488 @@ +import { test, expect } from '@playwright/test'; + +// ============================================================ +// SCHEMATIC API ENDPOINTS +// ============================================================ + +test.describe('Schematic API', () => { + test('GET /api/v1/schematic returns nodes and communities', async ({ request }) => { + const res = await request.get('/api/v1/schematic?depth=2'); + expect(res.status()).toBe(200); + const data = await res.json(); + expect(data.nodes.length).toBeGreaterThan(0); + expect(Array.isArray(data.communities)).toBe(true); + expect(data.meta.total_files).toBeGreaterThan(0); + }); + + test('schematic nodes have correct shape', async ({ request }) => { + const res = await request.get('/api/v1/schematic?depth=2'); + const data = await res.json(); + const node = data.nodes[0]; + expect(node.id).toBeTruthy(); + expect(node.type).toMatch(/^(directory|file|symbol|community)$/); + expect(typeof node.label).toBe('string'); + expect(typeof node.has_children).toBe('boolean'); + }); + + test('schematic communities have enrichment when present', async ({ request }) => { + const res = await request.get('/api/v1/schematic?depth=2'); + const data = await res.json(); + if (data.communities.length > 0) { + const comm = data.communities[0]; + expect(typeof comm.id).toBe('number'); + expect(typeof comm.label).toBe('string'); + expect(typeof comm.color).toBe('string'); + expect(comm.color).toMatch(/^#[0-9a-f]{6}$/i); + } + // Always passes — communities may not exist for all codebases + expect(Array.isArray(data.communities)).toBe(true); + }); + + test('schematic expand returns children for first dir', async ({ request }) => { + // Get the first non-root directory from depth=1 + const res = await request.get('/api/v1/schematic?depth=1'); + const data = await res.json(); + const dir = data.nodes.find((n: any) => n.type === 'directory' && n.id !== 'dir:.'); + expect(dir).toBeTruthy(); + + const expandRes = await request.get(`/api/v1/schematic/expand?node_id=${encodeURIComponent(dir.id)}`); + expect(expandRes.status()).toBe(200); + const expandData = await expandRes.json(); + expect(expandData.nodes.length).toBeGreaterThan(0); + }); + + test('schematic detail returns callers/callees', async ({ request }) => { + // Find a symbol that has edges + const graphRes = await request.get('/api/v1/graph'); + const graph = await graphRes.json(); + const symbolWithEdge = graph.edges[0]?.source_id; + if (symbolWithEdge) { + const detailRes = await request.get(`/api/v1/schematic/detail?node_id=sym:${symbolWithEdge}`); + expect(detailRes.status()).toBe(200); + const detail = await detailRes.json(); + expect(detail.node_id).toBe(`sym:${symbolWithEdge}`); + expect(Array.isArray(detail.callers)).toBe(true); + expect(Array.isArray(detail.callees)).toBe(true); + } + }); + + test('community filter returns only community symbols', async ({ request }) => { + const res = await request.get('/api/v1/schematic?depth=10&community_id=2&include_symbols=true&include_edges=true'); + expect(res.status()).toBe(200); + const data = await res.json(); + const symbols = data.nodes.filter((n: any) => n.type === 'symbol'); + // All symbols should belong to community 2 + for (const sym of symbols) { + expect(sym.community_id).toBe(2); + } + // Edges should only reference loaded symbols + const symIds = new Set(symbols.map((s: any) => s.id)); + for (const edge of data.edges) { + expect(symIds.has(edge.source)).toBe(true); + expect(symIds.has(edge.target)).toBe(true); + } + }); +}); + +// ============================================================ +// SCHEMATIC TREE MODE +// ============================================================ + +test.describe('Schematic Tree Mode', () => { + test('page loads with tree view', async ({ page }) => { + await page.goto('/explore/schematic'); + // Should show the Tree/Graph toggle with Tree active + await expect(page.getByText('Tree')).toBeVisible({ timeout: 10000 }); + await expect(page.getByText('Schematic')).toBeVisible(); + // Should show nodes (SVG rects) + const rects = page.locator('svg rect'); + await expect(rects.first()).toBeVisible({ timeout: 10000 }); + }); + + test('shows codeilus root node', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg text:has-text("codeilus")')).toBeVisible({ timeout: 10000 }); + }); + + test('toolbar has Fit, Legend, ?, Search', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.getByText('Fit')).toBeVisible({ timeout: 5000 }); + await expect(page.getByText('Legend')).toBeVisible(); + await expect(page.getByText('?')).toBeVisible(); + await expect(page.locator('input[placeholder="Search..."]')).toBeVisible(); + }); + + test('clicking a directory expands it', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg text:has-text("codeilus")')).toBeVisible({ timeout: 10000 }); + // Count rects before + const before = await page.locator('svg rect').count(); + // Click the crates directory text + const cratesNode = page.locator('svg text:has-text("crates")'); + if (await cratesNode.count() > 0) { + await cratesNode.first().click(); + await page.waitForTimeout(1000); + const after = await page.locator('svg rect').count(); + expect(after).toBeGreaterThan(before); + } + }); + + test('clicking a file opens detail panel', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Look for a file node (has language text like "rust" or "typescript") + const fileText = page.locator('svg text:has-text(".rs"), svg text:has-text(".ts")'); + if (await fileText.count() > 0) { + await fileText.first().click(); + await page.waitForTimeout(500); + // Detail panel should appear with tabs + await expect(page.getByText('Overview')).toBeVisible({ timeout: 5000 }); + await expect(page.getByText('Source')).toBeVisible(); + await expect(page.getByText('Relations')).toBeVisible(); + } + }); + + test('search highlights nodes', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + const searchInput = page.locator('input[placeholder="Search..."]'); + await searchInput.fill('crates'); + await page.waitForTimeout(300); + // Should have highlighted nodes (accent stroke) + const accentRects = page.locator('svg rect[stroke*="accent"]'); + // At least the search should not crash + expect(await page.locator('svg rect').count()).toBeGreaterThan(0); + }); + + test('minimap is visible', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Minimap is a small SVG in bottom-right with specific dimensions + const minimap = page.locator('svg[width="180"][height="120"]'); + await expect(minimap).toBeVisible({ timeout: 5000 }); + }); + + test('no console errors on schematic', async ({ page }) => { + const errors: string[] = []; + page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); }); + await page.goto('/explore/schematic'); + await page.waitForTimeout(3000); + const realErrors = errors.filter(e => + !e.includes('WebSocket') && !e.includes('ws://') && !e.includes('ERR_CONNECTION') + ); + expect(realErrors).toEqual([]); + }); +}); + +// ============================================================ +// SCHEMATIC GRAPH MODE +// ============================================================ + +test.describe('Schematic Graph Mode', () => { + test('switching to graph mode works', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Click Graph toggle + await page.getByRole('button', { name: 'Graph' }).click(); + await page.waitForTimeout(1000); + // Graph mode should activate (button highlighted) + // If communities exist, sidebar shows; if not, canvas may be empty + // Just verify no crash + expect(await page.locator('svg').count()).toBeGreaterThan(0); + }); + + test('community sidebar lists all communities with counts', async ({ page }) => { + await page.goto('/explore/schematic'); + await page.getByRole('button', { name: 'Graph' }).click(); + await expect(page.getByText('COMMUNITIES')).toBeVisible({ timeout: 5000 }); + // Each community in sidebar should have a number + const sidebarItems = page.locator('.shrink-0 button'); + const count = await sidebarItems.count(); + expect(count).toBeGreaterThan(0); + }); + + test('clicking a community drills into symbols', async ({ page }) => { + await page.goto('/explore/schematic'); + await page.getByRole('button', { name: 'Graph' }).click(); + await expect(page.getByText('COMMUNITIES')).toBeVisible({ timeout: 5000 }); + // Click first community in sidebar + const firstComm = page.locator('.shrink-0 button').first(); + await firstComm.click(); + await page.waitForTimeout(2000); + // Should show breadcrumb "Communities" + await expect(page.getByText('Communities')).toBeVisible({ timeout: 5000 }); + // Should show symbol nodes + const nodeCount = await page.locator('svg rect').count(); + expect(nodeCount).toBeGreaterThan(0); + }); +}); + +// ============================================================ +// SCHEMATIC INTERACTIONS +// ============================================================ + +test.describe('Schematic Interactions', () => { + test('hover shows tooltip', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Hover over a node + const nodeG = page.locator('svg g[style*="cursor"]').first(); + await nodeG.hover(); + await page.waitForTimeout(300); + // Tooltip should appear (fixed position div with type badge) + const tooltip = page.locator('.fixed.z-\\[60\\]'); + // Tooltip may or may not be visible depending on node type + // Just verify no crash + }); + + test('right-click shows context menu', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Right-click a node + const nodeG = page.locator('svg g[style*="cursor"]').first(); + await nodeG.click({ button: 'right' }); + await page.waitForTimeout(300); + // Context menu should appear + await expect(page.getByText('Copy name')).toBeVisible({ timeout: 3000 }); + await expect(page.getByText('Focus here')).toBeVisible(); + }); + + test('keyboard ? shows help overlay', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Press ? + await page.keyboard.press('?'); + await page.waitForTimeout(300); + await expect(page.getByText('Keyboard Shortcuts')).toBeVisible({ timeout: 3000 }); + // Press Escape to close + await page.keyboard.press('Escape'); + await expect(page.getByText('Keyboard Shortcuts')).not.toBeVisible({ timeout: 3000 }); + }); + + test('Fit button resets viewport', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Click Fit button — should not crash + await page.getByRole('button', { name: 'Fit' }).click(); + await page.waitForTimeout(500); + // Nodes should still be visible + expect(await page.locator('svg rect').count()).toBeGreaterThan(0); + }); + + test('Legend toggle shows/hides legend panel', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Click Legend + await page.getByRole('button', { name: 'Legend' }).click(); + await page.waitForTimeout(300); + await expect(page.getByText('Nodes')).toBeVisible({ timeout: 3000 }); + await expect(page.getByText('Edges')).toBeVisible(); + await expect(page.getByText('Interactions')).toBeVisible(); + // Click again to hide + await page.getByRole('button', { name: 'Legend' }).click(); + await page.waitForTimeout(300); + }); + + test('detail panel Source tab loads code', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Expand crates → find a file + const cratesNode = page.locator('svg text:has-text("crates")'); + if (await cratesNode.count() > 0) { + await cratesNode.first().click(); + await page.waitForTimeout(1000); + // Look for a .rs file and click it + const fileNode = page.locator('svg text:has-text(".rs")').first(); + if (await fileNode.count() > 0) { + await fileNode.click(); + await page.waitForTimeout(500); + // Click Source tab + const sourceTab = page.getByRole('button', { name: 'Source' }); + if (await sourceTab.count() > 0) { + await sourceTab.click(); + await page.waitForTimeout(2000); + // Should show source code (pre element) + const pre = page.locator('pre'); + await expect(pre).toBeVisible({ timeout: 5000 }); + } + } + } + }); +}); + +// ============================================================ +// RED TESTS — Expected behavior, may not pass yet +// These define the target UX. Fix code until they go green. +// ============================================================ + +test.describe('Schematic Expected Behaviors (TDD)', () => { + test('file detail panel shows source code when Source tab clicked', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Expand first directory + const firstDir = page.locator('svg text:has-text("▶")').first(); + if (await firstDir.count() > 0) { + await firstDir.click(); + await page.waitForTimeout(1500); + // Keep expanding until we find a file + const secondDir = page.locator('svg text:has-text("▶")').first(); + if (await secondDir.count() > 0) { + await secondDir.click(); + await page.waitForTimeout(1500); + } + // Find any file node (has "loc" in the text) + const fileNode = page.locator('svg text:has-text("loc")').first(); + if (await fileNode.count() > 0) { + await fileNode.click(); + await page.waitForTimeout(500); + // Click Source tab + const sourceTab = page.getByRole('button', { name: 'Source' }); + if (await sourceTab.count() > 0) { + await sourceTab.click(); + await page.waitForTimeout(3000); + // Should show source code + const pre = page.locator('pre'); + await expect(pre).toBeVisible({ timeout: 5000 }); + const text = await pre.textContent(); + expect(text).toBeTruthy(); + expect(text!.length).toBeGreaterThan(20); + } + } + } + }); + + test('community sidebar shows in graph mode with member counts', async ({ page }) => { + await page.goto('/explore/schematic'); + await page.getByRole('button', { name: 'Graph' }).click(); + await page.waitForTimeout(1000); + await expect(page.getByText('COMMUNITIES')).toBeVisible({ timeout: 5000 }); + // Sidebar buttons should have community names and counts + const sidebarButtons = page.locator('.shrink-0 button'); + const count = await sidebarButtons.count(); + expect(count).toBeGreaterThan(0); + // Each should have a label + const firstText = await sidebarButtons.first().textContent(); + expect(firstText).toBeTruthy(); + expect(firstText!.length).toBeGreaterThan(0); + }); + + test('detail panel Learn tab shows chapter link with progress', async ({ page }) => { + await page.goto('/explore/schematic'); + await page.getByRole('button', { name: 'Graph' }).click(); + await page.waitForTimeout(1000); + // Click a community to drill in + await page.locator('.shrink-0 button').first().click(); + await page.waitForTimeout(2000); + // Click a symbol node + const symbolNode = page.locator('svg g[style*="cursor"] text').first(); + if (await symbolNode.count() > 0) { + await symbolNode.click(); + await page.waitForTimeout(500); + // Click Learn tab + const learnTab = page.getByRole('button', { name: 'Learn' }); + if (await learnTab.count() > 0) { + await learnTab.click(); + await page.waitForTimeout(500); + // Should show a chapter link or "No learning chapter" message + const hasChapter = await page.getByText('Start Learning').count() > 0; + const noChapter = await page.getByText('No learning chapter').count() > 0; + expect(hasChapter || noChapter).toBe(true); + } + } + }); + + test('right-click Copy name action executes without error', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + const nodeG = page.locator('svg g[style*="cursor"]').first(); + await nodeG.click({ button: 'right' }); + await page.waitForTimeout(300); + await expect(page.getByText('Copy name')).toBeVisible(); + // Click Copy name — should close the menu without crashing + await page.getByText('Copy name').click(); + await page.waitForTimeout(300); + // Context menu should be gone + await expect(page.getByText('Copy name')).not.toBeVisible({ timeout: 2000 }); + }); + + test('right-click Focus here zooms to the node', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Right-click crates + const cratesText = page.locator('svg text:has-text("crates")').first(); + if (await cratesText.count() > 0) { + await cratesText.click({ button: 'right' }); + await page.waitForTimeout(300); + await page.getByText('Focus here').click(); + await page.waitForTimeout(500); + // The transform should have changed (node centered) + const gTransform = await page.locator('svg > g').first().getAttribute('transform'); + expect(gTransform).toBeTruthy(); + } + }); + + test('keyboard F fits all nodes in viewport', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + // Press F + await page.keyboard.press('f'); + await page.waitForTimeout(500); + // All nodes should be visible (within viewport) + const rects = await page.locator('svg rect').count(); + expect(rects).toBeGreaterThan(0); + }); + + test('double-click directory recursively expands', async ({ page }) => { + await page.goto('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + const before = await page.locator('svg rect').count(); + // Double-click crates + const cratesText = page.locator('svg text:has-text("crates")').first(); + if (await cratesText.count() > 0) { + await cratesText.dblclick(); + await page.waitForTimeout(2000); + const after = await page.locator('svg rect').count(); + expect(after).toBeGreaterThan(before); + } + }); +}); + +// ============================================================ +// SCHEMATIC + FULL NAVIGATION +// ============================================================ + +test.describe('Schematic Navigation Flow', () => { + test('navigate explore → schematic → tree → graph without errors', async ({ page }) => { + test.setTimeout(60000); + const errors: string[] = []; + page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); }); + + // Explore hub + await page.goto('/explore'); + await expect(page.getByText('Schematic Explorer')).toBeVisible({ timeout: 5000 }); + + // Click Schematic card + await page.getByRole('link', { name: 'Schematic Explorer' }).click(); + await expect(page).toHaveURL('/explore/schematic'); + await expect(page.locator('svg rect').first()).toBeVisible({ timeout: 10000 }); + + // Switch to Graph mode + await page.getByRole('button', { name: 'Graph' }).click(); + await page.waitForTimeout(1000); + await expect(page.getByText('COMMUNITIES')).toBeVisible({ timeout: 5000 }); + + // Switch back to Tree mode + await page.getByRole('button', { name: 'Tree' }).click(); + await page.waitForTimeout(1000); + await expect(page.locator('svg text:has-text("codeilus")').first()).toBeVisible({ timeout: 5000 }); + + // Go back to explore + await page.getByRole('link', { name: '←' }).click(); + await expect(page).toHaveURL('/explore'); + + const realErrors = errors.filter(e => + !e.includes('WebSocket') && !e.includes('ws://') && !e.includes('ERR_CONNECTION') + ); + expect(realErrors).toEqual([]); + }); +}); diff --git a/frontend/package.json b/frontend/package.json index 2f52821..2a40952 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,12 +2,17 @@ "name": "codeilus-frontend", "version": "0.1.0", "private": true, + "sideEffects": [ + "./src/lib/schematic/elk-layout.ts" + ], "scripts": { "dev": "vite dev", "build": "vite build", "preview": "vite preview", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch" + "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui" }, "devDependencies": { "@playwright/test": "^1.58.2", @@ -21,12 +26,12 @@ "svelte-check": "^4.0.0", "tailwindcss": "^4.0.0", "typescript": "^5.0.0", - "vite": "^6.0.0" + "vite": "^6.0.0", + "vite-plugin-compression2": "^2.5.2" }, "type": "module", "dependencies": { "3d-force-graph": "^1.79.1", - "elkjs": "^0.11.1", "force-graph": "^1.51.2", "shiki": "^4.0.2", "three": "^0.183.2" diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 4a30a50..236078d 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -11,9 +11,6 @@ importers: 3d-force-graph: specifier: ^1.79.1 version: 1.79.1 - elkjs: - specifier: ^0.11.1 - version: 0.11.1 force-graph: specifier: ^1.51.2 version: 1.51.2 @@ -60,6 +57,9 @@ importers: vite: specifier: ^6.0.0 version: 6.4.1(jiti@2.6.1)(lightningcss@1.31.1) + vite-plugin-compression2: + specifier: ^2.5.2 + version: 2.5.2(rollup@4.59.0) packages: @@ -251,6 +251,15 @@ packages: '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} + '@rollup/pluginutils@5.3.0': + resolution: {integrity: sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + '@rollup/rollup-android-arm-eabi@4.59.0': resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} cpu: [arm] @@ -720,9 +729,6 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} - elkjs@0.11.1: - resolution: {integrity: sha512-zxxR9k+rx5ktMwT/FwyLdPCrq7xN6e4VGGHH8hA01vVYKjTFik7nHOxBnAYtrgYUB1RpAiLvA1/U2YraWxyKKg==} - enhanced-resolve@5.20.0: resolution: {integrity: sha512-/ce7+jQ1PQ6rVXwe+jKEg5hW5ciicHwIQUagZkp6IufBoY3YDgdTTY1azVs0qoRgVmvsNB+rbjLJxDAeHHtwsQ==} engines: {node: '>=10.13.0'} @@ -738,6 +744,9 @@ packages: esrap@2.2.3: resolution: {integrity: sha512-8fOS+GIGCQZl/ZIlhl59htOlms6U8NvX6ZYgYHpRU/b6tVSh3uHkOHZikl3D4cMbYM0JlpBe+p/BkZEi8J9XIQ==} + estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -1037,6 +1046,9 @@ packages: resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} engines: {node: '>=6'} + tar-mini@0.2.0: + resolution: {integrity: sha512-+qfUHz700DWnRutdUsxRRVZ38G1Qr27OetwaMYTdg8hcPxf46U0S1Zf76dQMWRBmusOt2ZCK5kbIaiLkoGO7WQ==} + three-forcegraph@1.43.1: resolution: {integrity: sha512-lQnYPLvR31gb91mF5xHhU0jPHJgBPw9QB23R6poCk8Tgvz8sQtq7wTxwClcPdfKCBbHXsb7FSqK06Osiu1kQ5A==} engines: {node: '>=12'} @@ -1092,6 +1104,9 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite-plugin-compression2@2.5.2: + resolution: {integrity: sha512-a1eAY2Ux5JrflaWStWUa7yoMnjhw3tx7y6vQIp20QZ9K57wcKxYT+DUJiAG4VSEvmFMjeo9a5R2phdBriOJ/1A==} + vite@6.4.1: resolution: {integrity: sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -1261,6 +1276,14 @@ snapshots: '@polka/url@1.0.0-next.29': {} + '@rollup/pluginutils@5.3.0(rollup@4.59.0)': + dependencies: + '@types/estree': 1.0.8 + estree-walker: 2.0.2 + picomatch: 4.0.3 + optionalDependencies: + rollup: 4.59.0 + '@rollup/rollup-android-arm-eabi@4.59.0': optional: true @@ -1643,8 +1666,6 @@ snapshots: dependencies: dequal: 2.0.3 - elkjs@0.11.1: {} - enhanced-resolve@5.20.0: dependencies: graceful-fs: 4.2.11 @@ -1685,6 +1706,8 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + estree-walker@2.0.2: {} + fdir@6.5.0(picomatch@4.0.3): optionalDependencies: picomatch: 4.0.3 @@ -2017,6 +2040,8 @@ snapshots: tapable@2.3.0: {} + tar-mini@0.2.0: {} + three-forcegraph@1.43.1(three@0.183.2): dependencies: accessor-fn: 1.5.3 @@ -2088,6 +2113,13 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 + vite-plugin-compression2@2.5.2(rollup@4.59.0): + dependencies: + '@rollup/pluginutils': 5.3.0(rollup@4.59.0) + tar-mini: 0.2.0 + transitivePeerDependencies: + - rollup + vite@6.4.1(jiti@2.6.1)(lightningcss@1.31.1): dependencies: esbuild: 0.25.12 diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1e8938a..027502a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -8,10 +8,16 @@ import type { NarrativeResponse, Chapter, SourceResponse, + SchematicResponse, + SchematicDetail, } from '$lib/types'; const BASE = '/api/v1'; +// ── In-memory cache with TTL ── +const _cache = new Map(); +const DEFAULT_CACHE_TTL = 5 * 60 * 1000; // 5 minutes + async function get(url: string, fallback: T): Promise { try { const res = await fetch(url); @@ -31,13 +37,24 @@ async function get(url: string, fallback: T): Promise { } } +/** Cached GET — returns from in-memory cache if fresh, otherwise fetches. */ +async function cachedGet(url: string, fallback: T, ttl = DEFAULT_CACHE_TTL): Promise { + const entry = _cache.get(url); + if (entry && entry.expires > Date.now()) { + return entry.data as T; + } + const data = await get(url, fallback); + _cache.set(url, { data, expires: Date.now() + ttl }); + return data; +} + export async function fetchHealth(): Promise<{ status: string }> { return get(`${BASE}/health`, { status: 'disconnected' }); } export async function fetchFiles(language?: string): Promise { const params = language ? `?language=${encodeURIComponent(language)}` : ''; - return get(`${BASE}/files${params}`, []); + return cachedGet(`${BASE}/files${params}`, []); } export async function fetchFile(id: number): Promise { @@ -50,7 +67,7 @@ export async function fetchFileSymbols(fileId: number): Promise { export async function fetchSymbols(kind?: string): Promise { const params = kind ? `?kind=${encodeURIComponent(kind)}` : ''; - return get(`${BASE}/symbols${params}`, []); + return cachedGet(`${BASE}/symbols${params}`, []); } export async function fetchSymbol(id: number): Promise { @@ -66,11 +83,11 @@ export async function fetchGraph(): Promise { } export async function fetchCommunityGraph(): Promise { - return get(`${BASE}/graph/communities`, { nodes: [], edges: [] }); + return cachedGet(`${BASE}/graph/communities`, { nodes: [], edges: [] }); } export async function fetchCommunities(): Promise { - return get(`${BASE}/communities`, []); + return cachedGet(`${BASE}/communities`, []); } export async function fetchProcesses(): Promise { @@ -78,7 +95,7 @@ export async function fetchProcesses(): Promise { } export async function fetchNarrative(kind: string): Promise { - return get(`${BASE}/narratives/${kind}`, null); + return cachedGet(`${BASE}/narratives/${kind}`, null); } export async function fetchNarrativeByTarget(kind: string, targetId: number): Promise { @@ -86,7 +103,7 @@ export async function fetchNarrativeByTarget(kind: string, targetId: number): Pr } export async function fetchChapters(): Promise { - return get(`${BASE}/chapters`, []); + return cachedGet(`${BASE}/chapters`, []); } export async function fetchChapter(id: number): Promise { @@ -268,3 +285,24 @@ export async function askQuestion( onError(`Network error: ${e}`); } } + +// ── Schematic API ── + +const EMPTY_SCHEMATIC: SchematicResponse = { nodes: [], edges: [], communities: [], meta: { total_files: 0, total_symbols: 0, total_communities: 0, depth_returned: 0 } }; + +export async function fetchSchematic(depth = 2, communityId?: number, includeSymbols = false, includeEdges = false): Promise { + const params = new URLSearchParams({ depth: String(depth), include_symbols: String(includeSymbols), include_edges: String(includeEdges) }); + if (communityId !== undefined) params.set('community_id', String(communityId)); + return cachedGet(`${BASE}/schematic?${params}`, EMPTY_SCHEMATIC); +} + +export async function fetchSchematicExpand(nodeId: string, includeSymbols = true, includeEdges = true): Promise { + const params = new URLSearchParams({ node_id: nodeId, include_symbols: String(includeSymbols), include_edges: String(includeEdges) }); + return get(`${BASE}/schematic/expand?${params}`, EMPTY_SCHEMATIC); +} + +export async function fetchSchematicDetail(nodeId: string, includeSource = true): Promise { + return cachedGet(`${BASE}/schematic/detail?node_id=${encodeURIComponent(nodeId)}&include_source=${includeSource}`, { + node_id: nodeId, callers: [], callees: [], + }); +} diff --git a/frontend/src/lib/schematic/SchematicCanvas.svelte b/frontend/src/lib/schematic/SchematicCanvas.svelte deleted file mode 100644 index 55a45df..0000000 --- a/frontend/src/lib/schematic/SchematicCanvas.svelte +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - {#if children} - {@render children()} - {/if} - - diff --git a/frontend/src/lib/schematic/SchematicContextMenu.svelte b/frontend/src/lib/schematic/SchematicContextMenu.svelte new file mode 100644 index 0000000..0da6ed8 --- /dev/null +++ b/frontend/src/lib/schematic/SchematicContextMenu.svelte @@ -0,0 +1,91 @@ + + + + + + +
e.stopPropagation()} +> + {#each items as item} + + {/each} +
diff --git a/frontend/src/lib/schematic/SchematicDetailTabs.svelte b/frontend/src/lib/schematic/SchematicDetailTabs.svelte new file mode 100644 index 0000000..8c9451d --- /dev/null +++ b/frontend/src/lib/schematic/SchematicDetailTabs.svelte @@ -0,0 +1,241 @@ + + +
+ +
+
+
+ {node.type} +

{node.label}

+
+ +
+ + +
+ {#each tabs as tab} + + {/each} +
+
+ + +
+ {#if loading} + + {:else if detail} + + + {#if activeTab === 'overview'} +
+
+ {#if node.language} + {node.language} + {/if} + {#if node.kind} + {node.kind} + {/if} + {#if node.sloc} + {node.sloc} loc + {/if} +
+ + {#if node.community_label} +
+ + {node.community_label} +
+ {/if} + + {#if detail.narrative} +
+

Explanation

+
+ +
+
+ {:else} +

No AI explanation available yet. Click "Source" tab to view the code, or right-click → "Ask AI" to generate one.

+ {/if} + + {#if node.signature} +
+

Signature

+ 
{node.signature}
+
+ {/if} +
+ + + {:else if activeTab === 'source'} + {#if sourceLoading} + + {:else if sourceData} + 
{sourceData.lines.map(l => `${String(l.number).padStart(4)} ${l.content}`).join('\n')}
+ {:else if !node.file_id} +

No source available for this node.

+ {/if} + + + {:else if activeTab === 'relations'} +
+ {#if detail.callers.length > 0} +
+

Called by ({detail.callers.length})

+
+ {#each detail.callers as c} + + {/each} +
+
+ {/if} + + {#if detail.callees.length > 0} +
+

Calls ({detail.callees.length})

+
+ {#each detail.callees as c} + + {/each} +
+
+ {/if} + + {#if detail.callers.length === 0 && detail.callees.length === 0} +

No connections found.

+ {/if} +
+ + + {:else if activeTab === 'learn'} + {#if detail.chapter} +
+ +
{detail.chapter.title}
+
+ {detail.chapter.difficulty} +
+
+
+ {detail.chapter.progress.completed}/{detail.chapter.progress.total} +
+ + Start Learning → +
+ {:else} +

No learning chapter linked to this node.

+ {/if} + + + {:else if activeTab === 'notes'} +
+ {#if annotationsLoading} + + {:else} + {#each annotations as ann} +
+

{ann.content}

+
+ {new Date(ann.created_at).toLocaleDateString()} + + +
+
+ {/each} + + {#if annotations.length === 0} +

No notes yet.

+ {/if} + +
+ e.key === 'Enter' && submitNote()} + /> + +
+ {/if} +
+ {/if} + {/if} +
+
diff --git a/frontend/src/lib/schematic/SchematicKeyboardOverlay.svelte b/frontend/src/lib/schematic/SchematicKeyboardOverlay.svelte new file mode 100644 index 0000000..7044c37 --- /dev/null +++ b/frontend/src/lib/schematic/SchematicKeyboardOverlay.svelte @@ -0,0 +1,41 @@ + + +{#if visible} + + +
+
e.stopPropagation()}> +
+

Keyboard Shortcuts

+ +
+
+ {#each shortcuts as s} +
+ {s.key} + {s.desc} +
+ {/each} +
+
+
+{/if} diff --git a/frontend/src/lib/schematic/SchematicMinimap.svelte b/frontend/src/lib/schematic/SchematicMinimap.svelte new file mode 100644 index 0000000..a2bd19d --- /dev/null +++ b/frontend/src/lib/schematic/SchematicMinimap.svelte @@ -0,0 +1,77 @@ + + + + + dragging && handleClick(e)} + onpointerdown={() => dragging = true} + onpointerup={() => dragging = false} +> + + {#each nodes as node} + {@const nx = node.x * minimapScale} + {@const ny = node.y * minimapScale} + {@const nw = Math.max(node.width * minimapScale, 2)} + {@const nh = Math.max(node.height * minimapScale, 1.5)} + + {/each} + + + + diff --git a/frontend/src/lib/schematic/SchematicModal.svelte b/frontend/src/lib/schematic/SchematicModal.svelte deleted file mode 100644 index c1489ba..0000000 --- a/frontend/src/lib/schematic/SchematicModal.svelte +++ /dev/null @@ -1,49 +0,0 @@ - - -{#if open} - - -
- -
e.stopPropagation()} - > -
-

{title}

- -
-
- {#if children} - {@render children()} - {/if} -
-
-
-{/if} - - diff --git a/frontend/src/lib/schematic/SchematicSearch.svelte b/frontend/src/lib/schematic/SchematicSearch.svelte deleted file mode 100644 index 672328e..0000000 --- a/frontend/src/lib/schematic/SchematicSearch.svelte +++ /dev/null @@ -1,78 +0,0 @@ - - -
-
- - - {#if query} - - {/if} -
- {#if matches.length > 0} - {matchIndex + 1}/{matches.length} - {/if} -
diff --git a/frontend/src/lib/schematic/SchematicSourcePopup.svelte b/frontend/src/lib/schematic/SchematicSourcePopup.svelte new file mode 100644 index 0000000..4153e32 --- /dev/null +++ b/frontend/src/lib/schematic/SchematicSourcePopup.svelte @@ -0,0 +1,73 @@ + + + + +
+
e.stopPropagation()} + > + +
+
+ {node.kind || node.type} + {node.label} +
+
+ {#if onviewfull} + + {/if} + +
+
+ + +
+ {#if loading} +
+ {:else if sourceData} + 
{sourceData.lines.map(l => `${String(l.number).padStart(4)} │ ${l.content}`).join('\n')}
+ {:else} +

No source available.

+ {/if} +
+
+
diff --git a/frontend/src/lib/schematic/SchematicTooltip.svelte b/frontend/src/lib/schematic/SchematicTooltip.svelte new file mode 100644 index 0000000..52adacd --- /dev/null +++ b/frontend/src/lib/schematic/SchematicTooltip.svelte @@ -0,0 +1,76 @@ + + +{#if node} +
+ +
+ {node.type} + {node.label} +
+ + +
+ {#if node.language} + {node.language} + {/if} + {#if node.sloc} + {node.sloc} loc + {/if} + {#if node.kind} + {KIND_LABELS[node.kind] || node.kind} + {/if} + {#if node.symbol_count} + {node.symbol_count} symbols + {/if} + {#if node.has_children && node.type === 'directory'} + click to expand + {/if} + {#if node.type === 'file' && node.has_children} + expandable + {/if} +
+ + + {#if node.community_label} +
+ + {node.community_label} +
+ {/if} + + + {#if node.signature} +
{node.signature.slice(0, 60)}
+ {/if} +
+{/if} diff --git a/frontend/src/lib/schematic/edge-path.ts b/frontend/src/lib/schematic/edge-path.ts deleted file mode 100644 index 6bbd0f8..0000000 --- a/frontend/src/lib/schematic/edge-path.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** Convert ELK edge bend points to an SVG path d-attribute. */ -export function edgePathD(points: { x: number; y: number }[]): string { - if (points.length === 0) return ''; - if (points.length === 1) return `M ${points[0].x} ${points[0].y}`; - if (points.length === 2) { - return `M ${points[0].x} ${points[0].y} L ${points[1].x} ${points[1].y}`; - } - - // Smooth cubic bezier through points - let d = `M ${points[0].x} ${points[0].y}`; - for (let i = 1; i < points.length - 1; i++) { - const prev = points[i - 1]; - const curr = points[i]; - const next = points[i + 1]; - const cpx1 = (prev.x + curr.x) / 2; - const cpy1 = (prev.y + curr.y) / 2; - const cpx2 = (curr.x + next.x) / 2; - const cpy2 = (curr.y + next.y) / 2; - if (i === 1) { - d += ` Q ${curr.x} ${curr.y} ${cpx2} ${cpy2}`; - } else { - d += ` T ${cpx2} ${cpy2}`; - } - } - const last = points[points.length - 1]; - d += ` L ${last.x} ${last.y}`; - return d; -} - -/** Get the midpoint of a path for label placement. */ -export function edgeMidpoint(points: { x: number; y: number }[]): { x: number; y: number } { - if (points.length === 0) return { x: 0, y: 0 }; - if (points.length === 1) return points[0]; - const mid = Math.floor(points.length / 2); - if (points.length % 2 === 0) { - return { - x: (points[mid - 1].x + points[mid].x) / 2, - y: (points[mid - 1].y + points[mid].y) / 2, - }; - } - return points[mid]; -} diff --git a/frontend/src/lib/schematic/elk-layout.ts b/frontend/src/lib/schematic/elk-layout.ts deleted file mode 100644 index e5cd415..0000000 --- a/frontend/src/lib/schematic/elk-layout.ts +++ /dev/null @@ -1,140 +0,0 @@ -import ELK from 'elkjs/lib/elk.bundled.js'; -import type { SchematicNode, SchematicEdge, LayoutResult } from './types'; - -const elk = new ELK(); - -interface LayoutOptions { - algorithm?: 'layered' | 'mrtree' | 'force'; - direction?: 'RIGHT' | 'DOWN' | 'LEFT' | 'UP'; - nodeSpacing?: number; - layerSpacing?: number; -} - -interface ElkNode { - id: string; - width: number; - height: number; - children?: ElkNode[]; - x?: number; - y?: number; -} - -interface ElkEdge { - id: string; - sources: string[]; - targets: string[]; -} - -function toElkNode(node: SchematicNode): ElkNode { - const elkNode: ElkNode = { - id: node.id, - width: node.width, - height: node.height, - }; - if (node.children && node.children.length > 0) { - elkNode.children = node.children.map(toElkNode); - } - return elkNode; -} - -function flattenPositions( - elkNode: { id?: string; x?: number; y?: number; width?: number; height?: number; children?: unknown[] }, - offsetX: number, - offsetY: number, - result: Map, -) { - const children = elkNode.children as typeof elkNode[] | undefined; - if (children) { - for (const child of children) { - const cx = offsetX + (child.x ?? 0); - const cy = offsetY + (child.y ?? 0); - result.set(child.id!, { x: cx, y: cy, width: child.width ?? 0, height: child.height ?? 0 }); - flattenPositions(child, cx, cy, result); - } - } -} - -export async function computeLayout( - nodes: SchematicNode[], - edges: SchematicEdge[], - options: LayoutOptions = {}, -): Promise { - const { - algorithm = 'layered', - direction = 'RIGHT', - nodeSpacing = 25, - layerSpacing = 50, - } = options; - - const elkGraph = { - id: 'root', - layoutOptions: { - 'elk.algorithm': algorithm === 'mrtree' ? 'org.eclipse.elk.mrtree' : - algorithm === 'force' ? 'org.eclipse.elk.force' : - 'org.eclipse.elk.layered', - 'elk.direction': direction, - 'elk.spacing.nodeNode': String(nodeSpacing), - 'elk.layered.spacing.edgeNodeBetweenLayers': String(layerSpacing), - 'elk.hierarchyHandling': 'INCLUDE_CHILDREN', - 'elk.layered.nodePlacement.strategy': 'NETWORK_SIMPLEX', - 'elk.padding': '[top=20,left=20,bottom=20,right=20]', - }, - children: nodes.map(toElkNode), - edges: edges.map((e): ElkEdge => ({ - id: e.id, - sources: [e.source], - targets: [e.target], - })), - }; - - const laid = await elk.layout(elkGraph); - - const nodePositions = new Map(); - flattenPositions(laid, 0, 0, nodePositions); - // Also add top-level children - if (laid.children) { - for (const child of laid.children) { - if (!nodePositions.has(child.id)) { - nodePositions.set(child.id, { - x: child.x ?? 0, y: child.y ?? 0, - width: child.width ?? 0, height: child.height ?? 0, - }); - } - } - } - - const edgePositions = new Map(); - if (laid.edges) { - for (const edge of laid.edges) { - const sections = (edge as unknown as { sections?: { startPoint: { x: number; y: number }; endPoint: { x: number; y: number }; bendPoints?: { x: number; y: number }[] }[] }).sections; - if (sections && sections.length > 0) { - const pts: { x: number; y: number }[] = []; - for (const sec of sections) { - pts.push(sec.startPoint); - if (sec.bendPoints) pts.push(...sec.bendPoints); - pts.push(sec.endPoint); - } - edgePositions.set(edge.id, { points: pts }); - } else { - // Fallback: connect source center to target center - const src = nodePositions.get((edge as unknown as { sources: string[] }).sources[0]); - const tgt = nodePositions.get((edge as unknown as { targets: string[] }).targets[0]); - if (src && tgt) { - edgePositions.set(edge.id, { - points: [ - { x: src.x + src.width, y: src.y + src.height / 2 }, - { x: tgt.x, y: tgt.y + tgt.height / 2 }, - ], - }); - } - } - } - } - - return { - nodes: nodePositions, - edges: edgePositions, - width: laid.width ?? 800, - height: laid.height ?? 600, - }; -} diff --git a/frontend/src/lib/schematic/layout.ts b/frontend/src/lib/schematic/layout.ts new file mode 100644 index 0000000..989927f --- /dev/null +++ b/frontend/src/lib/schematic/layout.ts @@ -0,0 +1,216 @@ +/** Pure-TS layout algorithms. No browser APIs, no external deps. */ + +export interface LayoutNode { + id: string; + label: string; + width: number; + height: number; + x: number; + y: number; + data: Record; + children?: LayoutNode[]; +} + +export interface LayoutEdge { + id: string; + from: string; + to: string; + label?: string; + kind?: string; +} + +// ── Tree Layout (right-flowing) ── + +interface TreeInput { + id: string; + label: string; + data: Record; + children: TreeInput[]; +} + +const NODE_H = 36; +const NODE_PAD_Y = 6; +const LAYER_GAP = 180; + +function measureLabel(label: string, minW = 100): number { + return Math.max(minW, label.length * 7.5 + 32); +} + +function layoutTreeRecursive( + node: TreeInput, + depth: number, + yOffset: number, + nodes: LayoutNode[], + edges: LayoutEdge[], +): number { + const w = measureLabel(node.label, depth === 0 ? 120 : 140); + const x = depth * LAYER_GAP; + + if (node.children.length === 0) { + const ln: LayoutNode = { id: node.id, label: node.label, width: w, height: NODE_H, x, y: yOffset, data: node.data }; + nodes.push(ln); + return yOffset + NODE_H + NODE_PAD_Y; + } + + const childStartY = yOffset; + let currentY = yOffset; + for (const child of node.children) { + edges.push({ id: `e-${node.id}-${child.id}`, from: node.id, to: child.id }); + currentY = layoutTreeRecursive(child, depth + 1, currentY, nodes, edges); + } + + // Center parent vertically among its children + const childEndY = currentY - NODE_PAD_Y; + const centerY = (childStartY + childEndY - NODE_H) / 2; + + const ln: LayoutNode = { id: node.id, label: node.label, width: w, height: NODE_H, x, y: centerY, data: node.data }; + nodes.push(ln); + + return currentY; +} + +export function layoutTree(root: TreeInput): { nodes: LayoutNode[]; edges: LayoutEdge[]; width: number; height: number } { + const nodes: LayoutNode[] = []; + const edges: LayoutEdge[] = []; + const totalH = layoutTreeRecursive(root, 0, 20, nodes, edges); + const maxX = Math.max(...nodes.map(n => n.x + n.width)); + return { nodes, edges, width: maxX + 40, height: totalH + 20 }; +} + +// ── Layered Layout (top-down, for symbol graphs) ── + +interface LayeredInput { + nodes: { id: string; label: string; data: Record }[]; + edges: { from: string; to: string; label?: string; kind?: string }[]; +} + +const LAYER_V_GAP = 70; +const NODE_H_GAP = 24; + +export function layoutLayered(input: LayeredInput): { nodes: LayoutNode[]; edges: LayoutEdge[]; width: number; height: number } { + const { nodes: inputNodes, edges: inputEdges } = input; + + // If no edges, use grid layout + if (inputEdges.length === 0) { + return layoutGrid(inputNodes); + } + + // Build adjacency + const incoming = new Map>(); + const outgoing = new Map>(); + for (const n of inputNodes) { + incoming.set(n.id, new Set()); + outgoing.set(n.id, new Set()); + } + for (const e of inputEdges) { + outgoing.get(e.from)?.add(e.to); + incoming.get(e.to)?.add(e.from); + } + + // Topological layering (Kahn's algorithm) + const layers: string[][] = []; + const assigned = new Set(); + const remaining = new Set(inputNodes.map(n => n.id)); + + while (remaining.size > 0) { + const layer: string[] = []; + for (const id of remaining) { + const inc = incoming.get(id)!; + const unassignedIncoming = [...inc].filter(i => !assigned.has(i)); + if (unassignedIncoming.length === 0) { + layer.push(id); + } + } + if (layer.length === 0) { + layer.push(...remaining); + } + for (const id of layer) { + assigned.add(id); + remaining.delete(id); + } + layers.push(layer); + } + + // Position nodes + const nodeMap = new Map(inputNodes.map(n => [n.id, n])); + const layoutNodes: LayoutNode[] = []; + let maxWidth = 0; + + for (let li = 0; li < layers.length; li++) { + const layer = layers[li]; + const y = 20 + li * LAYER_V_GAP; + let x = 20; + for (const id of layer) { + const n = nodeMap.get(id)!; + const w = measureLabel(n.label, 130); + layoutNodes.push({ id, label: n.label, width: w, height: NODE_H, x, y, data: n.data }); + x += w + NODE_H_GAP; + } + maxWidth = Math.max(maxWidth, x); + } + + const maxY = Math.max(...layoutNodes.map(n => n.y + n.height)); + const layoutEdges: LayoutEdge[] = inputEdges.map((e, i) => ({ + id: `e-${i}`, + from: e.from, + to: e.to, + label: e.label, + kind: e.kind, + })); + + return { nodes: layoutNodes, edges: layoutEdges, width: maxWidth + 20, height: maxY + 40 }; +} + +// ── Grid Layout (for nodes without edges, e.g. community overview) ── + +function layoutGrid(inputNodes: { id: string; label: string; data: Record }[]): { nodes: LayoutNode[]; edges: LayoutEdge[]; width: number; height: number } { + const CARD_W = 200; + const CARD_H = 60; + const GAP_X = 24; + const GAP_Y = 20; + const COLS = Math.max(1, Math.min(5, Math.ceil(Math.sqrt(inputNodes.length)))); + + const layoutNodes: LayoutNode[] = []; + for (let i = 0; i < inputNodes.length; i++) { + const col = i % COLS; + const row = Math.floor(i / COLS); + const n = inputNodes[i]; + layoutNodes.push({ + id: n.id, + label: n.label, + width: CARD_W, + height: CARD_H, + x: 20 + col * (CARD_W + GAP_X), + y: 20 + row * (CARD_H + GAP_Y), + data: n.data, + }); + } + + const maxX = Math.max(...layoutNodes.map(n => n.x + n.width), 0); + const maxY = Math.max(...layoutNodes.map(n => n.y + n.height), 0); + return { nodes: layoutNodes, edges: [], width: maxX + 20, height: maxY + 20 }; +} + +// ── Fit-to-view ── + +export function computeFitToView( + nodes: LayoutNode[], + viewportW: number, + viewportH: number, + padding = 40, +): { tx: number; ty: number; scale: number } { + if (nodes.length === 0) return { tx: padding, ty: padding, scale: 1 }; + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + for (const n of nodes) { + minX = Math.min(minX, n.x); + minY = Math.min(minY, n.y); + maxX = Math.max(maxX, n.x + n.width); + maxY = Math.max(maxY, n.y + n.height); + } + const bboxW = maxX - minX; + const bboxH = maxY - minY; + const scale = Math.min((viewportW - padding * 2) / bboxW, (viewportH - padding * 2) / bboxH, 1.5); + const tx = (viewportW - bboxW * scale) / 2 - minX * scale; + const ty = (viewportH - bboxH * scale) / 2 - minY * scale; + return { tx, ty, scale }; +} diff --git a/frontend/src/lib/schematic/types.ts b/frontend/src/lib/schematic/types.ts deleted file mode 100644 index 8858336..0000000 --- a/frontend/src/lib/schematic/types.ts +++ /dev/null @@ -1,26 +0,0 @@ -export interface SchematicNode { - id: string; - label: string; - width: number; - height: number; - x?: number; - y?: number; - children?: SchematicNode[]; - metadata: Record; -} - -export interface SchematicEdge { - id: string; - source: string; - target: string; - label?: string; - kind?: string; - points?: { x: number; y: number }[]; -} - -export interface LayoutResult { - nodes: Map; - edges: Map; - width: number; - height: number; -} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 6f54c90..ac37c21 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -157,6 +157,68 @@ export interface LearnerStats { badges: Badge[]; } +// ── Schematic types ── + +export interface SchematicNode { + id: string; + type: 'directory' | 'file' | 'symbol' | 'community'; + label: string; + parent_id: string | null; + file_id?: number; + symbol_id?: number; + language?: string; + sloc?: number; + kind?: string; + signature?: string; + community_id?: number; + community_label?: string; + community_color?: string; + chapter_id?: number; + chapter_title?: string; + difficulty?: string; + progress?: { completed: number; total: number }; + has_children: boolean; + child_count?: number; + symbol_count?: number; +} + +export interface SchematicEdge { + id: string; + source: string; + target: string; + type: string; + confidence?: number; +} + +export interface SchematicCommunity { + id: number; + label: string; + color: string; + cohesion: number; + member_count: number; + chapter_id?: number; + chapter_title?: string; + difficulty?: string; + progress?: { completed: number; total: number }; +} + +export interface SchematicResponse { + nodes: SchematicNode[]; + edges: SchematicEdge[]; + communities: SchematicCommunity[]; + meta: { total_files: number; total_symbols: number; total_communities: number; depth_returned: number }; +} + +export interface SchematicDetail { + node_id: string; + narrative?: string; + narrative_kind?: string; + source?: { path: string; language?: string; lines: { number: number; content: string }[]; total_lines: number }; + callers: { id: string; name: string; kind: string; file_path: string }[]; + callees: { id: string; name: string; kind: string; file_path: string }[]; + chapter?: { id: number; title: string; difficulty: string; progress: { completed: number; total: number } }; +} + export interface Annotation { id: number; target_type: 'node' | 'edge'; diff --git a/frontend/src/routes/explore/+page.svelte b/frontend/src/routes/explore/+page.svelte index 495b3be..7e4e807 100644 --- a/frontend/src/routes/explore/+page.svelte +++ b/frontend/src/routes/explore/+page.svelte @@ -38,20 +38,12 @@

SLOC, language distribution, and largest files

- +
- +
-

Tree Schematic

-

Interactive codebase tree with clickable files and details

- - - -
- -
-

Symbol Schematic

-

Layered symbol graph with community drill-down and learning content

+

Schematic Explorer

+

Unified tree + graph with lazy loading, communities, and learning links

 diff --git a/frontend/src/routes/explore/schematic/+page.svelte b/frontend/src/routes/explore/schematic/+page.svelte new file mode 100644 index 0000000..5e013e8 --- /dev/null +++ b/frontend/src/routes/explore/schematic/+page.svelte @@ -0,0 +1,610 @@ + + + + +
+ +
+ ← + Schematic + +
+ + +
+ + {#if selectedCommunity} + › + + › + {communities.find(c => c.id === selectedCommunity)?.label} + {/if} + +
+ + + + {meta.total_files}f · {meta.total_symbols}s + +
+
+ +
+ + {#if mode === 'graph' && !selectedCommunity} +
+

Communities

+ {#each communities as c} + + {/each} +
+ {/if} + + +
+ {#if loading} +
+ {:else} + + + + + + + + + + + + + {#each layoutEdges as edge} + {@const from = nodeMap.get(edge.from)} + {@const to = nodeMap.get(edge.to)} + {#if from && to} + + {/if} + {/each} + + + {#each layoutNodes as node} + {@const sn = allNodes.get(node.id)} + {@const isHovered = node.id === hoveredNodeId} + {@const isGhost = ghostNodeIds.has(node.id)} + {@const dimmed = hoveredNodeId && !isHovered && !isGhost} + + + handleNodePointerDown(node, e)} + oncontextmenu={(e) => handleContextMenu(node, e)} + onpointerenter={(e) => { hoveredNodeId = node.id; tooltipX = e.clientX; tooltipY = e.clientY; }} + onpointerleave={() => hoveredNodeId = null} + > + + {#if sn?.community_color} + + {/if} + + {#if sn?.type === 'directory'} + {@const isExp = expandedSet.has(node.id)} + + {isExp ? '▼' : '▶'} {node.label === '.' ? 'codeilus' : node.label} + + {#if expandingSet.has(node.id)} + ... + {/if} + {:else if sn?.type === 'file'} + + {node.label} + {sn.language || ''} · {sn.sloc} loc + {:else if sn?.type === 'symbol'} + {KIND_LABELS[sn.kind || ''] || sn.kind} + {node.label} + {:else} + {@const data = node.data} + {#if data.color} + + {/if} + {node.label} + {data.memberCount} symbols + {/if} + + {/each} + + + + + { tx = newTx; ty = newTy; }} + /> + + + {#if showLegend} +
+
Legend
+ +
+
Nodes
+
+
▶ Directory (click to expand)
+
File
+
FN Symbol (function, class...)
+
Community
+
+
+ +
+
Edges
+
+
Calls
+
Imports
+
Extends
+
Implements
+
+
+ +
+
Colors
+
Left stripe = community color
+
+ {#each communities.slice(0, 8) as c} + + {/each} + {#if communities.length > 8} + +{communities.length - 8} + {/if} +
+
+ +
+
Interactions
+
+
Click · expand / select
+
Double-click · deep dive
+
Right-click · actions menu
+
Hover · highlight connections
+
+
+
+ {/if} + {/if} +
+ + + {#if detailOpen && detailNode} + detailOpen = false} + onnavigate={handleNavigate} + onannotationcreate={handleAnnotationCreate} + onannotationdelete={handleAnnotationDelete} + onannotationflag={handleAnnotationFlag} + /> + {/if} +
+
+ + + + + +{#if sourcePopupNode} + sourcePopupNode = null} + onviewfull={() => { if (sourcePopupNode?.file_id) goto(`/explore/tree?fileId=${sourcePopupNode.file_id}`); sourcePopupNode = null; }} + /> +{/if} + + +{#if contextMenuNode} + contextMenuNode = null} + /> +{/if} + + + showKeyboardHelp = false} /> diff --git a/frontend/src/routes/explore/schematic/graph/+page.svelte b/frontend/src/routes/explore/schematic/graph/+page.svelte index a9fb2ab..b0d25ab 100644 --- a/frontend/src/routes/explore/schematic/graph/+page.svelte +++ b/frontend/src/routes/explore/schematic/graph/+page.svelte @@ -1,64 +1,36 @@
- - - - + ←

Symbol Graph

- - {#if breadcrumb.length > 0} - {#each breadcrumb as crumb} - - - {/each} - - - {#if activeCommunityId !== null} - {communities.find(c => c.id === activeCommunityId)?.label || `Community ${activeCommunityId}`} - {/if} - - {/if} - - {flatNodes.length} nodes + {#each breadcrumb as crumb} + › + + {/each} + {nodes.length} nodes +
{#if loading} -
- -
+
{:else} - highlighted = ids} - /> - - {#snippet children()} - - {#each visibleEdges as edge} - {@const pts = layout?.edges.get(edge.id)?.points} - {@const color = EDGE_COLORS[edge.kind || ''] || 'var(--c-border-hover)'} - {#if pts} + + + + + + + + + {#each edges as edge} + {@const from = nodeMap.get(edge.from)} + {@const to = nodeMap.get(edge.to)} + {#if from && to} - {#if edge.label} - {@const mid = edgeMidpoint(pts)} - {edge.label} - {/if} {/if} {/each} - - {#each flatNodes as node} - {@const pos = layout?.nodes.get(node.id)} - {#if pos} - - - handleNodeClick(node.id)} - style="cursor: pointer" - > - {#if node.metadata.type === 'community'} - {@const cid = node.metadata.communityId as number} - - -
-
- - {node.label} -
- {node.metadata.memberCount} symbols -
- - - {:else if node.metadata.type === 'symbol'} - {@const gn = node.metadata.node as GraphNode} - {@const cid = gn.community_id ?? 0} - {@const badge = KIND_BADGES[gn.kind] || { label: gn.kind.slice(0,3).toUpperCase(), bg: 'rgba(156,163,175,0.15)' }} - - - - -
- {badge.label} - {node.label} - {fileNameById(gn.file_id)} -
- - {/if} - - {/if} + {#each nodes as node} + + + handleNodeClick(node)} style="cursor: pointer"> + {#if node.data.type === 'community'} + {@const cid = node.data.communityId as number} + + + {node.label} + {node.data.memberCount} symbols + {:else} + {@const gn = node.data.node as GraphNode} + {@const cid = gn.community_id ?? 0} + + + {KIND_LABELS[gn.kind] || gn.kind} + {node.label} + {/if} + {/each} - {/snippet} - + + {/if}
- - modalOpen = false}> - {#snippet children()} - {#if modalLoading} -
- {:else if modalSymbol} - {@const badge = KIND_BADGES[modalSymbol.kind] || { label: modalSymbol.kind, bg: 'rgba(156,163,175,0.15)' }} -
-
- {badge.label} - {fileNameById(modalSymbol.file_id)} -
- - {#if modalNarrative?.content} -
-

Explanation

-

{modalNarrative.content}

+ +{#if modalOpen} + + +
modalOpen = false}> + +
e.stopPropagation()}> +
+

{modalTitle}

+ +
+
+ {#if modalLoading} + + {:else if modalSymbol} +
+ {KIND_LABELS[modalSymbol.kind] || modalSymbol.kind} + {fileName(modalSymbol.file_id)}
- {/if} - {#if modalConnections.callers.length > 0} -
-

Called by ({modalConnections.callers.length})

-
- {#each modalConnections.callers.slice(0, 10) as caller} - {caller.name} - {/each} + {#if modalNarrative?.content} +
+

Explanation

+

{modalNarrative.content}

-
- {/if} - - {#if modalConnections.callees.length > 0} -
-

Calls ({modalConnections.callees.length})

-
- {#each modalConnections.callees.slice(0, 10) as callee} - {callee.name} - {/each} + {/if} + + {#if modalCallers.length > 0} +
+

Called by ({modalCallers.length})

+
+ {#each modalCallers.slice(0, 10) as c} + {c.name} + {/each} +
-
- {/if} + {/if} + + {#if modalCallees.length > 0} +
+

Calls ({modalCallees.length})

+
+ {#each modalCallees.slice(0, 10) as c} + {c.name} + {/each} +
+
+ {/if} - {#if modalSource} -
-

Source

- 
{modalSource.lines.map(l => `${String(l.number).padStart(3)} ${l.content}`).join('\n')}
-
+ {#if modalSource} +
+

Source

+ 
{modalSource.lines.map(l => `${String(l.number).padStart(3)} ${l.content}`).join('\n')}
+
+ {/if} {/if}
- {/if} - {/snippet} - +
+
+{/if} diff --git a/frontend/src/routes/explore/schematic/tree/+page.svelte b/frontend/src/routes/explore/schematic/tree/+page.svelte index 7c7c989..fd0508d 100644 --- a/frontend/src/routes/explore/schematic/tree/+page.svelte +++ b/frontend/src/routes/explore/schematic/tree/+page.svelte @@ -1,31 +1,31 @@
+
- - - - + ←

Codebase Tree

- {flatNodes.length} nodes + {nodes.length} nodes +
+ +
+
{#if loading} -
- -
- {:else if flatNodes.length === 0} -
- No files found. Analyze a codebase first. -
+
+ {:else if nodes.length === 0} +
No files found.
{:else} - highlighted = ids} - /> - - {#snippet children()} + + + + + + + + {#each edges as edge} - {@const pts = layout?.edges.get(edge.id)?.points} - {#if pts} - + {@const from = nodeMap.get(edge.from)} + {@const to = nodeMap.get(edge.to)} + {#if from && to} + {/if} {/each} - {#each flatNodes as node} - {@const pos = layout?.nodes.get(node.id)} - {#if pos} - - - handleNodeClick(node.id)} - style="cursor: {node.metadata.type === 'file' ? 'pointer' : 'default'}" - > - - {#if node.metadata.type === 'dir'} - -
- - {node.label} - {node.metadata.fileCount} -
- - {:else} - {@const lang = (node.metadata.language as string) || ''} - {@const color = LANG_COLORS[lang.toLowerCase()] || 'var(--c-text-muted)'} - -
-
- - {node.label} -
-
- {#if lang} - {lang} - {/if} - {node.metadata.sloc} loc -
-
- - {/if} - - {/if} + {#each nodes as node} + + + handleNodeClick(node)} + style="cursor: {node.data.type === 'file' ? 'pointer' : 'default'}" + > + + {#if node.data.type === 'dir'} + + 📁 {node.label} + + {:else} + {@const lang = (node.data.language as string) || ''} + {@const color = LANG_COLORS[lang.toLowerCase()] || 'var(--c-text-muted)'} + + {node.label} + {lang} · {node.data.sloc} loc + {/if} + {/each} - {/snippet} - + + {/if}
- - modalOpen = false}> - {#snippet children()} - {#if modalLoading} -
- {:else if modalFile} -
-
{modalFile.path}
-
- {#if modalFile.language} - {modalFile.language} - {/if} - {modalFile.sloc} lines -
- - {#if modalNarrative?.content} -
-

Overview

-

{modalNarrative.content}

+ +{#if modalOpen} + + +
modalOpen = false}> + +
e.stopPropagation()}> +
+

{modalTitle}

+ +
+
+ {#if modalLoading} + + {:else if modalFile} +
{modalFile.path}
+
+ {#if modalFile.language} + {modalFile.language} + {/if} + {modalFile.sloc} lines
- {/if} - {#if modalSymbols.length > 0} -
-

Symbols ({modalSymbols.length})

-
- {#each modalSymbols as sym} -
- {sym.kind} - {sym.name} - L{sym.start_line} -
- {/each} + {#if modalNarrative?.content} +
+

Overview

+

{modalNarrative.content}

-
- {/if} + {/if} - {#if modalSource} -
-

Source Preview

- 
{modalSource.lines.map(l => `${String(l.number).padStart(3)} ${l.content}`).join('\n')}
-
+ {#if modalSymbols.length > 0} +
+

Symbols ({modalSymbols.length})

+
+ {#each modalSymbols as sym} +
+ {sym.kind} + {sym.name} + L{sym.start_line} +
+ {/each} +
+
+ {/if} + + {#if modalSource} +
+

Source

+ 
{modalSource.lines.map(l => `${String(l.number).padStart(3)} ${l.content}`).join('\n')}
+
+ {/if} {/if}
- {/if} - {/snippet} - +
+
+{/if} diff --git a/frontend/src/routes/explore/tree/+page.svelte b/frontend/src/routes/explore/tree/+page.svelte index ef4515d..a29b951 100644 --- a/frontend/src/routes/explore/tree/+page.svelte +++ b/frontend/src/routes/explore/tree/+page.svelte @@ -52,12 +52,12 @@ let symbolFilter = $state(''); let searchQuery = $state(''); - // Initialize shiki highlighter + // Initialize shiki highlighter — starts empty, languages loaded on demand per file async function getHighlighter(): Promise { if (highlighter) return highlighter; highlighter = await createHighlighter({ themes: ['github-dark'], - langs: Object.values(LANG_MAP).filter((v, i, a) => a.indexOf(v) === i), + langs: [], }); return highlighter; } diff --git a/frontend/src/routes/learn/+page.svelte b/frontend/src/routes/learn/+page.svelte index 663341d..e0e2ff2 100644 --- a/frontend/src/routes/learn/+page.svelte +++ b/frontend/src/routes/learn/+page.svelte @@ -244,7 +244,7 @@ Module Explanation
- +
{/if} diff --git a/frontend/src/routes/learn/[id]/+page.svelte b/frontend/src/routes/learn/[id]/+page.svelte index 689ba13..c2d8755 100644 --- a/frontend/src/routes/learn/[id]/+page.svelte +++ b/frontend/src/routes/learn/[id]/+page.svelte @@ -262,7 +262,7 @@ {#if chapter.narrative}
- +
{/if} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 36ece35..567c539 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,12 +1,17 @@ import { sveltekit } from '@sveltejs/kit/vite'; import tailwindcss from '@tailwindcss/vite'; +import { compression } from 'vite-plugin-compression2'; import { defineConfig } from 'vite'; export default defineConfig({ - plugins: [tailwindcss(), sveltekit()], + plugins: [ + tailwindcss(), + sveltekit(), + compression({ algorithms: ['brotliCompress'] }), + ], server: { proxy: { '/api': 'http://localhost:4174', } - } + }, }); diff --git a/migrations/0010_add_indexes.sql b/migrations/0010_add_indexes.sql new file mode 100644 index 0000000..0e9bf2f --- /dev/null +++ b/migrations/0010_add_indexes.sql @@ -0,0 +1,10 @@ +-- Performance indexes on foreign keys used in JOINs/WHEREs + +CREATE INDEX IF NOT EXISTS idx_community_members_community ON community_members(community_id); +CREATE INDEX IF NOT EXISTS idx_community_members_symbol ON community_members(symbol_id); +CREATE INDEX IF NOT EXISTS idx_process_steps_process ON process_steps(process_id); +CREATE INDEX IF NOT EXISTS idx_chapters_community ON chapters(community_id); +CREATE INDEX IF NOT EXISTS idx_chapter_sections_chapter ON chapter_sections(chapter_id); +CREATE INDEX IF NOT EXISTS idx_progress_chapter ON progress(chapter_id); + +INSERT INTO schema_version (version) VALUES (10);