From 50c8ab81b0f915b9f5b20fbb5ffdb3233f1f1578 Mon Sep 17 00:00:00 2001 From: LiberiFatali Date: Thu, 27 Aug 2026 19:46:31 +0700 Subject: [PATCH 1/4] refactor: enforce consistent lint and formatting with pre-commit and CI - Format whole repo with ruff (one-time cleanup, 206 files) - Drop black, pin ruff==0.16.3, keep mypy as optional - Add pre-commit hook (ruff check --fix + ruff format) and restore CI format check on changed files - Add CONTRIBUTING.md and .git-blame-ignore-revs for blame hygiene - Fix remaining lint issues (bare except, F821) and ignore E402 for intentional sys.path imports Prevents repeat of CI failure #33059834372 where lint errors were only caught after push. --- .codebuddy/hooks/capture_session_end.py | 27 +- .../plans/multi-ide-hook-wiring_458a6f24.md | 29 +- ...(\346\234\252\345\256\214\346\210\220).md" | 29 +- ...5\245\346\226\271\346\241\210_c8ec3f3e.md" | 1 + .git-blame-ignore-revs | 5 + .github/workflows/ci.yml | 5 + .pre-commit-config.yaml | 12 + .qoder/hooks/capture_session_end.py | 27 +- CONTRIBUTING.md | 11 + codewiki/__main__.py | 2 +- codewiki/cli/__init__.py | 1 - codewiki/cli/adapters/__init__.py | 1 - codewiki/cli/adapters/doc_generator.py | 183 ++- codewiki/cli/commands/__init__.py | 1 - codewiki/cli/commands/config.py | 353 ++--- codewiki/cli/commands/generate.py | 219 +-- codewiki/cli/commands/install_hooks.py | 4 +- codewiki/cli/commands/query.py | 55 +- codewiki/cli/config_manager.py | 69 +- codewiki/cli/git_manager.py | 100 +- codewiki/cli/html_generator.py | 195 +-- codewiki/cli/main.py | 7 +- codewiki/cli/models/__init__.py | 1 - codewiki/cli/models/config.py | 200 +-- codewiki/cli/models/job.py | 66 +- codewiki/cli/utils/__init__.py | 1 - codewiki/cli/utils/api_errors.py | 36 +- codewiki/cli/utils/errors.py | 19 +- codewiki/cli/utils/fs.py | 66 +- codewiki/cli/utils/instructions.py | 61 +- codewiki/cli/utils/logging.py | 30 +- codewiki/cli/utils/progress.py | 92 +- codewiki/cli/utils/repo_validator.py | 111 +- codewiki/cli/utils/validation.py | 168 +- codewiki/hooks/capture_session_end.py | 27 +- codewiki/mcp/_ide_hook.py | 97 +- codewiki/mcp/cache.py | 1119 ++++++++++---- codewiki/mcp/cbm_client.py | 19 +- codewiki/mcp/resources.py | 303 +++- codewiki/mcp/server.py | 4 + codewiki/mcp/session.py | 49 +- codewiki/mcp/tools/adoption.py | 3 + codewiki/mcp/tools/agents_md.py | 14 +- codewiki/mcp/tools/aggregation_state.py | 17 +- codewiki/mcp/tools/analysis.py | 407 +++-- codewiki/mcp/tools/batch_ingest.py | 43 +- codewiki/mcp/tools/capture_conversation.py | 207 +-- codewiki/mcp/tools/cbm_integration.py | 14 +- codewiki/mcp/tools/change_analysis.py | 88 +- codewiki/mcp/tools/close_session.py | 25 +- codewiki/mcp/tools/code_reader.py | 37 +- codewiki/mcp/tools/component_list.py | 19 +- codewiki/mcp/tools/cross_service.py | 56 +- codewiki/mcp/tools/crosslink.py | 34 +- codewiki/mcp/tools/distill_conversation.py | 2 +- codewiki/mcp/tools/doc_description.py | 13 +- codewiki/mcp/tools/doc_update_notify.py | 33 +- codewiki/mcp/tools/doc_writer.py | 4 +- codewiki/mcp/tools/doctrine.py | 132 +- codewiki/mcp/tools/file_viewer.py | 33 +- codewiki/mcp/tools/friction.py | 21 +- codewiki/mcp/tools/hook_registry.py | 13 +- codewiki/mcp/tools/html_export.py | 26 +- codewiki/mcp/tools/impact.py | 33 +- codewiki/mcp/tools/index_freshness.py | 39 +- codewiki/mcp/tools/init_wiki.py | 17 +- codewiki/mcp/tools/injection_budget.py | 3 + codewiki/mcp/tools/issue_tracker.py | 40 +- codewiki/mcp/tools/knowledge_loop.py | 547 ++++--- codewiki/mcp/tools/legacy_tools.py | 36 +- codewiki/mcp/tools/note_consolidation.py | 251 +-- codewiki/mcp/tools/note_merge.py | 14 +- codewiki/mcp/tools/note_types.py | 27 +- codewiki/mcp/tools/page_router.py | 39 +- codewiki/mcp/tools/prompt_server.py | 243 +-- codewiki/mcp/tools/reading_guide.py | 42 +- codewiki/mcp/tools/review_changes.py | 39 +- codewiki/mcp/tools/review_checklist.py | 4 +- codewiki/mcp/tools/schema_generator.py | 57 +- codewiki/mcp/tools/source_ingest.py | 143 +- codewiki/mcp/tools/task_manager.py | 24 +- codewiki/mcp/tools/telemetry.py | 56 +- codewiki/mcp/tools/watch.py | 34 +- codewiki/mcp/tools/wiki_index.py | 41 +- codewiki/mcp/tools/wiki_lint.py | 853 ++++++----- codewiki/mcp/tools/wiki_search.py | 428 ++++-- codewiki/mcp/tools/workspace_analyzer.py | 246 +-- codewiki/mcp/tools/workspace_result.py | 7 +- codewiki/mcp/workspace.py | 5 +- codewiki/run_web_app.py | 4 +- codewiki/src/__init__.py | 1 - codewiki/src/be/__init__.py | 1 - codewiki/src/be/agent_tools/__init__.py | 1 - codewiki/src/be/agent_tools/deps.py | 3 +- .../generate_sub_module_documentations.py | 48 +- .../be/agent_tools/read_code_components.py | 12 +- codewiki/src/be/backend.py | 2 + codewiki/src/be/caw_backend.py | 11 +- codewiki/src/be/caw_toolkit.py | 11 +- codewiki/src/be/cluster_modules.py | 37 +- .../src/be/dependency_analyzer/__init__.py | 36 +- .../analysis/analysis_service.py | 61 +- .../analysis/call_graph_analyzer.py | 79 +- .../dependency_analyzer/analysis/cloning.py | 13 +- .../analysis/cross_service_matcher.py | 104 +- .../analysis/infra_scanner.py | 46 +- .../analysis/repo_analyzer.py | 8 +- .../analysis/service_detector.py | 125 +- .../analysis/topology_visualizer.py | 5 +- .../src/be/dependency_analyzer/analyzers/c.py | 437 +++--- .../be/dependency_analyzer/analyzers/cpp.py | 1360 +++++++++-------- .../dependency_analyzer/analyzers/csharp.py | 155 +- .../be/dependency_analyzer/analyzers/go.py | 308 ++-- .../be/dependency_analyzer/analyzers/java.py | 1135 +++++++------- .../analyzers/javascript.py | 234 ++- .../dependency_analyzer/analyzers/kotlin.py | 328 ++-- .../be/dependency_analyzer/analyzers/php.py | 228 ++- .../dependency_analyzer/analyzers/python.py | 60 +- .../analyzers/route_extractors/__init__.py | 1 + .../analyzers/route_extractors/go_routes.py | 161 +- .../analyzers/route_extractors/java_routes.py | 255 ++-- .../analyzers/route_extractors/js_routes.py | 118 +- .../analyzers/route_extractors/mq_patterns.py | 111 +- .../route_extractors/python_routes.py | 172 ++- .../analyzers/typescript.py | 675 ++++---- .../src/be/dependency_analyzer/ast_parser.py | 108 +- .../dependency_graphs_builder.py | 55 +- .../src/be/dependency_analyzer/models/core.py | 26 +- .../models/cross_service.py | 17 +- .../src/be/dependency_analyzer/topo_sort.py | 156 +- .../utils/external_symbols.py | 141 +- .../utils/logging_config.py | 78 +- .../utils/path_canonicalizer.py | 1 + .../be/dependency_analyzer/utils/patterns.py | 15 +- .../be/dependency_analyzer/utils/security.py | 3 + codewiki/src/be/documentation_generator.py | 151 +- codewiki/src/be/main.py | 17 +- codewiki/src/be/prompt_template.py | 140 +- codewiki/src/be/pydantic_ai_backend.py | 1 + codewiki/src/be/utils.py | 88 +- codewiki/src/config.py | 164 +- codewiki/src/fe/__init__.py | 22 +- codewiki/src/fe/background_worker.py | 153 +- codewiki/src/fe/cache_manager.py | 58 +- codewiki/src/fe/config.py | 26 +- codewiki/src/fe/github_processor.py | 81 +- codewiki/src/fe/models.py | 6 +- codewiki/src/fe/routes.py | 158 +- codewiki/src/fe/template_utils.py | 39 +- codewiki/src/fe/templates.py | 2 +- codewiki/src/fe/visualise_docs.py | 137 +- codewiki/src/fe/web_app.py | 47 +- codewiki/src/frontmatter.py | 67 +- codewiki/src/utils.py | 14 +- ...51\345\261\225\346\226\271\346\241\210.md" | 38 +- ...7\233 Backlog\357\274\210CodeWiki-P....md" | 11 +- ...02\351\205\215\346\226\271\346\241\210.md" | 4 +- ...00\346\234\257\350\247\206\350\247\222.md" | 19 +- ...37\351\211\264\345\210\206\346\236\220.md" | 27 +- ...71\351\275\220\347\240\224\347\251\266.md" | 2 +- docs/plans/ember-bay-sparrow.md | 5 +- docs/plans/windy-mesa-stork.md | 1 + ...36\347\216\260\350\256\241\345\210\222.md" | 41 +- pyproject.toml | 18 +- repowiki/wiki/index.md | 2 +- repowiki/wiki/modules/AnalysisPipeline.md | 4 +- repowiki/wiki/modules/AnalyzerModels.md | 13 +- repowiki/wiki/modules/AnalyzerUtils.md | 10 +- repowiki/wiki/modules/CLI_Config.md | 22 +- repowiki/wiki/modules/CLI_Utils.md | 4 +- repowiki/wiki/modules/GraphAndSort.md | 10 +- repowiki/wiki/modules/LLM_Backend.md | 2 +- repowiki/wiki/modules/MCP_Tools_Dependency.md | 6 +- repowiki/wiki/modules/RouteExtractors.md | 6 +- repowiki/wiki/modules/SharedConfig.md | 3 +- scripts/_tmp_analyze_deleted.py | 18 +- scripts/backfill_aliases.py | 6 +- scripts/migrate_freshness.py | 6 +- scripts/migrate_okf.py | 47 +- tests/conftest.py | 6 +- tests/okf_regression_test.py | 738 ++++++--- tests/smoke_test_mcp.py | 861 +++++++---- tests/telemetry_seed.py | 1 + tests/test_adoption.py | 66 +- tests/test_authority_p0.py | 33 +- tests/test_change_analysis.py | 12 +- tests/test_consolidation_p2.py | 184 ++- tests/test_distill_cleanup.py | 160 +- tests/test_distill_p1.py | 360 +++-- tests/test_doctrine_p3.py | 141 +- tests/test_freshness.py | 144 +- tests/test_friction.py | 150 +- tests/test_hook_registry.py | 40 +- tests/test_ide_hook_capture.py | 485 +++--- tests/test_index_freshness.py | 52 +- tests/test_l0_archive.py | 223 ++- tests/test_latest_compat.py | 1 + tests/test_lint_fix.py | 16 +- tests/test_low_adoption.py | 47 +- tests/test_module_tree_validation.py | 18 +- tests/test_ontology_graph.py | 50 +- tests/test_openviking_borrowings.py | 125 +- tests/test_promotion.py | 56 +- tests/test_query_transparency.py | 31 +- tests/test_review_changes.py | 28 +- tests/test_strip_system_injection.py | 12 +- tests/test_task_manager.py | 30 +- tests/test_task_session_start.py | 3 +- tests/test_team_telemetry.py | 76 +- tests/test_transcript_filters.py | 1 + tests/test_usage_ranking.py | 187 ++- tests/test_watch.py | 27 +- uv.lock | 154 +- 213 files changed, 13129 insertions(+), 8345 deletions(-) create mode 100644 .git-blame-ignore-revs create mode 100644 .pre-commit-config.yaml create mode 100644 CONTRIBUTING.md diff --git a/.codebuddy/hooks/capture_session_end.py b/.codebuddy/hooks/capture_session_end.py index 4b1891b..9a4f60e 100644 --- a/.codebuddy/hooks/capture_session_end.py +++ b/.codebuddy/hooks/capture_session_end.py @@ -52,6 +52,7 @@ Stdout is emitted in the CodeBuddy-expected ``{continue, systemMessage}`` shape. """ + from __future__ import annotations import json @@ -59,7 +60,6 @@ import subprocess import sys import tempfile -from datetime import datetime, timezone from pathlib import Path REPO = Path(__file__).resolve().parents[2] # /.codebuddy/hooks/ -> @@ -166,9 +166,12 @@ def main() -> int: json.dump(event, fh) cmd = [ - sys.executable, "-m", "codewiki.mcp._ide_hook", + sys.executable, + "-m", + "codewiki.mcp._ide_hook", "--enable", - "--repo-path", repo_path, + "--repo-path", + repo_path, ] if tmp: cmd += ["--conversation", tmp] @@ -185,8 +188,12 @@ def main() -> int: child_env = dict(env) if tmp: child_env["CODEWIKI_HOOK_EVENT_FILE"] = tmp - kwargs: dict = {"cwd": str(REPO), "env": child_env, "stdout": subprocess.DEVNULL, - "stderr": subprocess.DEVNULL} + kwargs: dict = { + "cwd": str(REPO), + "env": child_env, + "stdout": subprocess.DEVNULL, + "stderr": subprocess.DEVNULL, + } if sys.platform == "win32": kwargs["creationflags"] = getattr(subprocess, "DETACHED_PROCESS", 0) else: @@ -195,12 +202,14 @@ def main() -> int: except Exception as e: # noqa: BLE001 - never crash the IDE hook # If we cannot even spawn the child, surface a non-blocking hint but # still let the session end cleanly. - print(json.dumps({"continue": True, - "systemMessage": f"team-memory capture not started: {e}"})) + print( + json.dumps({"continue": True, "systemMessage": f"team-memory capture not started: {e}"}) + ) return 0 - print(json.dumps({"continue": True, - "systemMessage": "team-memory capture started in background"})) + print( + json.dumps({"continue": True, "systemMessage": "team-memory capture started in background"}) + ) return 0 diff --git a/.codebuddy/plans/multi-ide-hook-wiring_458a6f24.md b/.codebuddy/plans/multi-ide-hook-wiring_458a6f24.md index f7babd7..c93d38a 100644 --- a/.codebuddy/plans/multi-ide-hook-wiring_458a6f24.md +++ b/.codebuddy/plans/multi-ide-hook-wiring_458a6f24.md @@ -146,15 +146,34 @@ repowiki/ ```python # codewiki/cli/utils/ide_config.py IDE_SPECS: dict[str, dict] = { - "codebuddy": {"dir": ".codebuddy", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, - "qoder": {"dir": ".qoder", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, - "claude-code": {"dir": ".claude", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, + "codebuddy": { + "dir": ".codebuddy", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, + "qoder": { + "dir": ".qoder", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, + "claude-code": { + "dir": ".claude", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, } HOOK_FILES = ("capture_session_end.py", "task_session_start.py") AGENT_FILE = "distill-worker.md" HOOKS_REGISTRATION = { # 事件注册骨架，command 运行时补全绝对路径 - "SessionStart": [{"matcher": "startup", "hooks": [{"type": "command", "command": "", "timeout": 15}]}], - "SessionEnd": [{"matcher": "other", "hooks": [{"type": "command", "command": "", "timeout": 30}]}], + "SessionStart": [ + {"matcher": "startup", "hooks": [{"type": "command", "command": "", "timeout": 15}]} + ], + "SessionEnd": [ + {"matcher": "other", "hooks": [{"type": "command", "command": "", "timeout": 30}]} + ], } ``` diff --git "a/.codebuddy/plans/multi-ide-hook-wiring_f1a46cd6(\346\234\252\345\256\214\346\210\220).md" "b/.codebuddy/plans/multi-ide-hook-wiring_f1a46cd6(\346\234\252\345\256\214\346\210\220).md" index c923fa6..7a41f53 100644 --- "a/.codebuddy/plans/multi-ide-hook-wiring_f1a46cd6(\346\234\252\345\256\214\346\210\220).md" +++ "b/.codebuddy/plans/multi-ide-hook-wiring_f1a46cd6(\346\234\252\345\256\214\346\210\220).md" @@ -140,15 +140,34 @@ repowiki/ ```python # codewiki/cli/utils/ide_config.py IDE_SPECS: dict[str, dict] = { - "codebuddy": {"dir": ".codebuddy", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, - "qoder": {"dir": ".qoder", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, - "claude-code": {"dir": ".claude", "settings": "settings.json", "agents_dir": "agents", "copy_agent": True}, + "codebuddy": { + "dir": ".codebuddy", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, + "qoder": { + "dir": ".qoder", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, + "claude-code": { + "dir": ".claude", + "settings": "settings.json", + "agents_dir": "agents", + "copy_agent": True, + }, } HOOK_FILES = ("capture_session_end.py", "task_session_start.py") AGENT_FILE = "distill-worker.md" HOOKS_REGISTRATION = { # 事件注册骨架，command 运行时补全绝对路径 - "SessionStart": [{"matcher": "startup", "hooks": [{"type": "command", "command": "", "timeout": 15}]}], - "SessionEnd": [{"matcher": "other", "hooks": [{"type": "command", "command": "", "timeout": 30}]}], + "SessionStart": [ + {"matcher": "startup", "hooks": [{"type": "command", "command": "", "timeout": 15}]} + ], + "SessionEnd": [ + {"matcher": "other", "hooks": [{"type": "command", "command": "", "timeout": 30}]} + ], } ``` diff --git "a/.codebuddy/plans/sessionStart-\350\241\245\350\222\270\351\246\217\346\263\250\345\205\245\346\226\271\346\241\210_c8ec3f3e.md" "b/.codebuddy/plans/sessionStart-\350\241\245\350\222\270\351\246\217\346\263\250\345\205\245\346\226\271\346\241\210_c8ec3f3e.md" index 7080124..26ca07b 100644 --- "a/.codebuddy/plans/sessionStart-\350\241\245\350\222\270\351\246\217\346\263\250\345\205\245\346\226\271\346\241\210_c8ec3f3e.md" +++ "b/.codebuddy/plans/sessionStart-\350\241\245\350\222\270\351\246\217\346\263\250\345\205\245\346\226\271\346\241\210_c8ec3f3e.md" @@ -133,6 +133,7 @@ d:/repos/CodeWiki-CN/ def pending_raws_by_task(output_dir: Path) -> Dict[str, List[Dict[str, str]]]: """按 task_id 聚合未蒸馏 raw 条目；无 task_id 的归入 "" 键。""" + # distill_conversation 新参数（现有参数不变） # arguments["task_id"]: Optional[str] — 仅蒸馏该任务的积压对话 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..7984006 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# Ignore bulk format commit(s) in git blame +# After this PR merges, replace the placeholder below with the actual bulk-format commit hash: +# git log --oneline --grep="ruff format whole repo" | head -n1 +# Then add the hash here (one per line, with comment). GitHub respects this file for blame view. +# Configure locally: git config blame.ignoreRevsFile .git-blame-ignore-revs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index febcd7b..4a083e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,3 +67,8 @@ jobs: run: | xargs -0 -r uv run ruff check --output-format=github \ < "$RUNNER_TEMP/changed-python-files" + + - name: Ruff format check changed files + run: | + xargs -0 -r uv run ruff format --check \ + < "$RUNNER_TEMP/changed-python-files" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..50d9f4c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +# Pre-commit hooks for CodeWiki-Plus +# Setup: uv sync --frozen && uv run pre-commit install +# Run manually: uv run pre-commit run --all-files +# CI also enforces these on changed files (see .github/workflows/ci.yml) + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.3 # must match ruff pin in pyproject.toml + hooks: + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/.qoder/hooks/capture_session_end.py b/.qoder/hooks/capture_session_end.py index 4b1891b..9a4f60e 100644 --- a/.qoder/hooks/capture_session_end.py +++ b/.qoder/hooks/capture_session_end.py @@ -52,6 +52,7 @@ Stdout is emitted in the CodeBuddy-expected ``{continue, systemMessage}`` shape. """ + from __future__ import annotations import json @@ -59,7 +60,6 @@ import subprocess import sys import tempfile -from datetime import datetime, timezone from pathlib import Path REPO = Path(__file__).resolve().parents[2] # /.codebuddy/hooks/ -> @@ -166,9 +166,12 @@ def main() -> int: json.dump(event, fh) cmd = [ - sys.executable, "-m", "codewiki.mcp._ide_hook", + sys.executable, + "-m", + "codewiki.mcp._ide_hook", "--enable", - "--repo-path", repo_path, + "--repo-path", + repo_path, ] if tmp: cmd += ["--conversation", tmp] @@ -185,8 +188,12 @@ def main() -> int: child_env = dict(env) if tmp: child_env["CODEWIKI_HOOK_EVENT_FILE"] = tmp - kwargs: dict = {"cwd": str(REPO), "env": child_env, "stdout": subprocess.DEVNULL, - "stderr": subprocess.DEVNULL} + kwargs: dict = { + "cwd": str(REPO), + "env": child_env, + "stdout": subprocess.DEVNULL, + "stderr": subprocess.DEVNULL, + } if sys.platform == "win32": kwargs["creationflags"] = getattr(subprocess, "DETACHED_PROCESS", 0) else: @@ -195,12 +202,14 @@ def main() -> int: except Exception as e: # noqa: BLE001 - never crash the IDE hook # If we cannot even spawn the child, surface a non-blocking hint but # still let the session end cleanly. - print(json.dumps({"continue": True, - "systemMessage": f"team-memory capture not started: {e}"})) + print( + json.dumps({"continue": True, "systemMessage": f"team-memory capture not started: {e}"}) + ) return 0 - print(json.dumps({"continue": True, - "systemMessage": "team-memory capture started in background"})) + print( + json.dumps({"continue": True, "systemMessage": "team-memory capture started in background"}) + ) return 0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fc609ec --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,11 @@ +# Contributing + +Prerequisites: Python 3.12+, [uv](https://docs.astral.sh/uv/) (or `pip`). + +```bash +git clone https://github.com/mambo-wang/CodeWiki-Plus.git +cd CodeWiki-Plus +uv sync --frozen # or `pip install -e .[dev]` +uv run pre-commit install # enables ruff check + format on commit +uv run pytest -q # verify setup +``` diff --git a/codewiki/__main__.py b/codewiki/__main__.py index bceeeb9..09af1ef 100644 --- a/codewiki/__main__.py +++ b/codewiki/__main__.py @@ -5,4 +5,4 @@ from codewiki.cli.main import cli if __name__ == "__main__": - cli() \ No newline at end of file + cli() diff --git a/codewiki/cli/__init__.py b/codewiki/cli/__init__.py index e6e485a..128a92f 100644 --- a/codewiki/cli/__init__.py +++ b/codewiki/cli/__init__.py @@ -1,4 +1,3 @@ """CLI module for CodeWiki.""" __all__ = [] - diff --git a/codewiki/cli/adapters/__init__.py b/codewiki/cli/adapters/__init__.py index 6204b52..05c9ace 100644 --- a/codewiki/cli/adapters/__init__.py +++ b/codewiki/cli/adapters/__init__.py @@ -1,4 +1,3 @@ """Adapters for integrating with backend modules.""" __all__ = [] - diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index c91ff6f..16784ed 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -26,11 +26,11 @@ class CLIDocumentationGenerator: """ CLI adapter for documentation generation with progress reporting. - + This class wraps the backend documentation generator and adds CLI-specific features like progress tracking and error handling. """ - + def __init__( self, repo_path: Path, @@ -42,7 +42,7 @@ def __init__( ): """ Initialize the CLI documentation generator. - + Args: repo_path: Repository path output_dir: Output directory @@ -59,133 +59,133 @@ def __init__( self.commit_id = commit_id self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) self.job = DocumentationJob() - + # Setup job metadata self.job.repository_path = str(repo_path) self.job.repository_name = repo_path.name self.job.output_directory = str(output_dir) self.job.llm_config = LLMConfig( - main_model=config.get('main_model', ''), - cluster_model=config.get('cluster_model', ''), - base_url=config.get('base_url', '') + main_model=config.get("main_model", ""), + cluster_model=config.get("cluster_model", ""), + base_url=config.get("base_url", ""), ) - + # Configure backend logging self._configure_backend_logging() - + def _configure_backend_logging(self): """Configure backend logger for CLI use with colored output.""" from codewiki.src.be.dependency_analyzer.utils.logging_config import ColoredFormatter - + # Get backend logger (parent of all backend modules) - backend_logger = logging.getLogger('codewiki.src.be') - + backend_logger = logging.getLogger("codewiki.src.be") + # Remove existing handlers to avoid duplicates backend_logger.handlers.clear() - + if self.verbose: # In verbose mode, show INFO and above backend_logger.setLevel(logging.INFO) - + # Create console handler with formatting console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) - + # Use colored formatter for better readability colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + # Add handler to logger backend_logger.addHandler(console_handler) else: # In non-verbose mode, suppress backend logs (use WARNING level to hide INFO/DEBUG) backend_logger.setLevel(logging.WARNING) - + # Create console handler for warnings and errors only console_handler = logging.StreamHandler(sys.stderr) console_handler.setLevel(logging.WARNING) - + # Use colored formatter even for warnings/errors colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + backend_logger.addHandler(console_handler) - + # Prevent propagation to root logger to avoid duplicate messages backend_logger.propagate = False - + def generate(self) -> DocumentationJob: """ Generate documentation with progress tracking. - + Returns: Completed DocumentationJob - + Raises: APIError: If LLM API call fails """ self.job.start() start_time = time.time() - + try: # Set CLI context for backend set_cli_context(True) - + # Create backend config with CLI settings backend_config = BackendConfig.from_cli( repo_path=str(self.repo_path), output_dir=str(self.output_dir), - llm_base_url=self.config.get('base_url'), - llm_api_key=self.config.get('api_key'), - main_model=self.config.get('main_model'), - cluster_model=self.config.get('cluster_model'), - fallback_model=self.config.get('fallback_model'), - provider=self.config.get('provider', 'openai-compatible'), - aws_region=self.config.get('aws_region', 'us-east-1'), - max_tokens=self.config.get('max_tokens', 32768), - max_token_per_module=self.config.get('max_token_per_module', 36369), - max_token_per_leaf_module=self.config.get('max_token_per_leaf_module', 16000), - max_depth=self.config.get('max_depth', 3), - agent_instructions=self.config.get('agent_instructions') + llm_base_url=self.config.get("base_url"), + llm_api_key=self.config.get("api_key"), + main_model=self.config.get("main_model"), + cluster_model=self.config.get("cluster_model"), + fallback_model=self.config.get("fallback_model"), + provider=self.config.get("provider", "openai-compatible"), + aws_region=self.config.get("aws_region", "us-east-1"), + max_tokens=self.config.get("max_tokens", 32768), + max_token_per_module=self.config.get("max_token_per_module", 36369), + max_token_per_leaf_module=self.config.get("max_token_per_leaf_module", 16000), + max_depth=self.config.get("max_depth", 3), + agent_instructions=self.config.get("agent_instructions"), ) - + # Run backend documentation generation asyncio.run(self._run_backend_generation(backend_config)) - + # Stage 4: HTML Generation (optional) if self.generate_html: self._run_html_generation() - + # Stage 5: Finalization (metadata already created by backend) self._finalize_job() - + # Complete job - generation_time = time.time() - start_time + time.time() - start_time self.job.complete() - + return self.job - + except APIError as e: self.job.fail(str(e)) raise except Exception as e: self.job.fail(str(e)) raise - + async def _run_backend_generation(self, backend_config: BackendConfig): """Run the backend documentation generation with progress tracking.""" - + # Stage 1: Dependency Analysis self.progress_tracker.start_stage(1, "Dependency Analysis") if self.verbose: self.progress_tracker.update_stage(0.2, "Initializing dependency analyzer...") - + # Create documentation generator doc_generator = DocumentationGenerator(backend_config, commit_id=self.commit_id) - + if self.verbose: self.progress_tracker.update_stage(0.5, "Parsing source files...") - + # Build dependency graph try: components, leaf_nodes, _routes = doc_generator.graph_builder.build_dependency_graph() @@ -193,29 +193,37 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.job.statistics.leaf_nodes = len(leaf_nodes) if self.verbose: - self.progress_tracker.update_stage(0.8, f"Analyzed {len(components)} files, found {len(leaf_nodes)} leaf nodes") + self.progress_tracker.update_stage( + 0.8, f"Analyzed {len(components)} files, found {len(leaf_nodes)} leaf nodes" + ) # Log individual files analyzed for comp_name in sorted(components.keys())[:20]: self.progress_tracker.update_stage(0.9, f" File: {comp_name}") if len(components) > 20: - self.progress_tracker.update_stage(0.9, f" ... and {len(components) - 20} more files") + self.progress_tracker.update_stage( + 0.9, f" ... and {len(components) - 20} more files" + ) except Exception as e: raise APIError(f"Dependency analysis failed: {e}") - + self.progress_tracker.complete_stage() - + # Stage 2: Module Clustering self.progress_tracker.start_stage(2, "Module Clustering") if self.verbose: self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...") - + # Import clustering function from codewiki.src.be.cluster_modules import ( cluster_modules, get_clustering_input_token_count, ) from codewiki.src.utils import file_manager - from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME, meta_resolve + from codewiki.src.config import ( + FIRST_MODULE_TREE_FILENAME, + MODULE_TREE_FILENAME, + meta_resolve, + ) working_dir = str(self.output_dir.absolute()) file_manager.ensure_directory(working_dir) @@ -229,9 +237,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self.progress_tracker.update_stage(0.5, "Loaded cached module tree") else: if self.verbose: - clustering_tokens = get_clustering_input_token_count( - leaf_nodes, components - ) + clustering_tokens = get_clustering_input_token_count(leaf_nodes, components) self.progress_tracker.update_stage( 0.3, ( @@ -274,78 +280,87 @@ async def _run_backend_generation(self, backend_config: BackendConfig): f"Created {len(module_tree)} modules", ) for mod_name in sorted(module_tree.keys()): - file_count = len(module_tree[mod_name]) if isinstance(module_tree[mod_name], list) else "?" - self.progress_tracker.update_stage(1.0, f" Module: {mod_name} ({file_count} files)") + file_count = ( + len(module_tree[mod_name]) + if isinstance(module_tree[mod_name], list) + else "?" + ) + self.progress_tracker.update_stage( + 1.0, f" Module: {mod_name} ({file_count} files)" + ) except Exception as e: raise APIError(f"Module clustering failed: {e}") - + self.progress_tracker.complete_stage() - + # Stage 3: Documentation Generation self.progress_tracker.start_stage(3, "Documentation Generation") if self.verbose: self.progress_tracker.update_stage(0.1, "Generating module documentation...") - + try: if self.verbose: - self.progress_tracker.update_stage(0.2, f"Generating documentation for {self.job.module_count} modules...") + self.progress_tracker.update_stage( + 0.2, f"Generating documentation for {self.job.module_count} modules..." + ) # Run the actual documentation generation await doc_generator.generate_module_documentation(components, leaf_nodes) if self.verbose: self.progress_tracker.update_stage(0.9, "Creating repository overview...") - + # Create metadata doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - + # Collect generated files for file_path in os.listdir(working_dir): - if file_path.endswith('.md') or file_path.endswith('.json'): + if file_path.endswith(".md") or file_path.endswith(".json"): self.job.files_generated.append(file_path) - + except Exception as e: raise APIError(f"Documentation generation failed: {e}") - + self.progress_tracker.complete_stage() - + def _run_html_generation(self): """Run HTML generation stage.""" self.progress_tracker.start_stage(4, "HTML Generation") - + from codewiki.cli.html_generator import HTMLGenerator - + # Generate HTML html_generator = HTMLGenerator() - + if self.verbose: self.progress_tracker.update_stage(0.3, "Loading module tree and metadata...") - + repo_info = html_generator.detect_repository_info(self.repo_path) - + # Generate HTML with auto-loading of module_tree and metadata from docs_dir output_path = self.output_dir / "index.html" html_generator.generate( output_path=output_path, - title=repo_info['name'], - repository_url=repo_info['url'], - github_pages_url=repo_info['github_pages_url'], - docs_dir=self.output_dir # Auto-load module_tree and metadata from here + title=repo_info["name"], + repository_url=repo_info["url"], + github_pages_url=repo_info["github_pages_url"], + docs_dir=self.output_dir, # Auto-load module_tree and metadata from here ) - + self.job.files_generated.append("index.html") - + if self.verbose: self.progress_tracker.update_stage(1.0, "Generated index.html") - + self.progress_tracker.complete_stage() - + def _finalize_job(self): """Finalize the job (metadata already created by backend).""" # Just verify metadata exists from codewiki.src.config import meta_join + metadata_path = Path(meta_join(self.output_dir, "metadata.json")) if not metadata_path.exists(): # Create our own if backend didn't - with open(metadata_path, 'w') as f: + with open(metadata_path, "w") as f: f.write(self.job.to_json()) diff --git a/codewiki/cli/commands/__init__.py b/codewiki/cli/commands/__init__.py index b790730..387901f 100644 --- a/codewiki/cli/commands/__init__.py +++ b/codewiki/cli/commands/__init__.py @@ -3,4 +3,3 @@ from codewiki.cli.commands.install_hooks import install_hooks __all__ = ["install_hooks"] - diff --git a/codewiki/cli/commands/config.py b/codewiki/cli/commands/config.py index d04193a..d614928 100644 --- a/codewiki/cli/commands/config.py +++ b/codewiki/cli/commands/config.py @@ -10,17 +10,16 @@ from codewiki.cli.config_manager import ConfigManager from codewiki.cli.models.config import AgentInstructions from codewiki.cli.utils.errors import ( - ConfigurationError, - handle_error, - EXIT_SUCCESS, - EXIT_CONFIG_ERROR + ConfigurationError, + handle_error, + EXIT_CONFIG_ERROR, ) from codewiki.cli.utils.validation import ( validate_url, validate_api_key, validate_model_name, is_top_tier_model, - mask_api_key + mask_api_key, ) @@ -28,7 +27,7 @@ def parse_patterns(patterns_str: str) -> List[str]: """Parse comma-separated patterns into a list.""" if not patterns_str: return [] - return [p.strip() for p in patterns_str.split(',') if p.strip()] + return [p.strip() for p in patterns_str.split(",") if p.strip()] @click.group(name="config") @@ -38,55 +37,27 @@ def config_group(): @config_group.command(name="set") -@click.option( - "--api-key", - type=str, - help="LLM API key (stored securely in system keychain)" -) -@click.option( - "--base-url", - type=str, - help="LLM API base URL (e.g., https://api.anthropic.com)" -) -@click.option( - "--main-model", - type=str, - help="Primary model for documentation generation" -) -@click.option( - "--cluster-model", - type=str, - help="Model for module clustering (recommend top-tier)" -) -@click.option( - "--fallback-model", - type=str, - help="Fallback model for documentation generation" -) -@click.option( - "--max-tokens", - type=int, - help="Maximum tokens for LLM response (default: 32768)" -) +@click.option("--api-key", type=str, help="LLM API key (stored securely in system keychain)") +@click.option("--base-url", type=str, help="LLM API base URL (e.g., https://api.anthropic.com)") +@click.option("--main-model", type=str, help="Primary model for documentation generation") +@click.option("--cluster-model", type=str, help="Model for module clustering (recommend top-tier)") +@click.option("--fallback-model", type=str, help="Fallback model for documentation generation") +@click.option("--max-tokens", type=int, help="Maximum tokens for LLM response (default: 32768)") @click.option( "--max-token-per-module", type=int, - help="Maximum tokens per module for clustering (default: 36369)" + help="Maximum tokens per module for clustering (default: 36369)", ) @click.option( - "--max-token-per-leaf-module", - type=int, - help="Maximum tokens per leaf module (default: 16000)" + "--max-token-per-leaf-module", type=int, help="Maximum tokens per leaf module (default: 16000)" ) @click.option( - "--max-depth", - type=int, - help="Maximum depth for hierarchical decomposition (default: 2)" + "--max-depth", type=int, help="Maximum depth for hierarchical decomposition (default: 2)" ) @click.option( "--provider", type=click.Choice( - ['openai-compatible', 'anthropic', 'bedrock', 'azure-openai', 'claude-code', 'codex'], + ["openai-compatible", "anthropic", "bedrock", "azure-openai", "claude-code", "codex"], case_sensitive=False, ), help=( @@ -94,21 +65,11 @@ def config_group(): "Use 'claude-code' or 'codex' to run on a CLI subscription instead of an API key." ), ) +@click.option("--aws-region", type=str, help="AWS region for Bedrock provider (default: us-east-1)") @click.option( - "--aws-region", - type=str, - help="AWS region for Bedrock provider (default: us-east-1)" -) -@click.option( - "--api-version", - type=str, - help="Azure OpenAI API version (default: 2024-12-01-preview)" -) -@click.option( - "--azure-deployment", - type=str, - help="Azure OpenAI deployment name" + "--api-version", type=str, help="Azure OpenAI API version (default: 2024-12-01-preview)" ) +@click.option("--azure-deployment", type=str, help="Azure OpenAI deployment name") def config_set( api_key: Optional[str], base_url: Optional[str], @@ -122,7 +83,7 @@ def config_set( provider: Optional[str] = None, aws_region: Optional[str] = None, api_version: Optional[str] = None, - azure_deployment: Optional[str] = None + azure_deployment: Optional[str] = None, ): """ Set configuration values for CodeWiki. @@ -166,80 +127,96 @@ def config_set( """ try: # Check if at least one option is provided - if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment]): + if not any( + [ + api_key, + base_url, + main_model, + cluster_model, + fallback_model, + max_tokens, + max_token_per_module, + max_token_per_leaf_module, + max_depth, + provider, + aws_region, + api_version, + azure_deployment, + ] + ): click.echo("No options provided. Use --help for usage information.") sys.exit(EXIT_CONFIG_ERROR) - + # Validate inputs before saving validated_data = {} - + if api_key: - validated_data['api_key'] = validate_api_key(api_key) - + validated_data["api_key"] = validate_api_key(api_key) + if base_url: - validated_data['base_url'] = validate_url(base_url) - + validated_data["base_url"] = validate_url(base_url) + if main_model: - validated_data['main_model'] = validate_model_name(main_model) - + validated_data["main_model"] = validate_model_name(main_model) + if cluster_model: - validated_data['cluster_model'] = validate_model_name(cluster_model) - + validated_data["cluster_model"] = validate_model_name(cluster_model) + if fallback_model: - validated_data['fallback_model'] = validate_model_name(fallback_model) - + validated_data["fallback_model"] = validate_model_name(fallback_model) + if max_tokens is not None: if max_tokens < 1: raise ConfigurationError("max_tokens must be a positive integer") - validated_data['max_tokens'] = max_tokens - + validated_data["max_tokens"] = max_tokens + if max_token_per_module is not None: if max_token_per_module < 1: raise ConfigurationError("max_token_per_module must be a positive integer") - validated_data['max_token_per_module'] = max_token_per_module - + validated_data["max_token_per_module"] = max_token_per_module + if max_token_per_leaf_module is not None: if max_token_per_leaf_module < 1: raise ConfigurationError("max_token_per_leaf_module must be a positive integer") - validated_data['max_token_per_leaf_module'] = max_token_per_leaf_module - + validated_data["max_token_per_leaf_module"] = max_token_per_leaf_module + if max_depth is not None: if max_depth < 1: raise ConfigurationError("max_depth must be a positive integer") - validated_data['max_depth'] = max_depth + validated_data["max_depth"] = max_depth if provider is not None: - validated_data['provider'] = provider + validated_data["provider"] = provider if aws_region is not None: - validated_data['aws_region'] = aws_region + validated_data["aws_region"] = aws_region if api_version is not None: - validated_data['api_version'] = api_version + validated_data["api_version"] = api_version if azure_deployment is not None: - validated_data['azure_deployment'] = azure_deployment + validated_data["azure_deployment"] = azure_deployment # Create config manager and save manager = ConfigManager() manager.load() # Load existing config if present manager.save( - api_key=validated_data.get('api_key'), - base_url=validated_data.get('base_url'), - main_model=validated_data.get('main_model'), - cluster_model=validated_data.get('cluster_model'), - fallback_model=validated_data.get('fallback_model'), - max_tokens=validated_data.get('max_tokens'), - max_token_per_module=validated_data.get('max_token_per_module'), - max_token_per_leaf_module=validated_data.get('max_token_per_leaf_module'), - max_depth=validated_data.get('max_depth'), - provider=validated_data.get('provider'), - aws_region=validated_data.get('aws_region'), - api_version=validated_data.get('api_version'), - azure_deployment=validated_data.get('azure_deployment') + api_key=validated_data.get("api_key"), + base_url=validated_data.get("base_url"), + main_model=validated_data.get("main_model"), + cluster_model=validated_data.get("cluster_model"), + fallback_model=validated_data.get("fallback_model"), + max_tokens=validated_data.get("max_tokens"), + max_token_per_module=validated_data.get("max_token_per_module"), + max_token_per_leaf_module=validated_data.get("max_token_per_leaf_module"), + max_depth=validated_data.get("max_depth"), + provider=validated_data.get("provider"), + aws_region=validated_data.get("aws_region"), + api_version=validated_data.get("api_version"), + azure_deployment=validated_data.get("azure_deployment"), ) - + # Display success messages click.echo() if api_key: @@ -247,42 +224,41 @@ def config_set( click.secho("✓ API key saved to system keychain", fg="green") else: click.secho( - "⚠️ System keychain unavailable. API key stored in encrypted file.", - fg="yellow" + "⚠️ System keychain unavailable. API key stored in encrypted file.", fg="yellow" ) - + if base_url: click.secho(f"✓ Base URL: {base_url}", fg="green") - + if main_model: click.secho(f"✓ Main model: {main_model}", fg="green") - + if cluster_model: click.secho(f"✓ Cluster model: {cluster_model}", fg="green") - + # Warn if not using top-tier model for clustering if not is_top_tier_model(cluster_model): click.secho( "\n⚠️ Cluster model is not a top-tier LLM. " "Documentation quality may be suboptimal.", - fg="yellow" + fg="yellow", ) click.echo( " Recommended models: claude-opus, claude-sonnet-4, gpt-4, gpt-4-turbo" ) - + if fallback_model: click.secho(f"✓ Fallback model: {fallback_model}", fg="green") - + if max_tokens: click.secho(f"✓ Max tokens: {max_tokens}", fg="green") - + if max_token_per_module: click.secho(f"✓ Max token per module: {max_token_per_module}", fg="green") - + if max_token_per_leaf_module: click.secho(f"✓ Max token per leaf module: {max_token_per_leaf_module}", fg="green") - + if max_depth: click.secho(f"✓ Max depth: {max_depth}", fg="green") @@ -299,7 +275,7 @@ def config_set( click.secho(f"✓ Azure Deployment: {azure_deployment}", fg="green") click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True)) - + except ConfigurationError as e: click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) sys.exit(e.exit_code) @@ -308,31 +284,26 @@ def config_set( @config_group.command(name="show") -@click.option( - "--json", - "output_json", - is_flag=True, - help="Output in JSON format" -) +@click.option("--json", "output_json", is_flag=True, help="Output in JSON format") def config_show(output_json: bool): """ Display current configuration. - + API keys are masked for security (showing only first and last 4 characters). - + Examples: - + \b # Display configuration $ codewiki config show - + \b # Display as JSON $ codewiki config show --json """ try: manager = ConfigManager() - + if not manager.load(): click.secho("\n✗ Configuration not found.", fg="red", err=True) click.echo("\nPlease run 'codewiki config set' to configure your API credentials:") @@ -340,10 +311,10 @@ def config_show(output_json: bool): click.echo(" --main-model --cluster-model --fallback-model ") click.echo("\nFor more help: codewiki config set --help") sys.exit(EXIT_CONFIG_ERROR) - + config = manager.get_config() api_key = manager.get_api_key() - + if output_json: # JSON output output = { @@ -358,8 +329,10 @@ def config_show(output_json: bool): "max_token_per_module": config.max_token_per_module if config else 36369, "max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000, "max_depth": config.max_depth if config else 2, - "agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {}, - "config_file": str(manager.config_file_path) + "agent_instructions": config.agent_instructions.to_dict() + if config and config.agent_instructions + else {}, + "config_file": str(manager.config_file_path), } click.echo(json.dumps(output, indent=2)) else: @@ -368,8 +341,9 @@ def config_show(output_json: bool): click.secho("CodeWiki Configuration", fg="blue", bold=True) click.echo("━" * 40) click.echo() - + from codewiki.src.be.backend import is_caw_provider + caw_mode = bool(config) and is_caw_provider(config.provider) click.secho("Credentials", fg="cyan", bold=True) @@ -401,24 +375,24 @@ def config_show(output_json: bool): click.echo(f" Azure Deployment: {config.azure_deployment or 'Not set'}") else: click.secho(" Not configured", fg="yellow") - + click.echo() click.secho("Output Settings", fg="cyan", bold=True) if config: click.echo(f" Default Output: {config.default_output}") - + click.echo() click.secho("Token Settings", fg="cyan", bold=True) if config: click.echo(f" Max Tokens: {config.max_tokens}") click.echo(f" Max Token/Module: {config.max_token_per_module}") click.echo(f" Max Token/Leaf Module: {config.max_token_per_leaf_module}") - + click.echo() click.secho("Decomposition Settings", fg="cyan", bold=True) if config: click.echo(f" Max Depth: {config.max_depth}") - + click.echo() click.secho("Agent Instructions", fg="cyan", bold=True) if config and config.agent_instructions and not config.agent_instructions.is_empty(): @@ -435,47 +409,38 @@ def config_show(output_json: bool): click.echo(f" Custom instructions: {agent.custom_instructions[:50]}...") else: click.secho(" Using defaults (no custom settings)", fg="yellow") - + click.echo() click.echo(f"Configuration file: {manager.config_file_path}") click.echo() - + except Exception as e: sys.exit(handle_error(e)) @config_group.command(name="validate") -@click.option( - "--quick", - is_flag=True, - help="Skip API connectivity test" -) -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Show detailed validation steps" -) +@click.option("--quick", is_flag=True, help="Skip API connectivity test") +@click.option("--verbose", "-v", is_flag=True, help="Show detailed validation steps") def config_validate(quick: bool, verbose: bool): """ Validate configuration and test LLM API connectivity. - + Checks: • Configuration file exists and is valid • API key is present • API settings are correctly formatted • (Optional) API connectivity test - + Examples: - + \b # Full validation with API test $ codewiki config validate - + \b # Quick validation (config only) $ codewiki config validate --quick - + \b # Verbose output $ codewiki config validate --verbose @@ -484,29 +449,32 @@ def config_validate(quick: bool, verbose: bool): click.echo() click.secho("Validating configuration...", fg="blue", bold=True) click.echo() - + manager = ConfigManager() - + # Step 1: Check config file if verbose: click.echo("[1/5] Checking configuration file...") click.echo(f" Path: {manager.config_file_path}") - + if not manager.load(): click.secho("✗ Configuration file not found", fg="red") click.echo() - click.echo("Error: Configuration is incomplete. Run 'codewiki config set --help' for setup instructions.") + click.echo( + "Error: Configuration is incomplete. Run 'codewiki config set --help' for setup instructions." + ) sys.exit(EXIT_CONFIG_ERROR) - + if verbose: click.secho(" ✓ File exists", fg="green") click.secho(" ✓ Valid JSON format", fg="green") else: click.secho("✓ Configuration file exists", fg="green") - + # Load config early so we know the provider for the rest of the checks. config = manager.get_config() from codewiki.src.be.backend import is_caw_provider + caw_mode = bool(config) and is_caw_provider(config.provider) # Step 2: Check API key (skipped for subscription providers) @@ -532,7 +500,7 @@ def config_validate(quick: bool, verbose: bool): sys.exit(EXIT_CONFIG_ERROR) if verbose: - click.secho(f" ✓ API key retrieved", fg="green") + click.secho(" ✓ API key retrieved", fg="green") click.secho(f" ✓ Length: {len(api_key)} characters", fg="green") else: click.secho("✓ API key present (stored in keychain)", fg="green") @@ -564,7 +532,7 @@ def config_validate(quick: bool, verbose: bool): except ConfigurationError as e: click.secho(f"✗ Invalid base URL: {e.message}", fg="red") sys.exit(EXIT_CONFIG_ERROR) - + # Step 4: Check models if verbose: click.echo() @@ -598,7 +566,7 @@ def config_validate(quick: bool, verbose: bool): if not is_top_tier_model(config.cluster_model): click.secho( "⚠️ Cluster model is not top-tier. Consider using claude-sonnet-4 or gpt-4.", - fg="yellow" + fg="yellow", ) # Step 5: API connectivity test (unless --quick) @@ -608,6 +576,7 @@ def config_validate(quick: bool, verbose: bool): click.echo("[5/5] Checking CLI availability...") import shutil + cli_name = "claude" if config.provider == "claude-code" else "codex" cli_path = shutil.which(cli_name) if not cli_path: @@ -625,7 +594,10 @@ def config_validate(quick: bool, verbose: bool): fg="cyan", ) else: - click.secho(f"✓ {cli_name} CLI available (run '{cli_name} login' if not yet authenticated)", fg="green") + click.secho( + f"✓ {cli_name} CLI available (run '{cli_name} login' if not yet authenticated)", + fg="green", + ) elif not quick: if verbose: click.echo() @@ -634,10 +606,11 @@ def config_validate(quick: bool, verbose: bool): try: base_url_lower = (config.base_url or "").lower() - provider = getattr(config, 'provider', 'openai-compatible') + provider = getattr(config, "provider", "openai-compatible") if provider == "azure-openai" or ".openai.azure.com" in base_url_lower: # Use Azure OpenAI SDK from openai import AzureOpenAI + client = AzureOpenAI( api_key=api_key, api_version=config.api_version, @@ -647,11 +620,13 @@ def config_validate(quick: bool, verbose: bool): elif "api.anthropic.com" in base_url_lower: # Use Anthropic SDK for native Anthropic endpoints import anthropic + client = anthropic.Anthropic(api_key=api_key) client.models.list(limit=1) else: # Use OpenAI SDK for OpenAI-compatible endpoints from openai import OpenAI + client = OpenAI(api_key=api_key, base_url=config.base_url) client.models.list() @@ -664,12 +639,12 @@ def config_validate(quick: bool, verbose: bool): if verbose: click.echo(f" Error: {e}") sys.exit(EXIT_CONFIG_ERROR) - + # Success click.echo() click.secho("✓ Configuration is valid!", fg="green", bold=True) click.echo() - + except ConfigurationError as e: click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) sys.exit(e.exit_code) @@ -702,7 +677,10 @@ def config_validate(quick: bool, verbose: bool): @click.option( "--doc-type", "-t", - type=click.Choice(['api', 'architecture', 'user-guide', 'developer', 'business', 'design'], case_sensitive=False), + type=click.Choice( + ["api", "architecture", "user-guide", "developer", "business", "design"], + case_sensitive=False, + ), default=None, help="Default type of documentation to generate", ) @@ -723,50 +701,52 @@ def config_agent( focus: Optional[str], doc_type: Optional[str], instructions: Optional[str], - clear: bool + clear: bool, ): """ Configure default agent instructions for documentation generation. - + These settings are used as defaults when running 'codewiki generate'. Runtime options (--include, --exclude, etc.) override these defaults. - + Examples: - + \b # Set include patterns for C# projects $ codewiki config agent --include "*.cs" - + \b # Exclude test projects $ codewiki config agent --exclude "*Tests*,*Specs*,test_*" - + \b # Focus on specific modules $ codewiki config agent --focus "src/core,src/api" - + \b # Set default doc type $ codewiki config agent --doc-type design - + \b # Add custom instructions $ codewiki config agent --instructions "Focus on public APIs and include usage examples" - + \b # Clear all agent instructions $ codewiki config agent --clear """ try: manager = ConfigManager() - + if not manager.load(): click.secho("\n✗ Configuration not found.", fg="red", err=True) - click.echo("\nPlease run 'codewiki config set' first to configure your API credentials.") + click.echo( + "\nPlease run 'codewiki config set' first to configure your API credentials." + ) sys.exit(EXIT_CONFIG_ERROR) - + config = manager.get_config() - + if clear: # Clear all agent instructions config.agent_instructions = AgentInstructions() @@ -775,7 +755,7 @@ def config_agent( click.secho("✓ Agent instructions cleared", fg="green") click.echo() return - + # Check if at least one option is provided if not any([include, exclude, focus, doc_type, instructions]): # Display current settings @@ -783,7 +763,7 @@ def config_agent( click.secho("Agent Instructions", fg="blue", bold=True) click.echo("━" * 40) click.echo() - + agent = config.agent_instructions if agent and not agent.is_empty(): if agent.include_patterns: @@ -798,15 +778,15 @@ def config_agent( click.echo(f" Custom instructions: {agent.custom_instructions}") else: click.secho(" No agent instructions configured (using defaults)", fg="yellow") - + click.echo() click.echo("Use 'codewiki config agent --help' for usage information.") click.echo() return - + # Update agent instructions current = config.agent_instructions or AgentInstructions() - + if include is not None: current.include_patterns = parse_patterns(include) if include else None if exclude is not None: @@ -817,10 +797,10 @@ def config_agent( current.doc_type = doc_type if doc_type else None if instructions is not None: current.custom_instructions = instructions if instructions else None - + config.agent_instructions = current manager.save() - + # Display success messages click.echo() if include: @@ -832,14 +812,15 @@ def config_agent( if doc_type: click.secho(f"✓ Doc type: {doc_type}", fg="green") if instructions: - click.secho(f"✓ Custom instructions set", fg="green") - - click.echo("\n" + click.style("Agent instructions updated successfully.", fg="green", bold=True)) + click.secho("✓ Custom instructions set", fg="green") + + click.echo( + "\n" + click.style("Agent instructions updated successfully.", fg="green", bold=True) + ) click.echo() - + except ConfigurationError as e: click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) sys.exit(e.exit_code) except Exception as e: sys.exit(handle_error(e)) - diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index cbe0316..f6d5ccd 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -6,7 +6,7 @@ import logging import traceback from pathlib import Path -from typing import Optional, List, Tuple +from typing import Optional, List import click import time @@ -36,14 +36,11 @@ def parse_patterns(patterns_str: str) -> List[str]: """Parse comma-separated patterns into a list.""" if not patterns_str: return [] - return [p.strip() for p in patterns_str.split(',') if p.strip()] + return [p.strip() for p in patterns_str.split(",") if p.strip()] def _detect_changed_files( - repo_path: Path, - output_dir: Path, - logger, - verbose: bool + repo_path: Path, output_dir: Path, logger, verbose: bool ) -> Optional[List[str]]: """ Detect files changed since the last documentation generation. @@ -77,6 +74,7 @@ def _detect_changed_files( # Get current HEAD commit try: import git + repo = git.Repo(repo_path, search_parent_directories=True) current_commit = repo.head.commit.hexsha except Exception: @@ -123,7 +121,7 @@ def _detect_changed_files( prefix = subpath_prefix + "/" for path in changed: if path.startswith(prefix): - filtered.append(path[len(prefix):]) + filtered.append(path[len(prefix) :]) if verbose: logger.debug(f"Changes between {prev_commit[:8]} and {current_commit[:8]}:") @@ -141,12 +139,7 @@ def _detect_changed_files( return None -def _invalidate_affected_modules( - output_dir: Path, - changed_files: List[str], - logger, - verbose: bool -): +def _invalidate_affected_modules(output_dir: Path, changed_files: List[str], logger, verbose: bool): """ Remove cached module documentation for modules that contain changed files. @@ -156,6 +149,7 @@ def _invalidate_affected_modules( import json from codewiki.src.config import meta_resolve + module_tree_path = Path(meta_resolve(output_dir, "module_tree.json")) if not module_tree_path.exists(): return @@ -176,7 +170,9 @@ def _find_affected(tree, parent_names=None): # Check if any component path overlaps with changed files for comp in components: # Component IDs may be class names, check if they match any changed file path - if any(changed_file in comp or comp in changed_file for changed_file in changed_set): + if any( + changed_file in comp or comp in changed_file for changed_file in changed_set + ): modules_to_invalidate.add(mod_name) # Also invalidate parent modules for parent in parent_names: @@ -252,7 +248,10 @@ def _find_affected(tree, parent_names=None): @click.option( "--doc-type", "-t", - type=click.Choice(['api', 'architecture', 'user-guide', 'developer', 'business', 'design'], case_sensitive=False), + type=click.Choice( + ["api", "architecture", "user-guide", "developer", "business", "design"], + case_sensitive=False, + ), default=None, help="Type of documentation to generate", ) @@ -314,62 +313,62 @@ def generate_command( max_token_per_module: Optional[int], max_token_per_leaf_module: Optional[int], max_depth: Optional[int], - update: bool = False + update: bool = False, ): """ Generate comprehensive documentation for a code repository. - + Analyzes the current repository and generates documentation using LLM-powered analysis. Documentation is output to ./docs/ by default. - + Examples: - + \b # Basic generation $ codewiki generate - + \b # With git branch creation and GitHub Pages $ codewiki generate --create-branch --github-pages - + \b # Force full regeneration $ codewiki generate --no-cache - + \b # C# project: only .cs files, exclude tests $ codewiki generate --include "*.cs" --exclude "*Tests*,*Specs*" - + \b # Focus on specific modules with architecture docs $ codewiki generate --focus "src/core,src/api" --doc-type architecture - + \b # Custom instructions $ codewiki generate --instructions "Focus on public APIs and include usage examples" - + \b # Override max tokens for this generation $ codewiki generate --max-tokens 16384 - + \b # Set all max token limits $ codewiki generate --max-tokens 32768 --max-token-per-module 40000 --max-token-per-leaf-module 20000 - + \b # Override max depth for hierarchical decomposition $ codewiki generate --max-depth 3 """ logger = create_logger(verbose=verbose) start_time = time.time() - + # Suppress httpx INFO logs logging.getLogger("httpx").setLevel(logging.WARNING) - + try: # Pre-generation checks logger.step("Validating configuration...", 1, 4) - + # Load configuration config_manager = ConfigManager() if not config_manager.load(): @@ -380,27 +379,29 @@ def generate_command( " --main-model --cluster-model \n\n" "For more help: codewiki config --help" ) - + if not config_manager.is_configured(): raise ConfigurationError( "Configuration is incomplete. Please run 'codewiki config validate'" ) - + config = config_manager.get_config() api_key = config_manager.get_api_key() - + logger.success("Configuration valid") - + # Validate repository logger.step("Validating repository...", 2, 4) - + repo_path = Path.cwd() repo_path, languages = validate_repository(repo_path) - + logger.success(f"Repository valid: {repo_path.name}") if verbose: - logger.debug(f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}") - + logger.debug( + f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}" + ) + # Check git repository if not is_git_repository(repo_path): if create_branch: @@ -411,43 +412,46 @@ def generate_command( ) else: logger.warning("Not a git repository. Git features unavailable.") - + # Validate output directory output_dir = Path(output).expanduser().resolve() check_writable_output(output_dir.parent) - + logger.success(f"Output directory: {output_dir}") - + # Incremental update: detect changed files and selectively regenerate changed_files = None if update and output_dir.exists(): changed_files = _detect_changed_files(repo_path, output_dir, logger, verbose) if changed_files is not None and len(changed_files) == 0: - logger.success("No changes detected since last generation. Documentation is up to date.") + logger.success( + "No changes detected since last generation. Documentation is up to date." + ) sys.exit(EXIT_SUCCESS) if changed_files is not None: - logger.info(f" Detected {len(changed_files)} changed files — regenerating affected modules.") + logger.info( + f" Detected {len(changed_files)} changed files — regenerating affected modules." + ) # Remove cached module docs for affected files so they get regenerated _invalidate_affected_modules(output_dir, changed_files, logger, verbose) # Check for existing documentation if not update and output_dir.exists() and list(output_dir.glob("*.md")): if not click.confirm( - f"\n{output_dir} already contains documentation. Overwrite?", - default=True + f"\n{output_dir} already contains documentation. Overwrite?", default=True ): logger.info("Generation cancelled by user.") sys.exit(EXIT_SUCCESS) - + # Git branch creation (if requested) branch_name = None if create_branch: logger.step("Creating git branch...", 3, 4) - + from codewiki.cli.git_manager import GitManager - + git_manager = GitManager(repo_path) - + # Check clean working directory is_clean, status_msg = git_manager.check_clean_working_directory() if not is_clean: @@ -456,27 +460,27 @@ def generate_command( f"{status_msg}\n\n" "Cannot create documentation branch with uncommitted changes.\n" "Please commit or stash your changes first:\n" - " git add -A && git commit -m \"Your message\"\n" + ' git add -A && git commit -m "Your message"\n' " # or\n" " git stash" ) - + # Create branch branch_name = git_manager.create_documentation_branch() logger.success(f"Created branch: {branch_name}") - + # Generate documentation logger.step("Generating documentation...", 4, 4) click.echo() - + # Create generation options - generation_options = GenerationOptions( + GenerationOptions( create_branch=create_branch, github_pages=github_pages, no_cache=no_cache, - custom_output=output if output != "docs" else None + custom_output=output if output != "docs" else None, ) - + # Create runtime agent instructions from CLI options runtime_instructions = None if any([include, exclude, focus, doc_type, instructions]): @@ -487,7 +491,7 @@ def generate_command( doc_type=doc_type, custom_instructions=instructions, ) - + if verbose: if include: logger.debug(f"Include patterns: {parse_patterns(include)}") @@ -499,33 +503,58 @@ def generate_command( logger.debug(f"Doc type: {doc_type}") if instructions: logger.debug(f"Custom instructions: {instructions}") - + # Log max token settings if verbose if verbose: effective_max_tokens = max_tokens if max_tokens is not None else config.max_tokens - effective_max_token_per_module = max_token_per_module if max_token_per_module is not None else config.max_token_per_module - effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module + effective_max_token_per_module = ( + max_token_per_module + if max_token_per_module is not None + else config.max_token_per_module + ) + effective_max_token_per_leaf = ( + max_token_per_leaf_module + if max_token_per_leaf_module is not None + else config.max_token_per_leaf_module + ) effective_max_depth = max_depth if max_depth is not None else config.max_depth logger.debug(f"Max tokens: {effective_max_tokens}") logger.debug(f"Max token/module: {effective_max_token_per_module}") logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}") logger.debug(f"Max depth: {effective_max_depth}") - + # Get agent instructions (merge runtime with persistent) agent_instructions_dict = None if runtime_instructions and not runtime_instructions.is_empty(): # Merge with persistent settings merged = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or (config.agent_instructions.include_patterns if config.agent_instructions else None), - exclude_patterns=runtime_instructions.exclude_patterns or (config.agent_instructions.exclude_patterns if config.agent_instructions else None), - focus_modules=runtime_instructions.focus_modules or (config.agent_instructions.focus_modules if config.agent_instructions else None), - doc_type=runtime_instructions.doc_type or (config.agent_instructions.doc_type if config.agent_instructions else None), - custom_instructions=runtime_instructions.custom_instructions or (config.agent_instructions.custom_instructions if config.agent_instructions else None), + include_patterns=runtime_instructions.include_patterns + or ( + config.agent_instructions.include_patterns + if config.agent_instructions + else None + ), + exclude_patterns=runtime_instructions.exclude_patterns + or ( + config.agent_instructions.exclude_patterns + if config.agent_instructions + else None + ), + focus_modules=runtime_instructions.focus_modules + or (config.agent_instructions.focus_modules if config.agent_instructions else None), + doc_type=runtime_instructions.doc_type + or (config.agent_instructions.doc_type if config.agent_instructions else None), + custom_instructions=runtime_instructions.custom_instructions + or ( + config.agent_instructions.custom_instructions + if config.agent_instructions + else None + ), ) agent_instructions_dict = merged.to_dict() elif config.agent_instructions and not config.agent_instructions.is_empty(): agent_instructions_dict = config.agent_instructions.to_dict() - + # Create generator # Get commit_id early so it can be stored in metadata.json for --update support commit_id = get_git_commit_hash(repo_path) @@ -533,45 +562,50 @@ def generate_command( repo_path=repo_path, output_dir=output_dir, config={ - 'main_model': config.main_model, - 'cluster_model': config.cluster_model, - 'fallback_model': config.fallback_model, - 'base_url': config.base_url, - 'api_key': api_key, - 'provider': getattr(config, 'provider', 'openai-compatible'), - 'aws_region': getattr(config, 'aws_region', 'us-east-1'), - 'agent_instructions': agent_instructions_dict, + "main_model": config.main_model, + "cluster_model": config.cluster_model, + "fallback_model": config.fallback_model, + "base_url": config.base_url, + "api_key": api_key, + "provider": getattr(config, "provider", "openai-compatible"), + "aws_region": getattr(config, "aws_region", "us-east-1"), + "agent_instructions": agent_instructions_dict, # Max token settings (runtime overrides take precedence) - 'max_tokens': max_tokens if max_tokens is not None else config.max_tokens, - 'max_token_per_module': max_token_per_module if max_token_per_module is not None else config.max_token_per_module, - 'max_token_per_leaf_module': max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module, + "max_tokens": max_tokens if max_tokens is not None else config.max_tokens, + "max_token_per_module": max_token_per_module + if max_token_per_module is not None + else config.max_token_per_module, + "max_token_per_leaf_module": max_token_per_leaf_module + if max_token_per_leaf_module is not None + else config.max_token_per_leaf_module, # Max depth setting (runtime override takes precedence) - 'max_depth': max_depth if max_depth is not None else config.max_depth, + "max_depth": max_depth if max_depth is not None else config.max_depth, }, verbose=verbose, generate_html=github_pages, commit_id=commit_id, ) - + # Run generation job = generator.generate() - + # Post-generation generation_time = time.time() - start_time - + # Get repository info repo_url = None - current_branch = get_git_branch(repo_path) - + get_git_branch(repo_path) + if is_git_repository(repo_path): try: import git + repo = git.Repo(repo_path) if repo.remotes: repo_url = repo.remotes.origin.url - except: + except Exception: pass - + # Display instructions display_post_generation_instructions( output_dir=output_dir, @@ -581,13 +615,13 @@ def generate_command( github_pages=github_pages, files_generated=job.files_generated, statistics={ - 'module_count': job.module_count, - 'total_files_analyzed': job.statistics.total_files_analyzed, - 'generation_time': generation_time, - 'total_tokens_used': job.statistics.total_tokens_used, - } + "module_count": job.module_count, + "total_files_analyzed": job.statistics.total_files_analyzed, + "generation_time": generation_time, + "total_tokens_used": job.statistics.total_tokens_used, + }, ) - + except ConfigurationError as e: logger.error(e.message) logger.error(f"Traceback: {traceback.format_exc()}") @@ -605,4 +639,3 @@ def generate_command( sys.exit(130) except Exception as e: sys.exit(handle_error(e, verbose=verbose)) - diff --git a/codewiki/cli/commands/install_hooks.py b/codewiki/cli/commands/install_hooks.py index 25b8bfe..c60e6c5 100644 --- a/codewiki/cli/commands/install_hooks.py +++ b/codewiki/cli/commands/install_hooks.py @@ -125,9 +125,7 @@ def install_hooks(ide: str, create_dir: bool, repo_path: str) -> None: targets = [target] else: if create_dir: - raise IdeWiringError( - "--create-dir only makes sense together with --ide ." - ) + raise IdeWiringError("--create-dir only makes sense together with --ide .") targets = detect_ide_dirs(repo_path) if not targets: click.secho( diff --git a/codewiki/cli/commands/query.py b/codewiki/cli/commands/query.py index 02b8dfd..645b714 100644 --- a/codewiki/cli/commands/query.py +++ b/codewiki/cli/commands/query.py @@ -31,7 +31,7 @@ def _render_result_block(payload: dict) -> str: """JSON payload → delimited text block an agent can read inline.""" lines = [] lines.append("--- codewiki:query:start ---") - q = payload.get('query') + q = payload.get("query") if q: lines.append(f"query: {q}") lines.append(f"search_method: {payload.get('search_method', '')}") @@ -53,7 +53,9 @@ def _render_result_block(payload: dict) -> str: lines.append(f"relevant: {str(payload.get('relevant', False)).lower()}") lines.append(f"top_score: {payload.get('top_score', 0)}") for r in payload.get("top_results") or []: - lines.append(f" - [{r.get('relevance_score', 0)}] {r.get('file', '')} | {r.get('title', '')}") + lines.append( + f" - [{r.get('relevance_score', 0)}] {r.get('file', '')} | {r.get('title', '')}" + ) hint = payload.get("hint") if hint: lines.append(f"hint: {hint}") @@ -91,20 +93,34 @@ def _render_result_block(payload: dict) -> str: @click.command(name="query") @click.argument("query") -@click.option("--output-dir", "-o", default=None, - help="repowiki directory (default: /repowiki)") -@click.option("--top", type=int, default=10, show_default=True, - help="Max results (1-20)") -@click.option("--check", "check_mode", is_flag=True, - help="Lightweight relevance pre-check: verdict + top titles only, " - "no stats recorded. Use before deciding whether a full search " - "is worth the tokens.") -@click.option("--scope", default=None, - help="Limit search to a module name or directory prefix (e.g. 'notes')") -@click.option("--type-filter", "type_filter", default=None, - help="Filter by page type (module/entity/concept/note/source/...)") -@click.option("--expand", is_flag=False, flag_value="3000", default=None, - help="Include full page content (optional value: char budget 500-20000)") +@click.option( + "--output-dir", "-o", default=None, help="repowiki directory (default: /repowiki)" +) +@click.option("--top", type=int, default=10, show_default=True, help="Max results (1-20)") +@click.option( + "--check", + "check_mode", + is_flag=True, + help="Lightweight relevance pre-check: verdict + top titles only, " + "no stats recorded. Use before deciding whether a full search " + "is worth the tokens.", +) +@click.option( + "--scope", default=None, help="Limit search to a module name or directory prefix (e.g. 'notes')" +) +@click.option( + "--type-filter", + "type_filter", + default=None, + help="Filter by page type (module/entity/concept/note/source/...)", +) +@click.option( + "--expand", + is_flag=False, + flag_value="3000", + default=None, + help="Include full page content (optional value: char budget 500-20000)", +) def query_command(query, output_dir, top, check_mode, scope, type_filter, expand): """Search the wiki from the command line (agent-friendly delimited output). @@ -113,8 +129,7 @@ def query_command(query, output_dir, top, check_mode, scope, type_filter, expand """ from pathlib import Path - od = Path(output_dir).expanduser().resolve() if output_dir \ - else Path.cwd() / "repowiki" + od = Path(output_dir).expanduser().resolve() if output_dir else Path.cwd() / "repowiki" if not od.is_dir(): click.echo( f"error: output dir not found: {od}\n" @@ -123,8 +138,7 @@ def query_command(query, output_dir, top, check_mode, scope, type_filter, expand ) sys.exit(2) - arguments = {"output_dir": str(od), "query": query, - "max_results": max(1, min(20, top))} + arguments = {"output_dir": str(od), "query": query, "max_results": max(1, min(20, top))} if check_mode: arguments["mode"] = "check" if scope: @@ -142,6 +156,7 @@ def query_command(query, output_dir, top, check_mode, scope, type_filter, expand try: from codewiki.mcp.session import SessionStore from codewiki.mcp.tools.knowledge_loop import handle_query_wiki + raw = handle_query_wiki(arguments, SessionStore()) except Exception as e: # never crash the agent's shell pipeline click.echo(f"error: query failed: {e}", err=True) diff --git a/codewiki/cli/config_manager.py b/codewiki/cli/config_manager.py index c858a4e..40239ea 100644 --- a/codewiki/cli/config_manager.py +++ b/codewiki/cli/config_manager.py @@ -48,7 +48,11 @@ def __init__(self): """Initialize the configuration manager.""" self._api_key: Optional[str] = None self._config: Optional[Configuration] = None - self._force_no_keyring = os.environ.get("CODEWIKI_NO_KEYRING", "").strip() in ("1", "true", "yes") + self._force_no_keyring = os.environ.get("CODEWIKI_NO_KEYRING", "").strip() in ( + "1", + "true", + "yes", + ) self._keyring_available = self._check_keyring_available() def _check_keyring_available(self) -> bool: @@ -84,29 +88,29 @@ def _save_api_key_to_file(self, api_key: str): CREDENTIALS_FILE.chmod(0o600) except OSError: pass - + def load(self) -> bool: """ Load configuration from file and keyring. - + Returns: True if configuration exists, False otherwise """ # Load from JSON file if not CONFIG_FILE.exists(): return False - + try: content = safe_read(CONFIG_FILE) data = json.loads(content) - + # Validate version - if data.get('version') != CONFIG_VERSION: + if data.get("version") != CONFIG_VERSION: # Could implement migration here pass - + self._config = Configuration.from_dict(data) - + # Load API key from keyring, falling back to file if self._keyring_available: try: @@ -115,11 +119,11 @@ def load(self) -> bool: pass if self._api_key is None: self._api_key = self._load_api_key_from_file() - + return True except (json.JSONDecodeError, FileSystemError) as e: raise ConfigurationError(f"Failed to load configuration: {e}") - + def save( self, api_key: Optional[str] = None, @@ -135,7 +139,7 @@ def save( provider: Optional[str] = None, aws_region: Optional[str] = None, api_version: Optional[str] = None, - azure_deployment: Optional[str] = None + azure_deployment: Optional[str] = None, ): """ Save configuration to file and keyring. @@ -161,22 +165,23 @@ def save( ensure_directory(CONFIG_DIR) except FileSystemError as e: raise ConfigurationError(f"Cannot create config directory: {e}") - + # Load existing config or create new if self._config is None: if CONFIG_FILE.exists(): self.load() else: from codewiki.cli.models.config import AgentInstructions + self._config = Configuration( base_url="", main_model="", cluster_model="", fallback_model="glm-4p5", default_output="docs", - agent_instructions=AgentInstructions() + agent_instructions=AgentInstructions(), ) - + # Update fields if provided if base_url is not None: self._config.base_url = base_url @@ -210,12 +215,13 @@ def save( # cluster_model on top of that. The validate() method itself routes # by provider, so we only gate on whether enough is set to validate. from codewiki.src.be.backend import is_caw_provider + if is_caw_provider(self._config.provider): if self._config.main_model: self._config.validate() elif self._config.base_url and self._config.main_model and self._config.cluster_model: self._config.validate() - + # Save API key to keyring, falling back to file if api_key is not None: self._api_key = api_key @@ -229,22 +235,19 @@ def save( logger.warning( "System keychain unavailable. API key stored in %s " "(plaintext). Set CODEWIKI_NO_KEYRING=1 to suppress this warning.", - CREDENTIALS_FILE + CREDENTIALS_FILE, ) else: self._save_api_key_to_file(api_key) - + # Save non-sensitive config to JSON - config_data = { - "version": CONFIG_VERSION, - **self._config.to_dict() - } - + config_data = {"version": CONFIG_VERSION, **self._config.to_dict()} + try: safe_write(CONFIG_FILE, json.dumps(config_data, indent=2)) except FileSystemError as e: raise ConfigurationError(f"Failed to save configuration: {e}") - + def get_api_key(self) -> Optional[str]: """ Get API key from keyring or fallback file. @@ -262,16 +265,16 @@ def get_api_key(self) -> Optional[str]: self._api_key = self._load_api_key_from_file() return self._api_key - + def get_config(self) -> Optional[Configuration]: """ Get current configuration. - + Returns: Configuration object or None if not loaded """ return self._config - + def is_configured(self) -> bool: """ Check if configuration is complete and valid. @@ -286,6 +289,7 @@ def is_configured(self) -> bool: return False from codewiki.src.be.backend import is_caw_provider + if not is_caw_provider(self._config.provider): # Check if API key is set if self.get_api_key() is None: @@ -293,7 +297,7 @@ def is_configured(self) -> bool: # Check if config is complete return self._config.is_complete() - + def delete_api_key(self): """Delete API key from keyring and fallback file.""" if self._keyring_available: @@ -308,26 +312,25 @@ def delete_api_key(self): except OSError: pass self._api_key = None - + def clear(self): """Clear all configuration (file and keyring).""" # Delete API key from keyring self.delete_api_key() - + # Delete config file if CONFIG_FILE.exists(): CONFIG_FILE.unlink() - + self._config = None self._api_key = None - + @property def keyring_available(self) -> bool: """Check if keyring is available.""" return self._keyring_available - + @property def config_file_path(self) -> Path: """Get configuration file path.""" return CONFIG_FILE - diff --git a/codewiki/cli/git_manager.py b/codewiki/cli/git_manager.py index e75a9fa..8109ce3 100644 --- a/codewiki/cli/git_manager.py +++ b/codewiki/cli/git_manager.py @@ -14,72 +14,71 @@ class GitManager: """ Manages git operations for documentation generation. - + Handles: - Status checking - Branch creation - Committing documentation - Remote detection """ - + def __init__(self, repo_path: Path): """ Initialize git manager. - + Args: repo_path: Path to git repository - + Raises: RepositoryError: If not a valid git repository """ self.repo_path = Path(repo_path).expanduser().resolve() - + try: self.repo = git.Repo(repo_path, search_parent_directories=True) except git.InvalidGitRepositoryError: raise RepositoryError( - f"Not a git repository: {repo_path}\n\n" - "To initialize a git repository: git init" + f"Not a git repository: {repo_path}\n\nTo initialize a git repository: git init" ) - + def check_clean_working_directory(self) -> Tuple[bool, str]: """ Check if working directory is clean (no uncommitted changes). - + Returns: Tuple of (is_clean, status_message) """ if self.repo.is_dirty(untracked_files=True): status_lines = [] - + # Changed files changed = [item.a_path for item in self.repo.index.diff(None)] if changed: status_lines.append(f"Modified: {', '.join(changed[:3])}") if len(changed) > 3: status_lines.append(f"... and {len(changed) - 3} more") - + # Untracked files untracked = self.repo.untracked_files if untracked: status_lines.append(f"Untracked: {', '.join(untracked[:3])}") if len(untracked) > 3: status_lines.append(f"... and {len(untracked) - 3} more") - + return False, "\n".join(status_lines) - + return True, "Working directory is clean" - + def create_documentation_branch(self, force: bool = False) -> str: """ Create a new documentation branch with timestamp. - + Args: force: Force creation even if dirty working directory - + Returns: Branch name - + Raises: RepositoryError: If working directory is dirty (unless force=True) """ @@ -93,16 +92,16 @@ def create_documentation_branch(self, force: bool = False) -> str: "Cannot create documentation branch with uncommitted changes.\n" "Please commit or stash your changes first:\n" " git status\n" - " git add -A && git commit -m \"Your message\"\n" + ' git add -A && git commit -m "Your message"\n' " # or\n" " git stash\n\n" "Then re-run: codewiki generate --create-branch" ) - + # Generate branch name with timestamp timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") branch_name = f"docs/codewiki-{timestamp}" - + # Check if branch already exists (shouldn't happen with timestamp) existing_branches = [b.name for b in self.repo.branches] if branch_name in existing_branches: @@ -111,7 +110,7 @@ def create_documentation_branch(self, force: bool = False) -> str: while f"{branch_name}-{counter}" in existing_branches: counter += 1 branch_name = f"{branch_name}-{counter}" - + try: # Create and checkout new branch new_branch = self.repo.create_head(branch_name) @@ -119,46 +118,42 @@ def create_documentation_branch(self, force: bool = False) -> str: return branch_name except GitCommandError as e: raise RepositoryError(f"Failed to create branch: {e}") - - def commit_documentation( - self, - docs_path: Path, - message: Optional[str] = None - ) -> str: + + def commit_documentation(self, docs_path: Path, message: Optional[str] = None) -> str: """ Commit generated documentation. - + Args: docs_path: Path to documentation directory message: Commit message (optional) - + Returns: Commit hash - + Raises: RepositoryError: If commit fails """ if message is None: message = "Add generated documentation\n\nGenerated by CodeWiki CLI" - + try: # Add documentation files self.repo.index.add([str(docs_path)]) - + # Commit commit = self.repo.index.commit(message) - + return commit.hexsha except GitCommandError as e: raise RepositoryError(f"Failed to commit documentation: {e}") - + def get_remote_url(self, remote_name: str = "origin") -> Optional[str]: """ Get remote repository URL. - + Args: remote_name: Name of remote (default: origin) - + Returns: Remote URL or None if no remote """ @@ -167,11 +162,11 @@ def get_remote_url(self, remote_name: str = "origin") -> Optional[str]: return remote.url except ValueError: return None - + def get_current_branch(self) -> str: """ Get current branch name. - + Returns: Branch name """ @@ -180,48 +175,47 @@ def get_current_branch(self) -> str: except TypeError: # Detached HEAD return "HEAD" - + def get_commit_hash(self) -> str: """ Get current commit hash. - + Returns: Commit hash """ return self.repo.head.commit.hexsha - + def branch_exists(self, branch_name: str) -> bool: """ Check if a branch exists. - + Args: branch_name: Branch name to check - + Returns: True if exists, False otherwise """ return branch_name in [b.name for b in self.repo.branches] - + def get_github_pr_url(self, branch_name: str) -> Optional[str]: """ Get GitHub PR creation URL for a branch. - + Args: branch_name: Branch name - + Returns: PR URL or None if not a GitHub repo """ remote_url = self.get_remote_url() if not remote_url or "github.com" not in remote_url: return None - + # Clean URL - base_url = remote_url.rstrip('/').replace('.git', '') - + base_url = remote_url.rstrip("/").replace(".git", "") + # Convert SSH to HTTPS - if base_url.startswith('git@github.com:'): - base_url = base_url.replace('git@github.com:', 'https://github.com/') - - return f"{base_url}/compare/{branch_name}" + if base_url.startswith("git@github.com:"): + base_url = base_url.replace("git@github.com:", "https://github.com/") + return f"{base_url}/compare/{branch_name}" diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py index 273c59f..34c4633 100644 --- a/codewiki/cli/html_generator.py +++ b/codewiki/cli/html_generator.py @@ -13,75 +13,72 @@ class HTMLGenerator: """ Generates static HTML documentation viewer for GitHub Pages. - + Creates a self-contained index.html with embedded styles, scripts, and configuration for client-side markdown rendering. """ - + def __init__(self, template_dir: Optional[Path] = None): """ Initialize HTML generator. - + Args: template_dir: Path to template directory (default: package templates) """ if template_dir is None: # Use package templates template_dir = Path(__file__).parent.parent / "templates" / "github_pages" - + self.template_dir = Path(template_dir) - - + def load_module_tree(self, docs_dir: Path) -> Dict[str, Any]: """ Load module tree from documentation directory. - + Args: docs_dir: Documentation directory path - + Returns: Module tree structure """ from codewiki.src.config import meta_resolve + module_tree_path = Path(meta_resolve(docs_dir, "module_tree.json")) if not module_tree_path.exists(): # Fallback to a simple structure return { - "Overview": { - "description": "Repository overview", - "components": [], - "children": {} - } + "Overview": {"description": "Repository overview", "components": [], "children": {}} } - + try: content = safe_read(module_tree_path) return json.loads(content) except Exception as e: raise FileSystemError(f"Failed to load module tree: {e}") - + def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]: """ Load metadata from documentation directory. - + Args: docs_dir: Documentation directory path - + Returns: Metadata dictionary or None if not found """ from codewiki.src.config import meta_resolve + metadata_path = Path(meta_resolve(docs_dir, "metadata.json")) if not metadata_path.exists(): return None - + try: content = safe_read(metadata_path) return json.loads(content) except Exception: # Non-critical, return None return None - + def generate( self, output_path: Path, @@ -91,11 +88,11 @@ def generate( github_pages_url: Optional[str] = None, config: Optional[Dict[str, Any]] = None, docs_dir: Optional[Path] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ): """ Generate HTML documentation viewer. - + Args: output_path: Output file path (index.html) title: Documentation title @@ -112,29 +109,29 @@ def generate( module_tree = self.load_module_tree(docs_dir) if metadata is None: metadata = self.load_metadata(docs_dir) - + # Default values if module_tree is None: module_tree = {} if config is None: config = {} - + # Load template template_path = self.template_dir / "viewer_template.html" if not template_path.exists(): raise FileSystemError(f"Template not found: {template_path}") - + template_content = safe_read(template_path) - + # Build info content HTML info_content = self._build_info_content(metadata) show_info = "block" if info_content else "none" - + # Build repository link repo_link = "" if repository_url: repo_link = f'🔗 View Repository' - + # Determine docs base path # For GitHub Pages: relative path to docs folder # For local: relative path to docs folder @@ -145,12 +142,12 @@ def generate( docs_base_path = Path(docs_dir.name).as_posix() except Exception: docs_base_path = "." - + # Prepare JSON data for embedding config_json = json.dumps(config, indent=2) module_tree_json = json.dumps(module_tree, indent=2) metadata_json = json.dumps(metadata, indent=2) if metadata else "null" - + # Replace placeholders html_content = template_content replacements = { @@ -163,124 +160,134 @@ def generate( "{{METADATA_JSON}}": metadata_json, "{{DOCS_BASE_PATH}}": docs_base_path, } - + for placeholder, value in replacements.items(): html_content = html_content.replace(placeholder, value) - + # Write output output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) safe_write(output_path, html_content) - + def _build_info_content(self, metadata: Optional[Dict[str, Any]]) -> str: """ Build HTML content for repo info section. - + Args: metadata: Metadata dictionary - + Returns: HTML string for info content """ - if not metadata or not metadata.get('generation_info'): + if not metadata or not metadata.get("generation_info"): return "" - - info = metadata.get('generation_info', {}) - stats = metadata.get('statistics', {}) - + + info = metadata.get("generation_info", {}) + stats = metadata.get("statistics", {}) + html_parts = [] - - if info.get('main_model'): - html_parts.append(f'
Model: {self._escape_html(info["main_model"])}
') - - if info.get('timestamp'): + + if info.get("main_model"): + html_parts.append( + f'
Model: {self._escape_html(info["main_model"])}
' + ) + + if info.get("timestamp"): try: from datetime import datetime - timestamp = info['timestamp'] + + timestamp = info["timestamp"] # Parse ISO format timestamp if isinstance(timestamp, str): - dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - formatted_date = dt.strftime('%Y-%m-%d') - html_parts.append(f'
Generated: {formatted_date}
') + dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + formatted_date = dt.strftime("%Y-%m-%d") + html_parts.append( + f'
Generated: {formatted_date}
' + ) except Exception: pass - - if info.get('commit_id'): - commit_short = info['commit_id'][:8] - html_parts.append(f'
Commit: {commit_short}
') - - if stats.get('total_components'): + + if info.get("commit_id"): + commit_short = info["commit_id"][:8] + html_parts.append( + f'
Commit: {commit_short}
' + ) + + if stats.get("total_components"): components_str = f"{stats['total_components']:,}" - html_parts.append(f'
Components: {components_str}
') - - if stats.get('max_depth'): - html_parts.append(f'
Max Depth: {stats["max_depth"]}
') - - return '\n '.join(html_parts) - + html_parts.append( + f'
Components: {components_str}
' + ) + + if stats.get("max_depth"): + html_parts.append( + f'
Max Depth: {stats["max_depth"]}
' + ) + + return "\n ".join(html_parts) + def _escape_html(self, text: str) -> str: """ Escape HTML special characters. - + Args: text: Text to escape - + Returns: Escaped text """ - return (text - .replace('&', '&') - .replace('<', '<') - .replace('>', '>') - .replace('"', '"') - .replace("'", ''')) - - - + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]: """ Detect repository information from git. - + Args: repo_path: Repository path - + Returns: Dictionary with 'name', 'url', 'github_pages_url' """ info = { - 'name': repo_path.name, - 'url': None, - 'github_pages_url': None, + "name": repo_path.name, + "url": None, + "github_pages_url": None, } - + try: import git + repo = git.Repo(repo_path) - + # Get repository name - info['name'] = repo_path.name - + info["name"] = repo_path.name + # Get remote URL if repo.remotes: remote_url = repo.remotes.origin.url - + # Clean URL - if remote_url.startswith('git@github.com:'): - remote_url = remote_url.replace('git@github.com:', 'https://github.com/') - - remote_url = remote_url.rstrip('/').replace('.git', '') - info['url'] = remote_url - + if remote_url.startswith("git@github.com:"): + remote_url = remote_url.replace("git@github.com:", "https://github.com/") + + remote_url = remote_url.rstrip("/").replace(".git", "") + info["url"] = remote_url + # Compute GitHub Pages URL - if 'github.com' in remote_url: - parts = remote_url.split('/') + if "github.com" in remote_url: + parts = remote_url.split("/") if len(parts) >= 2: owner = parts[-2] repo = parts[-1] - info['github_pages_url'] = f"https://{owner}.github.io/{repo}/" - + info["github_pages_url"] = f"https://{owner}.github.io/{repo}/" + except Exception: pass - - return info + return info diff --git a/codewiki/cli/main.py b/codewiki/cli/main.py index 4069ba2..5542fef 100644 --- a/codewiki/cli/main.py +++ b/codewiki/cli/main.py @@ -4,7 +4,6 @@ import sys import click -from pathlib import Path from codewiki import __version__ @@ -15,7 +14,7 @@ def cli(ctx): """ CodeWiki: Transform codebases into comprehensive documentation. - + Generate AI-powered documentation for your code repositories with support for Python, Java, JavaScript, TypeScript, C, C++, and C#. """ @@ -28,7 +27,7 @@ def version(): """Display version information.""" click.echo(f"CodeWiki CLI v{__version__}") click.echo("Python-based documentation generator using AI analysis") - + # Import commands from codewiki.cli.commands.config import config_group @@ -62,6 +61,7 @@ def mcp_command(): """ import asyncio from codewiki.mcp.server import main as mcp_main + asyncio.run(mcp_main()) @@ -79,4 +79,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/codewiki/cli/models/__init__.py b/codewiki/cli/models/__init__.py index 07db84a..31f7fd7 100644 --- a/codewiki/cli/models/__init__.py +++ b/codewiki/cli/models/__init__.py @@ -1,4 +1,3 @@ """Data models for CLI.""" __all__ = [] - diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py index b005319..0aeea0c 100644 --- a/codewiki/cli/models/config.py +++ b/codewiki/cli/models/config.py @@ -6,13 +6,11 @@ to the backend Config class when running documentation generation. """ -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, field from typing import Optional, List -from pathlib import Path from codewiki.cli.utils.validation import ( validate_url, - validate_api_key, validate_model_name, ) @@ -21,13 +19,13 @@ class AgentInstructions: """ Custom instructions for the documentation agent. - + Allows users to customize: - File filtering (include/exclude patterns) - Module focus (prioritize certain modules) - Documentation type (API docs, architecture docs, etc.) - Custom instructions for the LLM - + Attributes: include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) exclude_patterns: File/directory patterns to exclude (e.g., ["*Tests*", "*test*"]) @@ -35,72 +33,79 @@ class AgentInstructions: doc_type: Type of documentation to generate custom_instructions: Additional instructions for the documentation agent """ + include_patterns: Optional[List[str]] = None # e.g., ["*.cs"] for C# projects exclude_patterns: Optional[List[str]] = None # e.g., ["*Tests*", "*Specs*"] focus_modules: Optional[List[str]] = None # e.g., ["src/core", "src/api"] - doc_type: Optional[str] = "design" # e.g., "api", "architecture", "user-guide", "business", "design" + doc_type: Optional[str] = ( + "design" # e.g., "api", "architecture", "user-guide", "business", "design" + ) custom_instructions: Optional[str] = None # Free-form instructions - + def to_dict(self) -> dict: """Convert to dictionary, excluding None values.""" result = {} if self.include_patterns: - result['include_patterns'] = self.include_patterns + result["include_patterns"] = self.include_patterns if self.exclude_patterns: - result['exclude_patterns'] = self.exclude_patterns + result["exclude_patterns"] = self.exclude_patterns if self.focus_modules: - result['focus_modules'] = self.focus_modules + result["focus_modules"] = self.focus_modules if self.doc_type: - result['doc_type'] = self.doc_type + result["doc_type"] = self.doc_type if self.custom_instructions: - result['custom_instructions'] = self.custom_instructions + result["custom_instructions"] = self.custom_instructions return result - + @classmethod - def from_dict(cls, data: dict) -> 'AgentInstructions': + def from_dict(cls, data: dict) -> "AgentInstructions": """Create AgentInstructions from dictionary.""" return cls( - include_patterns=data.get('include_patterns'), - exclude_patterns=data.get('exclude_patterns'), - focus_modules=data.get('focus_modules'), - doc_type=data.get('doc_type'), - custom_instructions=data.get('custom_instructions'), + include_patterns=data.get("include_patterns"), + exclude_patterns=data.get("exclude_patterns"), + focus_modules=data.get("focus_modules"), + doc_type=data.get("doc_type"), + custom_instructions=data.get("custom_instructions"), ) - + def is_empty(self) -> bool: """Check if all fields are empty/None.""" - return not any([ - self.include_patterns, - self.exclude_patterns, - self.focus_modules, - self.doc_type, - self.custom_instructions, - ]) - + return not any( + [ + self.include_patterns, + self.exclude_patterns, + self.focus_modules, + self.doc_type, + self.custom_instructions, + ] + ) + def get_prompt_addition(self) -> str: """Generate prompt additions based on instructions.""" additions = [] - + if self.doc_type: doc_type_instructions = { - 'api': "Focus on API documentation: endpoints, parameters, return types, and usage examples.", - 'architecture': "Focus on architecture documentation: system design, component relationships, and data flow.", - 'user-guide': "Focus on user guide documentation: how to use features, step-by-step tutorials.", - 'developer': "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", - 'business': "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details.", - 'design': "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", + "api": "Focus on API documentation: endpoints, parameters, return types, and usage examples.", + "architecture": "Focus on architecture documentation: system design, component relationships, and data flow.", + "user-guide": "Focus on user guide documentation: how to use features, step-by-step tutorials.", + "developer": "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", + "business": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details.", + "design": "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", } if self.doc_type.lower() in doc_type_instructions: additions.append(doc_type_instructions[self.doc_type.lower()]) else: additions.append(f"Focus on generating {self.doc_type} documentation.") - + if self.focus_modules: - additions.append(f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}") - + additions.append( + f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}" + ) + if self.custom_instructions: additions.append(f"Additional instructions: {self.custom_instructions}") - + return "\n".join(additions) if additions else "" @@ -125,6 +130,7 @@ class Configuration: max_depth: Maximum depth for hierarchical decomposition (default: 2) agent_instructions: Custom agent instructions for documentation generation """ + base_url: str main_model: str cluster_model: str @@ -139,7 +145,7 @@ class Configuration: max_token_per_leaf_module: int = 16000 max_depth: int = 3 agent_instructions: AgentInstructions = field(default_factory=AgentInstructions) - + def validate(self): """ Validate all configuration fields. @@ -151,6 +157,7 @@ def validate(self): ConfigurationError: If validation fails """ from codewiki.src.be.backend import is_caw_provider + if is_caw_provider(self.provider): validate_model_name(self.main_model) return @@ -158,60 +165,60 @@ def validate(self): validate_model_name(self.main_model) validate_model_name(self.cluster_model) validate_model_name(self.fallback_model) - + def to_dict(self) -> dict: """Convert to dictionary.""" result = { - 'base_url': self.base_url, - 'main_model': self.main_model, - 'cluster_model': self.cluster_model, - 'default_output': self.default_output, - 'provider': self.provider, - 'aws_region': self.aws_region, - 'api_version': self.api_version, - 'azure_deployment': self.azure_deployment, - 'max_tokens': self.max_tokens, - 'max_token_per_module': self.max_token_per_module, - 'max_token_per_leaf_module': self.max_token_per_leaf_module, - 'max_depth': self.max_depth, - 'fallback_model': self.fallback_model, + "base_url": self.base_url, + "main_model": self.main_model, + "cluster_model": self.cluster_model, + "default_output": self.default_output, + "provider": self.provider, + "aws_region": self.aws_region, + "api_version": self.api_version, + "azure_deployment": self.azure_deployment, + "max_tokens": self.max_tokens, + "max_token_per_module": self.max_token_per_module, + "max_token_per_leaf_module": self.max_token_per_leaf_module, + "max_depth": self.max_depth, + "fallback_model": self.fallback_model, } if self.agent_instructions and not self.agent_instructions.is_empty(): - result['agent_instructions'] = self.agent_instructions.to_dict() + result["agent_instructions"] = self.agent_instructions.to_dict() return result - + @classmethod - def from_dict(cls, data: dict) -> 'Configuration': + def from_dict(cls, data: dict) -> "Configuration": """ Create Configuration from dictionary. - + Args: data: Configuration dictionary - + Returns: Configuration instance """ agent_instructions = AgentInstructions() - if 'agent_instructions' in data: - agent_instructions = AgentInstructions.from_dict(data['agent_instructions']) - + if "agent_instructions" in data: + agent_instructions = AgentInstructions.from_dict(data["agent_instructions"]) + return cls( - base_url=data.get('base_url', ''), - main_model=data.get('main_model', ''), - cluster_model=data.get('cluster_model', ''), - fallback_model=data.get('fallback_model', 'glm-4p5'), - default_output=data.get('default_output', 'docs'), - provider=data.get('provider', 'openai-compatible'), - aws_region=data.get('aws_region', 'us-east-1'), - api_version=data.get('api_version', '2024-12-01-preview'), - azure_deployment=data.get('azure_deployment', ''), - max_tokens=data.get('max_tokens', 32768), - max_token_per_module=data.get('max_token_per_module', 36369), - max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000), - max_depth=data.get('max_depth', 3), + base_url=data.get("base_url", ""), + main_model=data.get("main_model", ""), + cluster_model=data.get("cluster_model", ""), + fallback_model=data.get("fallback_model", "glm-4p5"), + default_output=data.get("default_output", "docs"), + provider=data.get("provider", "openai-compatible"), + aws_region=data.get("aws_region", "us-east-1"), + api_version=data.get("api_version", "2024-12-01-preview"), + azure_deployment=data.get("azure_deployment", ""), + max_tokens=data.get("max_tokens", 32768), + max_token_per_module=data.get("max_token_per_module", 36369), + max_token_per_leaf_module=data.get("max_token_per_leaf_module", 16000), + max_depth=data.get("max_depth", 3), agent_instructions=agent_instructions, ) - + def is_complete(self) -> bool: """Check if all required fields are set. @@ -220,45 +227,53 @@ def is_complete(self) -> bool: are unused. """ from codewiki.src.be.backend import is_caw_provider + if is_caw_provider(self.provider): return bool(self.main_model) return bool( - self.base_url and - self.main_model and - self.cluster_model and - self.fallback_model + self.base_url and self.main_model and self.cluster_model and self.fallback_model ) - - def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runtime_instructions: AgentInstructions = None): + + def to_backend_config( + self, + repo_path: str, + output_dir: str, + api_key: str, + runtime_instructions: AgentInstructions = None, + ): """ Convert CLI Configuration to Backend Config. - + This method bridges the gap between persistent user settings (CLI Configuration) and runtime job configuration (Backend Config). - + Args: repo_path: Path to the repository to document output_dir: Output directory for generated documentation api_key: LLM API key (from keyring) runtime_instructions: Runtime agent instructions (override persistent settings) - + Returns: Backend Config instance ready for documentation generation """ from codewiki.src.config import Config - + # Merge runtime instructions with persistent settings # Runtime instructions take precedence final_instructions = self.agent_instructions if runtime_instructions and not runtime_instructions.is_empty(): final_instructions = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or self.agent_instructions.include_patterns, - exclude_patterns=runtime_instructions.exclude_patterns or self.agent_instructions.exclude_patterns, - focus_modules=runtime_instructions.focus_modules or self.agent_instructions.focus_modules, + include_patterns=runtime_instructions.include_patterns + or self.agent_instructions.include_patterns, + exclude_patterns=runtime_instructions.exclude_patterns + or self.agent_instructions.exclude_patterns, + focus_modules=runtime_instructions.focus_modules + or self.agent_instructions.focus_modules, doc_type=runtime_instructions.doc_type or self.agent_instructions.doc_type, - custom_instructions=runtime_instructions.custom_instructions or self.agent_instructions.custom_instructions, + custom_instructions=runtime_instructions.custom_instructions + or self.agent_instructions.custom_instructions, ) - + return Config.from_cli( repo_path=repo_path, output_dir=output_dir, @@ -275,6 +290,5 @@ def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runti max_token_per_module=self.max_token_per_module, max_token_per_leaf_module=self.max_token_per_leaf_module, max_depth=self.max_depth, - agent_instructions=final_instructions.to_dict() if final_instructions else None + agent_instructions=final_instructions.to_dict() if final_instructions else None, ) - diff --git a/codewiki/cli/models/job.py b/codewiki/cli/models/job.py index c0c49d1..19bde6f 100644 --- a/codewiki/cli/models/job.py +++ b/codewiki/cli/models/job.py @@ -12,6 +12,7 @@ class JobStatus(str, Enum): """Documentation job status.""" + PENDING = "pending" RUNNING = "running" COMPLETED = "completed" @@ -21,6 +22,7 @@ class JobStatus(str, Enum): @dataclass class GenerationOptions: """Options for documentation generation.""" + create_branch: bool = False github_pages: bool = False no_cache: bool = False @@ -30,6 +32,7 @@ class GenerationOptions: @dataclass class JobStatistics: """Statistics for a documentation job.""" + total_files_analyzed: int = 0 leaf_nodes: int = 0 max_depth: int = 0 @@ -39,6 +42,7 @@ class JobStatistics: @dataclass class LLMConfig: """LLM configuration for a job.""" + main_model: str cluster_model: str base_url: str @@ -48,7 +52,7 @@ class LLMConfig: class DocumentationJob: """ Represents a documentation generation job. - + Attributes: job_id: Unique job identifier repository_path: Absolute path to repository @@ -66,6 +70,7 @@ class DocumentationJob: llm_config: LLM configuration used statistics: Job statistics """ + job_id: str = field(default_factory=lambda: str(uuid.uuid4())) repository_path: str = "" repository_name: str = "" @@ -81,23 +86,23 @@ class DocumentationJob: generation_options: GenerationOptions = field(default_factory=GenerationOptions) llm_config: Optional[LLMConfig] = None statistics: JobStatistics = field(default_factory=JobStatistics) - + def start(self): """Mark job as started.""" self.status = JobStatus.RUNNING self.timestamp_start = datetime.now().isoformat() - + def complete(self): """Mark job as completed.""" self.status = JobStatus.COMPLETED self.timestamp_end = datetime.now().isoformat() - + def fail(self, error_message: str): """Mark job as failed.""" self.status = JobStatus.FAILED self.error_message = error_message self.timestamp_end = datetime.now().isoformat() - + def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for JSON serialization.""" data = { @@ -118,39 +123,38 @@ def to_dict(self) -> Dict[str, Any]: "statistics": asdict(self.statistics), } return data - + def to_json(self) -> str: """Convert to JSON string.""" return json.dumps(self.to_dict(), indent=2) - + @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'DocumentationJob': + def from_dict(cls, data: Dict[str, Any]) -> "DocumentationJob": """Create from dictionary.""" job = cls( - job_id=data.get('job_id', str(uuid.uuid4())), - repository_path=data.get('repository_path', ''), - repository_name=data.get('repository_name', ''), - output_directory=data.get('output_directory', ''), - commit_hash=data.get('commit_hash', ''), - branch_name=data.get('branch_name'), - timestamp_start=data.get('timestamp_start', datetime.now().isoformat()), - timestamp_end=data.get('timestamp_end'), - status=JobStatus(data.get('status', 'pending')), - error_message=data.get('error_message'), - files_generated=data.get('files_generated', []), - module_count=data.get('module_count', 0), + job_id=data.get("job_id", str(uuid.uuid4())), + repository_path=data.get("repository_path", ""), + repository_name=data.get("repository_name", ""), + output_directory=data.get("output_directory", ""), + commit_hash=data.get("commit_hash", ""), + branch_name=data.get("branch_name"), + timestamp_start=data.get("timestamp_start", datetime.now().isoformat()), + timestamp_end=data.get("timestamp_end"), + status=JobStatus(data.get("status", "pending")), + error_message=data.get("error_message"), + files_generated=data.get("files_generated", []), + module_count=data.get("module_count", 0), ) - + # Parse nested objects - if 'generation_options' in data: - opts = data['generation_options'] + if "generation_options" in data: + opts = data["generation_options"] job.generation_options = GenerationOptions(**opts) - - if 'llm_config' in data and data['llm_config']: - job.llm_config = LLMConfig(**data['llm_config']) - - if 'statistics' in data: - job.statistics = JobStatistics(**data['statistics']) - - return job + if "llm_config" in data and data["llm_config"]: + job.llm_config = LLMConfig(**data["llm_config"]) + + if "statistics" in data: + job.statistics = JobStatistics(**data["statistics"]) + + return job diff --git a/codewiki/cli/utils/__init__.py b/codewiki/cli/utils/__init__.py index a7a69c2..4068157 100644 --- a/codewiki/cli/utils/__init__.py +++ b/codewiki/cli/utils/__init__.py @@ -1,4 +1,3 @@ """Utility functions and helpers for CLI.""" __all__ = [] - diff --git a/codewiki/cli/utils/api_errors.py b/codewiki/cli/utils/api_errors.py index 286db4c..6a8d831 100644 --- a/codewiki/cli/utils/api_errors.py +++ b/codewiki/cli/utils/api_errors.py @@ -10,26 +10,24 @@ class APIErrorHandler: """Handler for LLM API errors with fail-fast behavior.""" - + @staticmethod def handle_api_error( - error: Exception, - context: Optional[str] = None, - fail_fast: bool = True + error: Exception, context: Optional[str] = None, fail_fast: bool = True ) -> APIError: """ Handle LLM API error and convert to APIError. - + Args: error: The original exception context: Additional context (e.g., module name) fail_fast: Whether to fail immediately (default: True) - + Returns: APIError instance """ error_message = str(error) - + # Detect specific error types if "429" in error_message or "rate limit" in error_message.lower(): message = ( @@ -79,17 +77,17 @@ def handle_api_error( " 2. Verify API service status\n" " 3. Review the error message above for specific details" ) - + if context: message = f"Context: {context}\n\n{message}" - + return APIError(message) - + @staticmethod def display_api_error(error: APIError, module_name: Optional[str] = None): """ Display API error with formatting. - + Args: error: The API error module_name: Optional module name for context @@ -97,34 +95,31 @@ def display_api_error(error: APIError, module_name: Optional[str] = None): click.echo() click.secho("✗ LLM API Error", fg="red", bold=True) click.echo() - + if module_name: click.echo(f"Module: {module_name}") click.echo() - + click.echo(error.message) click.echo() - click.secho( - "Documentation generation stopped. No partial results saved.", - fg="yellow" - ) + click.secho("Documentation generation stopped. No partial results saved.", fg="yellow") click.echo() def wrap_api_call(func, *args, fail_fast: bool = True, context: Optional[str] = None, **kwargs): """ Wrap an API call with error handling. - + Args: func: Function to call *args: Positional arguments fail_fast: Whether to raise on error (default: True) context: Optional context for error message **kwargs: Keyword arguments - + Returns: Function result - + Raises: APIError: If API call fails and fail_fast is True """ @@ -137,4 +132,3 @@ def wrap_api_call(func, *args, fail_fast: bool = True, context: Optional[str] = else: APIErrorHandler.display_api_error(api_error) return None - diff --git a/codewiki/cli/utils/errors.py b/codewiki/cli/utils/errors.py index 667c4a3..6bcf530 100644 --- a/codewiki/cli/utils/errors.py +++ b/codewiki/cli/utils/errors.py @@ -12,7 +12,6 @@ import sys import click -from typing import Optional # Exit codes @@ -26,7 +25,7 @@ class CodeWikiError(Exception): """Base exception for CodeWiki CLI errors.""" - + def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): self.message = message self.exit_code = exit_code @@ -35,28 +34,28 @@ def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): class ConfigurationError(CodeWikiError): """Configuration-related errors.""" - + def __init__(self, message: str): super().__init__(message, EXIT_CONFIG_ERROR) class RepositoryError(CodeWikiError): """Repository-related errors.""" - + def __init__(self, message: str): super().__init__(message, EXIT_REPOSITORY_ERROR) class APIError(CodeWikiError): """LLM API-related errors.""" - + def __init__(self, message: str): super().__init__(message, EXIT_API_ERROR) class FileSystemError(CodeWikiError): """File system-related errors.""" - + def __init__(self, message: str): super().__init__(message, EXIT_FILESYSTEM_ERROR) @@ -64,11 +63,11 @@ def __init__(self, message: str): def handle_error(error: Exception, verbose: bool = False) -> int: """ Handle errors and return appropriate exit code. - + Args: error: The exception to handle verbose: Whether to show detailed error information - + Returns: Exit code for the error """ @@ -79,6 +78,7 @@ def handle_error(error: Exception, verbose: bool = False) -> int: click.secho(f"\n✗ Unexpected error: {error}", fg="red", err=True) if verbose: import traceback + click.echo(traceback.format_exc(), err=True) return EXIT_GENERAL_ERROR @@ -86,7 +86,7 @@ def handle_error(error: Exception, verbose: bool = False) -> int: def error_with_suggestion(message: str, suggestion: str, exit_code: int = EXIT_GENERAL_ERROR): """ Display error message with actionable suggestion and exit. - + Args: message: The error message suggestion: Suggested action to resolve the error @@ -110,4 +110,3 @@ def success(message: str): def info(message: str): """Display an info message.""" click.echo(message) - diff --git a/codewiki/cli/utils/fs.py b/codewiki/cli/utils/fs.py index 88f4cb3..7878fe9 100644 --- a/codewiki/cli/utils/fs.py +++ b/codewiki/cli/utils/fs.py @@ -13,14 +13,14 @@ def ensure_directory(path: Path, mode: int = 0o700) -> Path: """ Ensure directory exists, create if necessary. - + Args: path: Directory path mode: Directory permissions (default: 0o700 - user only) - + Returns: Path to the directory - + Raises: FileSystemError: If directory cannot be created """ @@ -30,8 +30,7 @@ def ensure_directory(path: Path, mode: int = 0o700) -> Path: return path except PermissionError: raise FileSystemError( - f"Permission denied: Cannot create directory {path}\n" - f"Try: chmod u+w {path.parent}" + f"Permission denied: Cannot create directory {path}\nTry: chmod u+w {path.parent}" ) except OSError as e: raise FileSystemError(f"Cannot create directory {path}: {e}") @@ -40,15 +39,15 @@ def ensure_directory(path: Path, mode: int = 0o700) -> Path: def check_writable(path: Path) -> bool: """ Check if a path is writable. - + Args: path: Path to check - + Returns: True if writable, False otherwise """ path = Path(path).expanduser().resolve() - + if path.exists(): return os.access(path, os.W_OK) else: @@ -60,23 +59,23 @@ def check_writable(path: Path) -> bool: def safe_write(path: Path, content: str, encoding: str = "utf-8"): """ Safely write content to a file using atomic write (temp file + rename). - + Args: path: File path content: Content to write encoding: File encoding - + Raises: FileSystemError: If write fails """ path = Path(path).expanduser().resolve() temp_path = path.with_suffix(path.suffix + ".tmp") - + try: # Write to temp file with open(temp_path, "w", encoding=encoding) as f: f.write(content) - + # Atomic rename temp_path.replace(path) except Exception as e: @@ -89,19 +88,19 @@ def safe_write(path: Path, content: str, encoding: str = "utf-8"): def safe_read(path: Path, encoding: str = "utf-8") -> str: """ Safely read content from a file. - + Args: path: File path encoding: File encoding - + Returns: File content - + Raises: FileSystemError: If read fails """ path = Path(path).expanduser().resolve() - + try: with open(path, "r", encoding=encoding) as f: return f.read() @@ -116,10 +115,10 @@ def safe_read(path: Path, encoding: str = "utf-8") -> str: def get_file_size(path: Path) -> int: """ Get file size in bytes. - + Args: path: File path - + Returns: File size in bytes """ @@ -127,64 +126,61 @@ def get_file_size(path: Path) -> int: def find_files( - directory: Path, - extensions: Optional[List[str]] = None, - recursive: bool = True + directory: Path, extensions: Optional[List[str]] = None, recursive: bool = True ) -> List[Path]: """ Find files in directory matching extensions. - + Args: directory: Directory to search extensions: List of file extensions (e.g., ['.py', '.java']) recursive: Search recursively - + Returns: List of matching file paths """ directory = Path(directory).expanduser().resolve() - + if not directory.exists(): return [] - + pattern = "**/*" if recursive else "*" files = [] - + for path in directory.glob(pattern): if not path.is_file(): continue - + if extensions is None or path.suffix in extensions: files.append(path) - + return files def cleanup_directory(path: Path, keep_hidden: bool = True): """ Clean up a directory by removing its contents. - + Args: path: Directory to clean keep_hidden: Keep hidden files/directories (starting with .) - + Raises: FileSystemError: If cleanup fails """ path = Path(path).expanduser().resolve() - + if not path.exists(): return - + try: for item in path.iterdir(): - if keep_hidden and item.name.startswith('.'): + if keep_hidden and item.name.startswith("."): continue - + if item.is_file(): item.unlink() elif item.is_dir(): shutil.rmtree(item) except Exception as e: raise FileSystemError(f"Cannot clean directory {path}: {e}") - diff --git a/codewiki/cli/utils/instructions.py b/codewiki/cli/utils/instructions.py index 7c2bf05..f20e6e7 100644 --- a/codewiki/cli/utils/instructions.py +++ b/codewiki/cli/utils/instructions.py @@ -10,38 +10,38 @@ def compute_github_pages_url(repo_url: str, repo_name: str) -> str: """ Compute expected GitHub Pages URL from repository URL. - + Args: repo_url: GitHub repository URL repo_name: Repository name - + Returns: Expected GitHub Pages URL """ # Extract owner from GitHub URL # e.g., "https://github.com/owner/repo" -> "owner" if "github.com" in repo_url: - parts = repo_url.rstrip('/').split('/') + parts = repo_url.rstrip("/").split("/") if len(parts) >= 2: owner = parts[-2] - repo = parts[-1].replace('.git', '') + repo = parts[-1].replace(".git", "") return f"https://{owner}.github.io/{repo}/" - + return f"https://YOUR_USERNAME.github.io/{repo_name}/" def get_pr_creation_url(repo_url: str, branch_name: str) -> str: """ Get PR creation URL for GitHub. - + Args: repo_url: GitHub repository URL branch_name: Branch name - + Returns: PR creation URL """ - base_url = repo_url.rstrip('/').replace('.git', '') + base_url = repo_url.rstrip("/").replace(".git", "") return f"{base_url}/compare/{branch_name}" @@ -52,11 +52,11 @@ def display_post_generation_instructions( branch_name: Optional[str] = None, github_pages: bool = False, files_generated: list = None, - statistics: dict = None + statistics: dict = None, ): """ Display post-generation instructions. - + Args: output_dir: Output directory path repo_name: Repository name @@ -69,12 +69,12 @@ def display_post_generation_instructions( click.echo() click.secho("✓ Documentation generated successfully!", fg="green", bold=True) click.echo() - + # Output directory click.secho("Output directory:", fg="cyan", bold=True) click.echo(f" {output_dir}") click.echo() - + # Generated files if files_generated: click.secho("Generated files:", fg="cyan", bold=True) @@ -83,45 +83,45 @@ def display_post_generation_instructions( if len(files_generated) > 10: click.echo(f" ... and {len(files_generated) - 10} more") click.echo() - + # Statistics if statistics: click.secho("Statistics:", fg="cyan", bold=True) - if 'module_count' in statistics: + if "module_count" in statistics: click.echo(f" Total modules: {statistics['module_count']}") - if 'total_files_analyzed' in statistics: + if "total_files_analyzed" in statistics: click.echo(f" Files analyzed: {statistics['total_files_analyzed']}") - if 'generation_time' in statistics: - minutes = int(statistics['generation_time'] // 60) - seconds = int(statistics['generation_time'] % 60) + if "generation_time" in statistics: + minutes = int(statistics["generation_time"] // 60) + seconds = int(statistics["generation_time"] % 60) click.echo(f" Generation time: {minutes} minutes {seconds} seconds") # if 'total_tokens_used' in statistics: # tokens = statistics['total_tokens_used'] # click.echo(f" Tokens used: ~{tokens:,}") click.echo() - + # Next steps click.secho("Next steps:", fg="cyan", bold=True) click.echo() - + click.echo("1. Review the generated documentation:") click.echo(f" cat {output_dir}/overview.md") if github_pages: click.echo(f" open {output_dir}/index.html # View in browser") click.echo() - + if branch_name: # Git workflow with branch click.echo("2. Push the documentation branch:") click.secho(f" git push origin {branch_name}", fg="yellow") click.echo() - + if repo_url: pr_url = get_pr_creation_url(repo_url, branch_name) click.echo("3. Create a Pull Request to merge documentation:") click.secho(f" {pr_url}", fg="blue") click.echo() - + click.echo("4. After merge, enable GitHub Pages:") else: click.echo("3. Enable GitHub Pages:") @@ -131,18 +131,18 @@ def display_post_generation_instructions( click.secho(" git add docs/", fg="yellow") click.secho(' git commit -m "Add generated documentation"', fg="yellow") click.echo() - + click.echo("3. Push to GitHub:") click.secho(" git push origin main", fg="yellow") click.echo() - + click.echo("4. Enable GitHub Pages:") - + click.echo(" - Go to repository Settings → Pages") click.echo(" - Source: Deploy from a branch") click.echo(" - Branch: main, folder: /docs") click.echo() - + if repo_url: github_pages_url = compute_github_pages_url(repo_url, repo_name) click.echo("5. Your documentation will be available at:") @@ -151,13 +151,11 @@ def display_post_generation_instructions( def display_generation_summary( - success: bool, - error_message: Optional[str] = None, - output_dir: Optional[Path] = None + success: bool, error_message: Optional[str] = None, output_dir: Optional[Path] = None ): """ Display generation summary (success or failure). - + Args: success: Whether generation was successful error_message: Error message if failed @@ -176,4 +174,3 @@ def display_generation_summary( click.echo() click.echo(error_message) click.echo() - diff --git a/codewiki/cli/utils/logging.py b/codewiki/cli/utils/logging.py index 8086340..6da975d 100644 --- a/codewiki/cli/utils/logging.py +++ b/codewiki/cli/utils/logging.py @@ -2,7 +2,6 @@ Logging utilities for CLI with colored output and progress tracking. """ -import sys from datetime import datetime from typing import Optional import click @@ -10,43 +9,43 @@ class CLILogger: """Logger for CLI with support for verbose and normal modes.""" - + def __init__(self, verbose: bool = False): """ Initialize the logger. - + Args: verbose: Enable verbose output """ self.verbose = verbose self.start_time = datetime.now() - + def debug(self, message: str): """Log debug message (only in verbose mode).""" if self.verbose: timestamp = datetime.now().strftime("%H:%M:%S") click.secho(f"[{timestamp}] {message}", fg="cyan", dim=True) - + def info(self, message: str): """Log info message.""" click.echo(message) - + def success(self, message: str): """Log success message in green.""" click.secho(f"✓ {message}", fg="green") - + def warning(self, message: str): """Log warning message in yellow.""" click.secho(f"⚠️ {message}", fg="yellow") - + def error(self, message: str): """Log error message in red.""" click.secho(f"✗ {message}", fg="red", err=True) - + def step(self, message: str, step: Optional[int] = None, total: Optional[int] = None): """ Log a processing step. - + Args: message: Step description step: Current step number @@ -56,15 +55,15 @@ def step(self, message: str, step: Optional[int] = None, total: Optional[int] = prefix = f"[{step}/{total}]" else: prefix = "→" - + click.secho(f"{prefix} {message}", fg="blue", bold=True) - + def elapsed_time(self) -> str: """Get elapsed time since logger was created.""" elapsed = datetime.now() - self.start_time minutes = int(elapsed.total_seconds() // 60) seconds = int(elapsed.total_seconds() % 60) - + if minutes > 0: return f"{minutes}m {seconds}s" else: @@ -74,12 +73,11 @@ def elapsed_time(self) -> str: def create_logger(verbose: bool = False) -> CLILogger: """ Create and return a CLI logger. - + Args: verbose: Enable verbose output - + Returns: Configured CLILogger instance """ return CLILogger(verbose=verbose) - diff --git a/codewiki/cli/utils/progress.py b/codewiki/cli/utils/progress.py index f61efc1..2c6be08 100644 --- a/codewiki/cli/utils/progress.py +++ b/codewiki/cli/utils/progress.py @@ -3,15 +3,14 @@ """ import time -from typing import Optional, Callable -from datetime import datetime +from typing import Optional import click class ProgressTracker: """ Progress tracker with stages and ETA estimation. - + Stages: 1. Dependency Analysis (40% of time) 2. Module Clustering (20% of time) @@ -19,7 +18,7 @@ class ProgressTracker: 4. HTML Generation (5% of time, optional) 5. Finalization (5% of time) """ - + # Stage weights (percentage of total time) STAGE_WEIGHTS = { 1: 0.40, # Dependency Analysis @@ -28,7 +27,7 @@ class ProgressTracker: 4: 0.05, # HTML Generation (optional) 5: 0.05, # Finalization } - + STAGE_NAMES = { 1: "Dependency Analysis", 2: "Module Clustering", @@ -36,11 +35,11 @@ class ProgressTracker: 4: "HTML Generation", 5: "Finalization", } - + def __init__(self, total_stages: int = 5, verbose: bool = False): """ Initialize progress tracker. - + Args: total_stages: Number of stages verbose: Enable verbose output @@ -51,11 +50,11 @@ def __init__(self, total_stages: int = 5, verbose: bool = False): self.start_time = time.time() self.verbose = verbose self.current_stage_start = self.start_time - + def start_stage(self, stage: int, description: Optional[str] = None): """ Start a new stage. - + Args: stage: Stage number (1-5) description: Optional custom description @@ -63,106 +62,96 @@ def start_stage(self, stage: int, description: Optional[str] = None): self.current_stage = stage self.stage_progress = 0.0 self.current_stage_start = time.time() - + stage_name = description or self.STAGE_NAMES.get(stage, f"Stage {stage}") - + if self.verbose: elapsed = self._format_elapsed() click.secho( f"\n[{elapsed}] Phase {stage}/{self.total_stages}: {stage_name}", fg="blue", - bold=True + bold=True, ) else: - click.secho( - f"[{stage}/{self.total_stages}] {stage_name}", - fg="blue", - bold=True - ) - + click.secho(f"[{stage}/{self.total_stages}] {stage_name}", fg="blue", bold=True) + def update_stage(self, progress: float, message: Optional[str] = None): """ Update progress within current stage. - + Args: progress: Progress percentage (0.0 to 1.0) message: Optional progress message """ self.stage_progress = min(1.0, max(0.0, progress)) - + if self.verbose and message: elapsed = self._format_elapsed() click.echo(f"[{elapsed}] {message}") - + def complete_stage(self, message: Optional[str] = None): """ Complete current stage. - + Args: message: Optional completion message """ self.stage_progress = 1.0 - + if self.verbose: elapsed = self._format_elapsed() stage_time = time.time() - self.current_stage_start stage_name = self.STAGE_NAMES.get(self.current_stage, f"Stage {self.current_stage}") - click.secho( - f"[{elapsed}] {stage_name} complete ({stage_time:.1f}s)", - fg="green" - ) + click.secho(f"[{elapsed}] {stage_name} complete ({stage_time:.1f}s)", fg="green") if message: click.echo(f"[{elapsed}] {message}") - + def get_overall_progress(self) -> float: """ Get overall progress percentage. - + Returns: Progress (0.0 to 1.0) """ - completed_weight = sum( - self.STAGE_WEIGHTS.get(s, 0) - for s in range(1, self.current_stage) - ) - + completed_weight = sum(self.STAGE_WEIGHTS.get(s, 0) for s in range(1, self.current_stage)) + current_weight = self.STAGE_WEIGHTS.get(self.current_stage, 0) * self.stage_progress - + return completed_weight + current_weight - + def _format_elapsed(self) -> str: """Format elapsed time.""" elapsed = time.time() - self.start_time minutes = int(elapsed // 60) seconds = int(elapsed % 60) - + if minutes > 0: return f"{minutes:02d}:{seconds:02d}" else: return f"00:{seconds:02d}" - + def get_eta(self) -> Optional[str]: """ Estimate time remaining. - + Returns: ETA string or None if cannot estimate """ elapsed = time.time() - self.start_time progress = self.get_overall_progress() - + if progress <= 0.0: return None - + total_estimated = elapsed / progress remaining = total_estimated - elapsed - + if remaining < 0: return "< 1 min" - + minutes = int(remaining // 60) seconds = int(remaining % 60) - + if minutes > 60: hours = minutes // 60 minutes = minutes % 60 @@ -175,11 +164,11 @@ def get_eta(self) -> Optional[str]: class ModuleProgressBar: """Progress bar for module-by-module generation.""" - + def __init__(self, total_modules: int, verbose: bool = False): """ Initialize module progress bar. - + Args: total_modules: Total number of modules to process verbose: Enable verbose output @@ -188,7 +177,7 @@ def __init__(self, total_modules: int, verbose: bool = False): self.current_module = 0 self.verbose = verbose self.bar = None - + if not verbose: self.bar = click.progressbar( length=total_modules, @@ -197,26 +186,25 @@ def __init__(self, total_modules: int, verbose: bool = False): show_percent=True, ) self.bar.__enter__() - + def update(self, module_name: str, cached: bool = False): """ Update progress for a module. - + Args: module_name: Name of the module cached: Whether the module was loaded from cache """ self.current_module += 1 - + if self.verbose: status = "✓ (cached)" if cached else "⟳ (generating)" click.echo(f" [{self.current_module}/{self.total_modules}] {module_name}... {status}") elif self.bar: self.bar.update(1) - + def finish(self): """Finish progress bar.""" if self.bar: self.bar.__exit__(None, None, None) self.bar = None - diff --git a/codewiki/cli/utils/repo_validator.py b/codewiki/cli/utils/repo_validator.py index 12e9f95..2c76ff5 100644 --- a/codewiki/cli/utils/repo_validator.py +++ b/codewiki/cli/utils/repo_validator.py @@ -12,53 +12,53 @@ # Supported file extensions by language SUPPORTED_EXTENSIONS = { - '.py', # Python - '.java', # Java - '.js', # JavaScript - '.jsx', # JavaScript (React) - '.ts', # TypeScript - '.tsx', # TypeScript (React) - '.c', # C - '.h', # C headers - '.cpp', # C++ - '.hpp', # C++ headers - '.cc', # C++ - '.hh', # C++ headers - '.cxx', # C++ - '.hxx', # C++ headers - '.cs', # C# - '.php', # PHP - '.phtml', # PHP templates - '.inc', # PHP includes - '.kt', # Kotlin - '.kts', # Kotlin Scripts + ".py", # Python + ".java", # Java + ".js", # JavaScript + ".jsx", # JavaScript (React) + ".ts", # TypeScript + ".tsx", # TypeScript (React) + ".c", # C + ".h", # C headers + ".cpp", # C++ + ".hpp", # C++ headers + ".cc", # C++ + ".hh", # C++ headers + ".cxx", # C++ + ".hxx", # C++ headers + ".cs", # C# + ".php", # PHP + ".phtml", # PHP templates + ".inc", # PHP includes + ".kt", # Kotlin + ".kts", # Kotlin Scripts } def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: """ Validate repository for documentation generation. - + Checks: - Path exists and is a directory - Contains supported code files - Has sufficient files for meaningful documentation - + Args: repo_path: Path to repository - + Returns: Tuple of (validated_path, language_counts) - + Raises: RepositoryError: If validation fails """ # Validate path exists repo_path = validate_repository_path(repo_path) - + # Detect languages languages = detect_supported_languages(repo_path) - + if not languages: raise RepositoryError( f"No supported code files found in {repo_path}\n\n" @@ -67,67 +67,63 @@ def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: " cd /path/to/your/project\n" " codewiki generate" ) - + return repo_path, languages def check_writable_output(output_dir: Path) -> Path: """ Check if output directory is writable. - + Args: output_dir: Output directory path - + Returns: Validated output directory path - + Raises: RepositoryError: If output directory is not writable """ output_dir = Path(output_dir).expanduser().resolve() - + # Check if output directory exists if output_dir.exists(): if not output_dir.is_dir(): - raise RepositoryError( - f"Output path exists but is not a directory: {output_dir}" - ) - + raise RepositoryError(f"Output path exists but is not a directory: {output_dir}") + # Check if writable if not os.access(output_dir, os.W_OK): raise RepositoryError( - f"Output directory is not writable: {output_dir}\n\n" - f"Try: chmod u+w {output_dir}" + f"Output directory is not writable: {output_dir}\n\nTry: chmod u+w {output_dir}" ) else: # Check if parent is writable parent = output_dir.parent if not parent.exists(): - raise RepositoryError( - f"Parent directory does not exist: {parent}" - ) - + raise RepositoryError(f"Parent directory does not exist: {parent}") + if not os.access(parent, os.W_OK): raise RepositoryError( f"Cannot create output directory (parent not writable): {parent}\n\n" f"Try: chmod u+w {parent}" ) - + return output_dir def _get_git_repo(repo_path: Path): """ Find a git repository starting at repo_path and searching parent directories. - + Args: repo_path: Path to start searching from - + Returns: git.Repo instance or None if no repository found """ try: import git + return git.Repo(repo_path, search_parent_directories=True) except Exception: return None @@ -136,13 +132,13 @@ def _get_git_repo(repo_path: Path): def is_git_repository(repo_path: Path) -> bool: """ Check if path is inside a git repository. - + Searches parent directories if .git is not directly at repo_path, supporting monorepo subdirectories. - + Args: repo_path: Path to check - + Returns: True if inside a git repository, False otherwise """ @@ -152,19 +148,19 @@ def is_git_repository(repo_path: Path) -> bool: def get_git_commit_hash(repo_path: Path) -> str: """ Get current git commit hash. - + Searches parent directories to support monorepo subdirectories. - + Args: repo_path: Path inside a git repository - + Returns: Commit hash or empty string if not in a git repo """ repo = _get_git_repo(repo_path) if repo is None: return "" - + try: return repo.head.commit.hexsha except Exception: @@ -174,19 +170,19 @@ def get_git_commit_hash(repo_path: Path) -> str: def get_git_branch(repo_path: Path) -> str: """ Get current git branch name. - + Searches parent directories to support monorepo subdirectories. - + Args: repo_path: Path inside a git repository - + Returns: Branch name or empty string if not in a git repo """ repo = _get_git_repo(repo_path) if repo is None: return "" - + try: return repo.active_branch.name except Exception: @@ -196,10 +192,10 @@ def get_git_branch(repo_path: Path) -> str: def count_code_files(repo_path: Path) -> int: """ Count supported code files in repository. - + Args: repo_path: Repository path - + Returns: Number of code files """ @@ -207,4 +203,3 @@ def count_code_files(repo_path: Path) -> int: for ext in SUPPORTED_EXTENSIONS: count += len(list(repo_path.rglob(f"*{ext}"))) return count - diff --git a/codewiki/cli/utils/validation.py b/codewiki/cli/utils/validation.py index 9711ba3..6f206d4 100644 --- a/codewiki/cli/utils/validation.py +++ b/codewiki/cli/utils/validation.py @@ -2,9 +2,8 @@ Validation utilities for CLI inputs and configuration. """ -import re from pathlib import Path -from typing import Optional, List, Tuple +from typing import List, Tuple from urllib.parse import urlparse from codewiki.cli.utils.errors import ConfigurationError, RepositoryError @@ -13,40 +12,39 @@ def validate_url(url: str, require_https: bool = False, allow_localhost: bool = True) -> str: """ Validate URL format. - + Args: url: URL to validate require_https: Require HTTPS scheme (except localhost) allow_localhost: Allow localhost URLs - + Returns: Validated URL - + Raises: ConfigurationError: If URL is invalid """ try: parsed = urlparse(url) - + # Check scheme if not parsed.scheme: raise ConfigurationError(f"Invalid URL (missing scheme): {url}") - + # Check HTTPS requirement - if require_https and parsed.scheme != 'https': + if require_https and parsed.scheme != "https": # Allow HTTP for localhost - if allow_localhost and parsed.hostname in ['localhost', '127.0.0.1', '::1']: + if allow_localhost and parsed.hostname in ["localhost", "127.0.0.1", "::1"]: pass else: raise ConfigurationError( - f"URL must use HTTPS: {url}\n" - f"HTTP is only allowed for localhost" + f"URL must use HTTPS: {url}\nHTTP is only allowed for localhost" ) - + # Check hostname if not parsed.hostname: raise ConfigurationError(f"Invalid URL (missing hostname): {url}") - + return url except ValueError as e: raise ConfigurationError(f"Invalid URL format: {url}\nError: {e}") @@ -55,75 +53,71 @@ def validate_url(url: str, require_https: bool = False, allow_localhost: bool = def validate_api_key(api_key: str, min_length: int = 10) -> str: """ Validate API key format. - + Args: api_key: API key to validate min_length: Minimum key length - + Returns: Validated API key - + Raises: ConfigurationError: If API key is invalid """ if not api_key or not api_key.strip(): raise ConfigurationError("API key cannot be empty") - + api_key = api_key.strip() - + if len(api_key) < min_length: - raise ConfigurationError( - f"API key too short (minimum {min_length} characters)" - ) - + raise ConfigurationError(f"API key too short (minimum {min_length} characters)") + return api_key def validate_model_name(model: str) -> str: """ Validate model name format. - + Args: model: Model name to validate - + Returns: Validated model name - + Raises: ConfigurationError: If model name is invalid """ if not model or not model.strip(): raise ConfigurationError("Model name cannot be empty") - + return model.strip() def validate_output_directory(path: str) -> Path: """ Validate output directory path. - + Args: path: Directory path to validate - + Returns: Validated Path object - + Raises: ConfigurationError: If path is invalid """ if not path or not path.strip(): raise ConfigurationError("Output directory cannot be empty") - + try: resolved_path = Path(path).expanduser().resolve() - + # Check if path is writable (or parent is writable if path doesn't exist) if resolved_path.exists(): if not resolved_path.is_dir(): - raise ConfigurationError( - f"Output path exists but is not a directory: {path}" - ) - + raise ConfigurationError(f"Output path exists but is not a directory: {path}") + return resolved_path except Exception as e: raise ConfigurationError(f"Invalid output directory path: {path}\nError: {e}") @@ -132,77 +126,96 @@ def validate_output_directory(path: str) -> Path: def validate_repository_path(path: Path) -> Path: """ Validate repository path exists and contains code files. - + Args: path: Repository path to validate - + Returns: Validated Path object - + Raises: RepositoryError: If repository is invalid """ path = Path(path).expanduser().resolve() - + if not path.exists(): raise RepositoryError(f"Repository path does not exist: {path}") - + if not path.is_dir(): raise RepositoryError(f"Repository path is not a directory: {path}") - + return path def detect_supported_languages(directory: Path) -> List[Tuple[str, int]]: """ Detect supported programming languages in a directory. - + Args: directory: Directory to scan - + Returns: List of (language, file_count) tuples """ language_extensions = { - 'Python': ['.py'], - 'Java': ['.java'], - 'JavaScript': ['.js', '.jsx'], - 'TypeScript': ['.ts', '.tsx'], - 'C': ['.c', '.h'], - 'C++': ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], - 'C#': ['.cs'], - 'PHP': ['.php', '.phtml', '.inc'], - 'Kotlin': ['.kt', '.kts'], + "Python": [".py"], + "Java": [".java"], + "JavaScript": [".js", ".jsx"], + "TypeScript": [".ts", ".tsx"], + "C": [".c", ".h"], + "C++": [".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx"], + "C#": [".cs"], + "PHP": [".php", ".phtml", ".inc"], + "Kotlin": [".kt", ".kts"], } - + # Directories to exclude from counting excluded_dirs = { - 'node_modules', '__pycache__', '.git', 'build', 'dist', - '.venv', 'venv', 'env', '.env', 'target', 'bin', 'obj', - '.pytest_cache', '.mypy_cache', '.tox', 'coverage', - 'htmlcov', '.eggs', '*.egg-info', 'vendor', 'bower_components', - '.idea', '.vscode', '.gradle', '.mvn' + "node_modules", + "__pycache__", + ".git", + "build", + "dist", + ".venv", + "venv", + "env", + ".env", + "target", + "bin", + "obj", + ".pytest_cache", + ".mypy_cache", + ".tox", + "coverage", + "htmlcov", + ".eggs", + "*.egg-info", + "vendor", + "bower_components", + ".idea", + ".vscode", + ".gradle", + ".mvn", } - + def should_exclude_file(file_path: Path) -> bool: """Check if file is in an excluded directory.""" parts = file_path.parts return any(excluded_dir in parts for excluded_dir in excluded_dirs) - + language_counts = {} - + for language, extensions in language_extensions.items(): count = 0 for ext in extensions: # Filter out files in excluded directories count += sum( - 1 for f in directory.rglob(f"*{ext}") - if f.is_file() and not should_exclude_file(f) + 1 for f in directory.rglob(f"*{ext}") if f.is_file() and not should_exclude_file(f) ) - + if count > 0: language_counts[language] = count - + # Sort by count descending return sorted(language_counts.items(), key=lambda x: x[1], reverse=True) @@ -210,21 +223,21 @@ def should_exclude_file(file_path: Path) -> bool: def is_top_tier_model(model: str) -> bool: """ Check if a model is considered top-tier for clustering. - + Args: model: Model name - + Returns: True if top-tier, False otherwise """ top_tier_models = [ - 'claude-opus', - 'claude-sonnet', - 'gpt-4', - 'gpt-5', - 'gemini-2.5', + "claude-opus", + "claude-sonnet", + "gpt-4", + "gpt-5", + "gemini-2.5", ] - + model_lower = model.lower() return any(tier in model_lower for tier in top_tier_models) @@ -232,20 +245,19 @@ def is_top_tier_model(model: str) -> bool: def mask_api_key(api_key: str, visible_chars: int = 4) -> str: """ Mask API key for display, showing only first and last few characters. - + Args: api_key: API key to mask visible_chars: Number of visible characters at start and end - + Returns: Masked API key (e.g., "sk-1234...5678") """ if not api_key: return "Not set" - + if len(api_key) <= visible_chars * 2: # Key too short, mask everything except edges return f"{api_key[:2]}...{api_key[-2:]}" - - return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" + return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" diff --git a/codewiki/hooks/capture_session_end.py b/codewiki/hooks/capture_session_end.py index 4b1891b..9a4f60e 100644 --- a/codewiki/hooks/capture_session_end.py +++ b/codewiki/hooks/capture_session_end.py @@ -52,6 +52,7 @@ Stdout is emitted in the CodeBuddy-expected ``{continue, systemMessage}`` shape. """ + from __future__ import annotations import json @@ -59,7 +60,6 @@ import subprocess import sys import tempfile -from datetime import datetime, timezone from pathlib import Path REPO = Path(__file__).resolve().parents[2] # /.codebuddy/hooks/ -> @@ -166,9 +166,12 @@ def main() -> int: json.dump(event, fh) cmd = [ - sys.executable, "-m", "codewiki.mcp._ide_hook", + sys.executable, + "-m", + "codewiki.mcp._ide_hook", "--enable", - "--repo-path", repo_path, + "--repo-path", + repo_path, ] if tmp: cmd += ["--conversation", tmp] @@ -185,8 +188,12 @@ def main() -> int: child_env = dict(env) if tmp: child_env["CODEWIKI_HOOK_EVENT_FILE"] = tmp - kwargs: dict = {"cwd": str(REPO), "env": child_env, "stdout": subprocess.DEVNULL, - "stderr": subprocess.DEVNULL} + kwargs: dict = { + "cwd": str(REPO), + "env": child_env, + "stdout": subprocess.DEVNULL, + "stderr": subprocess.DEVNULL, + } if sys.platform == "win32": kwargs["creationflags"] = getattr(subprocess, "DETACHED_PROCESS", 0) else: @@ -195,12 +202,14 @@ def main() -> int: except Exception as e: # noqa: BLE001 - never crash the IDE hook # If we cannot even spawn the child, surface a non-blocking hint but # still let the session end cleanly. - print(json.dumps({"continue": True, - "systemMessage": f"team-memory capture not started: {e}"})) + print( + json.dumps({"continue": True, "systemMessage": f"team-memory capture not started: {e}"}) + ) return 0 - print(json.dumps({"continue": True, - "systemMessage": "team-memory capture started in background"})) + print( + json.dumps({"continue": True, "systemMessage": "team-memory capture started in background"}) + ) return 0 diff --git a/codewiki/mcp/_ide_hook.py b/codewiki/mcp/_ide_hook.py index 408ffb3..8b05a9c 100644 --- a/codewiki/mcp/_ide_hook.py +++ b/codewiki/mcp/_ide_hook.py @@ -73,14 +73,10 @@ def _extract_inline_turns(data: Dict[str, Any]) -> Optional[list]: keeps the in-memory payload consistent). """ _KEEP = {"user", "assistant"} - for key in ("conversation", "messages", "turns", - "transcript_turns", "chat"): + for key in ("conversation", "messages", "turns", "transcript_turns", "chat"): val = data.get(key) if isinstance(val, list) and val: - kept = [ - t for t in val - if isinstance(t, dict) and t.get("role") in _KEEP - ] + kept = [t for t in val if isinstance(t, dict) and t.get("role") in _KEEP] return kept if kept else val return None @@ -101,8 +97,7 @@ def _load_event(args: argparse.Namespace) -> Optional[Dict[str, Any]]: if isinstance(data, dict): # Mirror the stdin branch: if a transcript path is provided, load it. if "transcript_path" in data or "transcript" in data: - loaded = _load_transcript( - data.get("transcript_path") or data.get("transcript")) + loaded = _load_transcript(data.get("transcript_path") or data.get("transcript")) data = dict(data) if loaded is not None: data["conversation"] = loaded @@ -142,8 +137,7 @@ def _load_event(args: argparse.Namespace) -> Optional[Dict[str, Any]]: # path is provided we load it; otherwise we keep the raw event # and let the caller decide (it cannot synthesize turns). if "transcript_path" in data or "transcript" in data: - loaded = _load_transcript(data.get("transcript_path") - or data.get("transcript")) + loaded = _load_transcript(data.get("transcript_path") or data.get("transcript")) data = dict(data) if loaded is not None: data["conversation"] = loaded @@ -251,10 +245,16 @@ def _extract_codebuddy_message_text(msg_data: dict) -> str: return "" -_NOISE_BLOCK_TYPES = frozenset({ - "tool-call", "tool_call", "tool-result", "tool_result", - "reasoning", "thinking", -}) +_NOISE_BLOCK_TYPES = frozenset( + { + "tool-call", + "tool_call", + "tool-result", + "tool_result", + "reasoning", + "thinking", + } +) def _text_from_content_blocks(blocks: list) -> str: @@ -354,19 +354,29 @@ def main(argv: Optional[list] = None) -> int: parser = argparse.ArgumentParser( description="IDE hook: capture a conversation into repowiki/raw/ (no distillation)." ) - parser.add_argument("--enable", action="store_true", - help="Enable the hook for this invocation " - "(otherwise requires CODEWIKI_TEAM_MEMORY_HOOK=1).") - parser.add_argument("--conversation", help="Path to a JSON file with the " - "conversation payload (list of turns or {turns: [...]}).") - parser.add_argument("--repo-path", help="Absolute path to the repo " - "(used to resolve repowiki/raw/).") + parser.add_argument( + "--enable", + action="store_true", + help="Enable the hook for this invocation " + "(otherwise requires CODEWIKI_TEAM_MEMORY_HOOK=1).", + ) + parser.add_argument( + "--conversation", + help="Path to a JSON file with the conversation payload (list of turns or {turns: [...]}).", + ) + parser.add_argument( + "--repo-path", help="Absolute path to the repo (used to resolve repowiki/raw/)." + ) parser.add_argument("--session-id", help="Active session id (optional).") parser.add_argument("--link-to", help="Wiki object id this conversation relates to.") - parser.add_argument("--task-id", help="Task id this conversation is bound to " - "(stamped into raw frontmatter so distillation routes memories back).") - parser.add_argument("--keep-raw", action="store_true", - help="Hint distill_conversation to retain the raw file.") + parser.add_argument( + "--task-id", + help="Task id this conversation is bound to " + "(stamped into raw frontmatter so distillation routes memories back).", + ) + parser.add_argument( + "--keep-raw", action="store_true", help="Hint distill_conversation to retain the raw file." + ) args = parser.parse_args(argv) # The wrapper passes the temp event file path via this env var so we can @@ -403,9 +413,11 @@ def _pick(key, cli_val): if not conversation: hook_event = event.get("hook_event_name") or event.get("event") if hook_event in ("SessionEnd", "Stop", "PreCompact") and "session_id" in event: - print(f"ide-hook: {hook_event} event has no conversation turns and no " - "usable transcript_path; capturing the event envelope only " - "(the IDE did not provide an inline transcript).") + print( + f"ide-hook: {hook_event} event has no conversation turns and no " + "usable transcript_path; capturing the event envelope only " + "(the IDE did not provide an inline transcript)." + ) # Fall through: capture the event envelope as a minimal record. # NOTE: role must be "user" (not "system") -- capture_conversation # drops every role outside {user, assistant} in _extract_transcript, @@ -413,14 +425,17 @@ def _pick(key, cli_val): # test_envelope_does_not_supersede_full_transcript). The envelope # body carries no system-injection tags, so stripping is a no-op. is_envelope = True - conversation = [{ - "role": "user", - "content": (f"[team-memory] {hook_event} hook fired but the IDE " - "provided no inline transcript and no readable " - "transcript_path. Raw event envelope preserved for " - "diagnosis. Event keys: " - + ", ".join(sorted(event.keys())) + "."), - }] + conversation = [ + { + "role": "user", + "content": ( + f"[team-memory] {hook_event} hook fired but the IDE " + "provided no inline transcript and no readable " + "transcript_path. Raw event envelope preserved for " + "diagnosis. Event keys: " + ", ".join(sorted(event.keys())) + "." + ), + } + ] else: print("ide-hook: payload has no 'conversation' turns; nothing to capture.") return 0 @@ -441,18 +456,12 @@ def _pick(key, cli_val): # captures sharing the same source_session_id; if the envelope carried # it, a later SessionEnd without transcript would overwrite a # previously captured full transcript (data loss). - "source_session_id": ( - "" if is_envelope - else (_pick("session_id", args.session_id) or "") - ), + "source_session_id": ("" if is_envelope else (_pick("session_id", args.session_id) or "")), # Task binding, stamped into raw frontmatter so distill_conversation can # route distilled memories back to the task. Like source_session_id, the # envelope (no real transcript) must NOT carry task_id — it is not a real # conversation and would otherwise pollute per-task memory routing. - "task_id": ( - "" if is_envelope - else (_pick("task_id", args.task_id) or "") - ), + "task_id": ("" if is_envelope else (_pick("task_id", args.task_id) or "")), } if not arguments["repo_path"]: print("ide-hook: repo_path is required to resolve repowiki/raw/.", file=sys.stderr) diff --git a/codewiki/mcp/cache.py b/codewiki/mcp/cache.py index 031ed88..2c0b3b3 100644 --- a/codewiki/mcp/cache.py +++ b/codewiki/mcp/cache.py @@ -1,4 +1,5 @@ """SQLite analysis cache: components, fingerprints, deps, search.""" + from __future__ import annotations import hashlib @@ -29,7 +30,8 @@ def _sql_chunks(items: List[Any], size: int = _SQL_CHUNK_SIZE) -> List[List[Any]]: """Split *items* into chunks small enough for SQL IN(...) placeholders.""" - return [items[i:i + size] for i in range(0, len(items), size)] + return [items[i : i + size] for i in range(0, len(items), size)] + # ------------------------------------------------------------------ Shared BM25 tokeniser @@ -40,22 +42,156 @@ def _sql_chunks(items: List[Any], size: int = _SQL_CHUNK_SIZE) -> List[List[Any] _STOPWORDS: Set[str] = { # English function words - "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", - "have", "has", "had", "do", "does", "did", "will", "would", "could", - "should", "may", "might", "shall", "can", "need", "must", "it", "its", - "this", "that", "these", "those", "i", "you", "he", "she", "we", "they", - "me", "him", "her", "us", "them", "my", "your", "his", "our", "their", - "what", "which", "who", "whom", "where", "when", "why", "how", "all", - "each", "every", "both", "few", "more", "most", "other", "some", "such", - "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", - "just", "because", "but", "and", "or", "if", "while", "about", "with", - "of", "at", "by", "for", "in", "on", "to", "from", "as", "into", + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "be", + "been", + "being", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "could", + "should", + "may", + "might", + "shall", + "can", + "need", + "must", + "it", + "its", + "this", + "that", + "these", + "those", + "i", + "you", + "he", + "she", + "we", + "they", + "me", + "him", + "her", + "us", + "them", + "my", + "your", + "his", + "our", + "their", + "what", + "which", + "who", + "whom", + "where", + "when", + "why", + "how", + "all", + "each", + "every", + "both", + "few", + "more", + "most", + "other", + "some", + "such", + "no", + "nor", + "not", + "only", + "own", + "same", + "so", + "than", + "too", + "very", + "just", + "because", + "but", + "and", + "or", + "if", + "while", + "about", + "with", + "of", + "at", + "by", + "for", + "in", + "on", + "to", + "from", + "as", + "into", # Chinese function words - "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一", - "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着", "没有", - "看", "好", "自己", "这", "他", "她", "它", "们", "那", "些", "什么", - "怎么", "如何", "可以", "能", "吗", "呢", "吧", "啊", "哦", "嗯", - "这个", "那个", "已经", "还是", "因为", "所以", "但是", "而且", "或者", + "的", + "了", + "在", + "是", + "我", + "有", + "和", + "就", + "不", + "人", + "都", + "一", + "一个", + "上", + "也", + "很", + "到", + "说", + "要", + "去", + "你", + "会", + "着", + "没有", + "看", + "好", + "自己", + "这", + "他", + "她", + "它", + "们", + "那", + "些", + "什么", + "怎么", + "如何", + "可以", + "能", + "吗", + "呢", + "吧", + "啊", + "哦", + "嗯", + "这个", + "那个", + "已经", + "还是", + "因为", + "所以", + "但是", + "而且", + "或者", } _JIEBA_AVAILABLE: Optional[bool] = None @@ -67,6 +203,7 @@ def _check_jieba() -> bool: if _JIEBA_AVAILABLE is None: try: import jieba + jieba.setLogLevel(logging.WARNING) _JIEBA_AVAILABLE = True except ImportError: @@ -87,13 +224,15 @@ def _tokenize(text: str) -> List[str]: text = _MARKUP_RE.sub(" ", text) if _check_jieba(): import jieba + raw = jieba.lcut(text) else: raw = _TOKEN_SPLIT_RE.split(text.lower()) return [ t.strip().lower() for t in raw - if t.strip() and len(t.strip()) >= 2 + if t.strip() + and len(t.strip()) >= 2 and not t.strip().isdigit() and t.strip().lower() not in _STOPWORDS ] @@ -128,6 +267,7 @@ def _parse_frontmatter_dict(text: str) -> Dict[str, Any]: return {} try: import yaml + result = yaml.safe_load(fm_text) return result if isinstance(result, dict) else {} except Exception: @@ -140,7 +280,9 @@ def _parse_frontmatter_dict(text: str) -> Dict[str, Any]: key = key.strip() val = val.strip().strip('"').strip("'") if val.startswith("[") and val.endswith("]"): - val = [v.strip().strip('"').strip("'") for v in val[1:-1].split(",") if v.strip()] + val = [ + v.strip().strip('"').strip("'") for v in val[1:-1].split(",") if v.strip() + ] if key: result[key] = val return result @@ -150,6 +292,7 @@ def _parse_frontmatter_dict(text: str) -> Dict[str, Any]: _ontology_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {} + def _load_ontology(output_dir: Optional[Path]) -> Dict[str, List[str]]: """Load ontology.yaml and build synonym expansion map. @@ -175,6 +318,7 @@ def _load_ontology(output_dir: Optional[Path]) -> Dict[str, List[str]]: if cached and cached[0] == mtime: return cached[1] import yaml + with open(onto_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) if not isinstance(data, dict) or "terms" not in data: @@ -321,13 +465,13 @@ def _build_indexable_text(content: str, page_type: Optional[str] = None) -> str: "workaround": 0.05, } _STATUS_AUTHORITY: Dict[str, float] = { - "draft": -0.25, # unreviewed knowledge sinks below verified content + "draft": -0.25, # unreviewed knowledge sinks below verified content "stable": 0.05, "deprecated": -0.35, } -_SCENARIO_AUTHORITY = 0.15 # L2 scenario blocks (wiki/scenarios/) -_DOCTRINE_AUTHORITY = 0.20 # L3 project doctrine (doctrine.md) -_SOURCE_AUTHORITY = -0.20 # raw/sources/ third-party material +_SCENARIO_AUTHORITY = 0.15 # L2 scenario blocks (wiki/scenarios/) +_DOCTRINE_AUTHORITY = 0.20 # L3 project doctrine (doctrine.md) +_SOURCE_AUTHORITY = -0.20 # raw/sources/ third-party material _AUTHORITY_MIN, _AUTHORITY_MAX = 0.7, 1.3 @@ -348,8 +492,17 @@ def _doc_authority(doc_key: str, source: str, content: str = "") -> float: elif source == "note" or dk.startswith("notes/"): fm = _parse_frontmatter_dict(content) if content else {} meta = fm.get("metadata") if isinstance(fm.get("metadata"), dict) else {} - note_type = str(fm.get("type") or fm.get("note_type") - or meta.get("type") or meta.get("note_type") or "").strip().lower() + note_type = ( + str( + fm.get("type") + or fm.get("note_type") + or meta.get("type") + or meta.get("note_type") + or "" + ) + .strip() + .lower() + ) status = str(fm.get("status") or meta.get("status") or "").strip().lower() offset += _NOTE_TYPE_AUTHORITY.get(note_type, 0.0) offset += _STATUS_AUTHORITY.get(status, 0.0) @@ -496,6 +649,7 @@ def _load_retrieval_usage_map( return {} try: from codewiki.mcp.tools import telemetry + usage = telemetry.aggregate_usage(Path(output_dir)) except Exception as e: logger.debug("Failed to load telemetry usage: %s", e) @@ -523,6 +677,7 @@ def _load_usage_schema(output_dir: Optional[Path]) -> dict: return {} try: from codewiki.src.config import SCHEMA_FILENAME + name = SCHEMA_FILENAME except Exception: name = "schema.yaml" @@ -539,6 +694,7 @@ def _load_usage_schema(output_dir: Optional[Path]) -> dict: if mtime is not None: try: import yaml + with open(p, "r", encoding="utf-8") as fh: loaded = yaml.safe_load(fh) or {} if isinstance(loaded, dict): @@ -566,58 +722,107 @@ def _usage_context( # ------------------------------------------------------------------ ComponentMeta / LazyStore + @dataclass class ComponentMeta: - id: str; name: str; component_type: str; file_path: str; relative_path: str - start_line: int = 0; end_line: int = 0; language: str = "" + id: str + name: str + component_type: str + file_path: str + relative_path: str + start_line: int = 0 + end_line: int = 0 + language: str = "" depends_on: Set[str] = field(default_factory=set) - node_type: Optional[str] = None; base_classes: Optional[List[str]] = None - class_name: Optional[str] = None; display_name: Optional[str] = None - qualified_name: Optional[str] = None; has_docstring: bool = False + node_type: Optional[str] = None + base_classes: Optional[List[str]] = None + class_name: Optional[str] = None + display_name: Optional[str] = None + qualified_name: Optional[str] = None + has_docstring: bool = False parameters: Optional[List[str]] = None def to_node(self, source_code: str = "", docstring: str = "") -> Node: return Node( - id=self.id, name=self.name, component_type=self.component_type, - file_path=self.file_path, relative_path=self.relative_path, - start_line=self.start_line, end_line=self.end_line, - language=self.language, depends_on=self.depends_on, - node_type=self.node_type, base_classes=self.base_classes, - class_name=self.class_name, display_name=self.display_name, - qualified_name=self.qualified_name, has_docstring=self.has_docstring, - parameters=self.parameters, source_code=source_code, docstring=docstring) + id=self.id, + name=self.name, + component_type=self.component_type, + file_path=self.file_path, + relative_path=self.relative_path, + start_line=self.start_line, + end_line=self.end_line, + language=self.language, + depends_on=self.depends_on, + node_type=self.node_type, + base_classes=self.base_classes, + class_name=self.class_name, + display_name=self.display_name, + qualified_name=self.qualified_name, + has_docstring=self.has_docstring, + parameters=self.parameters, + source_code=source_code, + docstring=docstring, + ) class LazyComponentStore: def __init__(self, cache, metas: Dict[str, ComponentMeta], lru_size=_DEFAULT_LRU_SIZE): - self._cache = cache; self._metas = metas - self._lru: OrderedDict[str, Node] = OrderedDict(); self._lru_size = lru_size + self._cache = cache + self._metas = metas + self._lru: OrderedDict[str, Node] = OrderedDict() + self._lru_size = lru_size def __getitem__(self, k: str) -> Node: - if k in self._lru: n = self._lru.pop(k); self._lru[k] = n; return n + if k in self._lru: + n = self._lru.pop(k) + self._lru[k] = n + return n n = self._cache.get_component(k) - if n is None: raise KeyError(k) + if n is None: + raise KeyError(k) self._lru[k] = n - if len(self._lru) > self._lru_size: self._lru.popitem(last=False) + if len(self._lru) > self._lru_size: + self._lru.popitem(last=False) return n - def __contains__(self, k): return k in self._metas - def __len__(self): return len(self._metas) - def __iter__(self): return iter(self._metas) + + def __contains__(self, k): + return k in self._metas + + def __len__(self): + return len(self._metas) + + def __iter__(self): + return iter(self._metas) + def get(self, k, d=None): - try: return self[k] - except KeyError: return d - def items(self): return self._metas.items() - def keys(self): return self._metas.keys() - def values(self): return self._metas.values() - def meta(self, k) -> Optional[ComponentMeta]: return self._metas.get(k) - def invalidate(self, k): self._lru.pop(k, None) + try: + return self[k] + except KeyError: + return d + + def items(self): + return self._metas.items() + + def keys(self): + return self._metas.keys() + + def values(self): + return self._metas.values() + + def meta(self, k) -> Optional[ComponentMeta]: + return self._metas.get(k) + + def invalidate(self, k): + self._lru.pop(k, None) + # ------------------------------------------------------------------ AnalysisCache + class AnalysisCache: def __init__(self, repo_path: Path, db_path: Optional[Path] = None): self.repo_path = Path(repo_path).resolve() - self.db_path = (Path(db_path) if db_path else self.repo_path / _CACHE_DIR / _DB_FILENAME) + self.db_path = Path(db_path) if db_path else self.repo_path / _CACHE_DIR / _DB_FILENAME self.db_path.parent.mkdir(parents=True, exist_ok=True) self._conn: Optional[sqlite3.Connection] = None @@ -705,7 +910,8 @@ def _create_tables(self): # Migration: add authority column for existing databases (P0 authority ranking) try: self.conn.execute( - "ALTER TABLE search_index ADD COLUMN authority REAL NOT NULL DEFAULT 1.0") + "ALTER TABLE search_index ADD COLUMN authority REAL NOT NULL DEFAULT 1.0" + ) self.conn.commit() except Exception: pass # Column already exists @@ -715,19 +921,32 @@ def _create_tables(self): def _mget(self, k: str, d: str = "") -> str: r = self.conn.execute("SELECT value FROM repo_meta WHERE key=?", (k,)).fetchone() return r["value"] if r else d + def _mset(self, k: str, v: str): - self.conn.execute("INSERT OR REPLACE INTO repo_meta VALUES(?,?)", (k, v)); self.conn.commit() + self.conn.execute("INSERT OR REPLACE INTO repo_meta VALUES(?,?)", (k, v)) + self.conn.commit() def get_last_commit_id(self) -> Optional[str]: - cid = self._mget("last_commit_id"); return cid if cid else None - def set_last_commit_id(self, cid: str): self._mset("last_commit_id", cid) + cid = self._mget("last_commit_id") + return cid if cid else None + + def set_last_commit_id(self, cid: str): + self._mset("last_commit_id", cid) + def get_output_dir(self) -> Optional[str]: """Return the output_dir recorded by the last analyze_repo, if any.""" - od = self._mget("output_dir"); return od if od else None - def set_output_dir(self, od: str): self._mset("output_dir", od) + od = self._mget("output_dir") + return od if od else None + + def set_output_dir(self, od: str): + self._mset("output_dir", od) + def get_component_count(self) -> int: - r = self.conn.execute("SELECT COUNT(*) as c FROM components").fetchone(); return r["c"] if r else 0 - def is_fresh(self) -> bool: return self.get_component_count() > 0 + r = self.conn.execute("SELECT COUNT(*) as c FROM components").fetchone() + return r["c"] if r else 0 + + def is_fresh(self) -> bool: + return self.get_component_count() > 0 # -- symbol map -- @@ -799,19 +1018,21 @@ def get_all_routes(self) -> List[Dict]: extra = json.loads(r["extra_json"]) if r["extra_json"] else {} except Exception: pass - result.append({ - "route_key": r["route_key"], - "protocol": r["protocol"], - "method": r["method"], - "path": r["path"], - "role": r["role"], - "component_id": r["component_id"], - "repo_name": r["repo_name"], - "file_path": r["file_path"], - "line_number": r["line_number"], - "framework": r["framework"], - "extra": extra, - }) + result.append( + { + "route_key": r["route_key"], + "protocol": r["protocol"], + "method": r["method"], + "path": r["path"], + "role": r["role"], + "component_id": r["component_id"], + "repo_name": r["repo_name"], + "file_path": r["file_path"], + "line_number": r["line_number"], + "framework": r["framework"], + "extra": extra, + } + ) return result def get_routes_by_role(self, role: str) -> List[Dict]: @@ -819,7 +1040,8 @@ def get_routes_by_role(self, role: str) -> List[Dict]: rows = self.conn.execute( "SELECT route_key, protocol, method, path, role, component_id, " "repo_name, file_path, line_number, framework, extra_json " - "FROM routes WHERE role=?", (role,), + "FROM routes WHERE role=?", + (role,), ).fetchall() result: List[Dict] = [] for r in rows: @@ -828,19 +1050,21 @@ def get_routes_by_role(self, role: str) -> List[Dict]: extra = json.loads(r["extra_json"]) if r["extra_json"] else {} except Exception: pass - result.append({ - "route_key": r["route_key"], - "protocol": r["protocol"], - "method": r["method"], - "path": r["path"], - "role": r["role"], - "component_id": r["component_id"], - "repo_name": r["repo_name"], - "file_path": r["file_path"], - "line_number": r["line_number"], - "framework": r["framework"], - "extra": extra, - }) + result.append( + { + "route_key": r["route_key"], + "protocol": r["protocol"], + "method": r["method"], + "path": r["path"], + "role": r["role"], + "component_id": r["component_id"], + "repo_name": r["repo_name"], + "file_path": r["file_path"], + "line_number": r["line_number"], + "framework": r["framework"], + "extra": extra, + } + ) return result def remove_routes_by_file(self, fp: str) -> int: @@ -864,20 +1088,36 @@ def remove_routes_by_file(self, fp: str) -> int: def get_component(self, cid: str) -> Optional[Node]: r = self.conn.execute("SELECT * FROM components WHERE id=?", (cid,)).fetchone() - if not r: return None + if not r: + return None extra = _parse_row(r) - return Node(id=r["id"], name=r["name"], component_type=r["component_type"], - file_path=r["file_path"], relative_path=r["relative_path"], - start_line=r["start_line"], end_line=r["end_line"], - language=r["language"], depends_on=extra[0], node_type=r["node_type"], - base_classes=extra[1], class_name=r["class_name"], - display_name=r["display_name"], qualified_name=r["qualified_name"], - has_docstring=bool(r["has_docstring"]), docstring=r["docstring"] or "", - parameters=extra[2], source_code="") - - def batch_insert_components(self, components: Dict[str, Node], - leaf_nodes: Optional[List[str]] = None, - incremental: bool = False): + return Node( + id=r["id"], + name=r["name"], + component_type=r["component_type"], + file_path=r["file_path"], + relative_path=r["relative_path"], + start_line=r["start_line"], + end_line=r["end_line"], + language=r["language"], + depends_on=extra[0], + node_type=r["node_type"], + base_classes=extra[1], + class_name=r["class_name"], + display_name=r["display_name"], + qualified_name=r["qualified_name"], + has_docstring=bool(r["has_docstring"]), + docstring=r["docstring"] or "", + parameters=extra[2], + source_code="", + ) + + def batch_insert_components( + self, + components: Dict[str, Node], + leaf_nodes: Optional[List[str]] = None, + incremental: bool = False, + ): if not components: return c = self.conn @@ -913,13 +1153,21 @@ def _comp_hash(n: Node) -> str: rows = [ ( - n.id, n.name, n.component_type, n.file_path, n.relative_path, - n.start_line, n.end_line, + n.id, + n.name, + n.component_type, + n.file_path, + n.relative_path, + n.start_line, + n.end_line, (n.language or "").strip() or "unknown", n.node_type, json.dumps(n.base_classes) if n.base_classes else None, - n.class_name, n.display_name, n.qualified_name, - 1 if n.has_docstring else 0, n.docstring or "", + n.class_name, + n.display_name, + n.qualified_name, + 1 if n.has_docstring else 0, + n.docstring or "", json.dumps(n.parameters) if n.parameters else None, json.dumps(sorted(n.depends_on)) if n.depends_on else "[]", "{}", @@ -970,8 +1218,12 @@ def _comp_hash(n: Node) -> str: (json.dumps(leaf_nodes),), ) self.conn.commit() - logger.info("Cached %d components (%s), %d dep edges", - len(components), "incremental" if incremental else "full", len(deps)) + logger.info( + "Cached %d components (%s), %d dep edges", + len(components), + "incremental" if incremental else "full", + len(deps), + ) def get_stale_components(self, new_components: Dict[str, "Node"]) -> Dict[str, List[str]]: """Compare incoming components against stored content hashes. @@ -1009,8 +1261,12 @@ def _hash_node(n) -> str: deleted = [cid for cid in stored if cid not in new_ids] if added or modified or deleted: - logger.info("Stale detection: %d added, %d modified, %d deleted", - len(added), len(modified), len(deleted)) + logger.info( + "Stale detection: %d added, %d modified, %d deleted", + len(added), + len(modified), + len(deleted), + ) return {"added": added, "modified": modified, "deleted": deleted} def get_leaf_nodes(self) -> List[str]: @@ -1021,18 +1277,29 @@ def get_all_metas(self) -> Dict[str, ComponentMeta]: rows = self.conn.execute( "SELECT id,name,component_type,file_path,relative_path,start_line,end_line," "language,node_type,base_classes,class_name,display_name,qualified_name," - "has_docstring,parameters,depends_on FROM components").fetchall() + "has_docstring,parameters,depends_on FROM components" + ).fetchall() out: Dict[str, ComponentMeta] = {} for r in rows: extra = _parse_row(r) out[r["id"]] = ComponentMeta( - id=r["id"], name=r["name"], component_type=r["component_type"], - file_path=r["file_path"], relative_path=r["relative_path"], - start_line=r["start_line"], end_line=r["end_line"], - language=r["language"] or "", depends_on=extra[0], node_type=r["node_type"], - base_classes=extra[1], class_name=r["class_name"], - display_name=r["display_name"], qualified_name=r["qualified_name"], - has_docstring=bool(r["has_docstring"]), parameters=extra[2]) + id=r["id"], + name=r["name"], + component_type=r["component_type"], + file_path=r["file_path"], + relative_path=r["relative_path"], + start_line=r["start_line"], + end_line=r["end_line"], + language=r["language"] or "", + depends_on=extra[0], + node_type=r["node_type"], + base_classes=extra[1], + class_name=r["class_name"], + display_name=r["display_name"], + qualified_name=r["qualified_name"], + has_docstring=bool(r["has_docstring"]), + parameters=extra[2], + ) return out def remove_by_file(self, fp: str) -> int: @@ -1081,9 +1348,7 @@ def remove_by_file(self, fp: str) -> int: if ids: for chunk in _sql_chunks(ids): ph = ",".join("?" * len(chunk)) - self.conn.execute( - f"DELETE FROM components WHERE id IN ({ph})", chunk - ) + self.conn.execute(f"DELETE FROM components WHERE id IN ({ph})", chunk) self.conn.execute( f"DELETE FROM dependencies WHERE source_id IN ({ph}) OR target_id IN ({ph})", chunk + chunk, @@ -1095,22 +1360,37 @@ def remove_by_file(self, fp: str) -> int: def get_depends_on(self, cid: str) -> List[str]: r = self.conn.execute("SELECT depends_on FROM components WHERE id=?", (cid,)).fetchone() - if not r: return [] - try: return json.loads(r["depends_on"]) or [] - except Exception: return [] + if not r: + return [] + try: + return json.loads(r["depends_on"]) or [] + except Exception: + return [] def get_depended_by(self, cid: str) -> List[str]: - return [r["source_id"] for r in self.conn.execute( - "SELECT source_id FROM dependencies WHERE target_id=?", (cid,)).fetchall()] + return [ + r["source_id"] + for r in self.conn.execute( + "SELECT source_id FROM dependencies WHERE target_id=?", (cid,) + ).fetchall() + ] def get_all_deps(self, direction="both") -> List[Dict[str, str]]: res = [] if direction in ("depends_on", "both"): - for r in self.conn.execute("SELECT source_id,target_id FROM dependencies ORDER BY source_id"): - res.append({"source": r["source_id"], "target": r["target_id"], "direction": "depends_on"}) + for r in self.conn.execute( + "SELECT source_id,target_id FROM dependencies ORDER BY source_id" + ): + res.append( + {"source": r["source_id"], "target": r["target_id"], "direction": "depends_on"} + ) if direction in ("depended_by", "both"): - for r in self.conn.execute("SELECT target_id,source_id FROM dependencies ORDER BY target_id"): - res.append({"source": r["target_id"], "target": r["source_id"], "direction": "depended_by"}) + for r in self.conn.execute( + "SELECT target_id,source_id FROM dependencies ORDER BY target_id" + ): + res.append( + {"source": r["target_id"], "target": r["source_id"], "direction": "depended_by"} + ) return res # -- file fingerprints -- @@ -1124,12 +1404,16 @@ def _hash_file(self, rel: str) -> Optional[Tuple[float, int, str]]: head = f.read(65536) h = hashlib.sha256(head).hexdigest() return s.st_mtime, s.st_size, h - except OSError: return None + except OSError: + return None def update_file_fingerprints(self, paths: List[str], commit_id=""): rows = [(p, *f, commit_id) for p in paths if (f := self._hash_file(p))] - if rows: self.conn.executemany( - "INSERT OR REPLACE INTO file_fingerprints VALUES(?,?,?,?,?)", rows); self.conn.commit() + if rows: + self.conn.executemany( + "INSERT OR REPLACE INTO file_fingerprints VALUES(?,?,?,?,?)", rows + ) + self.conn.commit() def remove_file_fingerprints(self, paths: List[str]): """Drop fingerprint rows for files that no longer exist on disk. @@ -1142,21 +1426,23 @@ def remove_file_fingerprints(self, paths: List[str]): return for chunk in _sql_chunks(paths): ph = ",".join("?" * len(chunk)) - self.conn.execute( - f"DELETE FROM file_fingerprints WHERE file_path IN ({ph})", chunk - ) + self.conn.execute(f"DELETE FROM file_fingerprints WHERE file_path IN ({ph})", chunk) self.conn.commit() def get_all_fingerprints(self) -> Dict[str, Dict[str, Any]]: - return {r["file_path"]: dict(mtime=r["mtime"], size=r["size"], - content_hash=r["content_hash"], commit_id=r["commit_id"]) - for r in self.conn.execute("SELECT * FROM file_fingerprints").fetchall()} + return { + r["file_path"]: dict( + mtime=r["mtime"], + size=r["size"], + content_hash=r["content_hash"], + commit_id=r["commit_id"], + ) + for r in self.conn.execute("SELECT * FROM file_fingerprints").fetchall() + } def get_cached_file_paths(self) -> Set[str]: """Return set of all file paths that have cached components.""" - rows = self.conn.execute( - "SELECT DISTINCT relative_path FROM components" - ).fetchall() + rows = self.conn.execute("SELECT DISTINCT relative_path FROM components").fetchall() return {r["relative_path"] for r in rows} def get_components_by_files(self, file_paths: Set[str]) -> Dict[str, Node]: @@ -1167,171 +1453,310 @@ def get_components_by_files(self, file_paths: Set[str]) -> Dict[str, Node]: norm_paths = [fp.replace("\\", "/") for fp in file_paths] for chunk in _sql_chunks(norm_paths): ph = ",".join("?" * len(chunk)) - rows.extend(self.conn.execute( - f"SELECT * FROM components WHERE replace(relative_path, '\\', '/') IN ({ph})", - chunk, - ).fetchall()) + rows.extend( + self.conn.execute( + f"SELECT * FROM components WHERE replace(relative_path, '\\', '/') IN ({ph})", + chunk, + ).fetchall() + ) result: Dict[str, Node] = {} for r in rows: extra = _parse_row(r) result[r["id"]] = Node( - id=r["id"], name=r["name"], component_type=r["component_type"], - file_path=r["file_path"], relative_path=r["relative_path"], - start_line=r["start_line"], end_line=r["end_line"], - language=r["language"], depends_on=extra[0], node_type=r["node_type"], - base_classes=extra[1], class_name=r["class_name"], - display_name=r["display_name"], qualified_name=r["qualified_name"], - has_docstring=bool(r["has_docstring"]), docstring=r["docstring"] or "", - parameters=extra[2], source_code="", + id=r["id"], + name=r["name"], + component_type=r["component_type"], + file_path=r["file_path"], + relative_path=r["relative_path"], + start_line=r["start_line"], + end_line=r["end_line"], + language=r["language"], + depends_on=extra[0], + node_type=r["node_type"], + base_classes=extra[1], + class_name=r["class_name"], + display_name=r["display_name"], + qualified_name=r["qualified_name"], + has_docstring=bool(r["has_docstring"]), + docstring=r["docstring"] or "", + parameters=extra[2], + source_code="", ) return result # -- git change detection -- - _SRC_EXTS = {".py", ".pyx", ".java", ".js", ".jsx", ".ts", ".tsx", ".c", ".h", - ".cpp", ".hpp", ".cc", ".hh", ".cs", ".kt", ".kts", ".go", ".php"} + _SRC_EXTS = { + ".py", + ".pyx", + ".java", + ".js", + ".jsx", + ".ts", + ".tsx", + ".c", + ".h", + ".cpp", + ".hpp", + ".cc", + ".hh", + ".cs", + ".kt", + ".kts", + ".go", + ".php", + } def detect_changes(self) -> Optional[Dict[str, Any]]: - ch = self._git_detect(); return ch if ch is not None else self._fp_detect() + ch = self._git_detect() + return ch if ch is not None else self._fp_detect() def _git_detect(self) -> Optional[Dict[str, Any]]: - try: import git; repo = git.Repo(self.repo_path, search_parent_directories=True) - except Exception: return None + try: + import git + + repo = git.Repo(self.repo_path, search_parent_directories=True) + except Exception: + return None prev = self.get_last_commit_id() - if not prev: return None - try: cur = repo.head.commit.hexsha - except Exception: return None + if not prev: + return None + try: + cur = repo.head.commit.hexsha + except Exception: + return None git_root = Path(repo.working_dir).resolve() - try: sp = self.repo_path.resolve().relative_to(git_root).as_posix() - except ValueError: sp = "" - if sp == ".": sp = "" + try: + sp = self.repo_path.resolve().relative_to(git_root).as_posix() + except ValueError: + sp = "" + if sp == ".": + sp = "" def _n(p: str) -> Optional[str]: - if sp and not p.startswith(sp + "/"): return None - p = p[len(sp)+1:] if sp else p + if sp and not p.startswith(sp + "/"): + return None + p = p[len(sp) + 1 :] if sp else p return None if p.startswith(".codewiki/") else p ch, seen = [], set() - def add(r): - if r and (p := _n(r)) and p not in seen: ch.append(p); seen.add(p) + + def add(r): + if r and (p := _n(r)) and p not in seen: + ch.append(p) + seen.add(p) + if prev != cur: try: for d in repo.commit(prev).diff(cur): - add(d.a_path); add(d.b_path) + add(d.a_path) + add(d.b_path) except Exception: - logger.warning("Commit %s unreachable", prev); return None + logger.warning("Commit %s unreachable", prev) + return None try: for d in list(repo.index.diff("HEAD")) + list(repo.index.diff(None)): - add(d.a_path); add(d.b_path) - for item in repo.untracked_files: add(item) - except Exception: pass + add(d.a_path) + add(d.b_path) + for item in repo.untracked_files: + add(item) + except Exception: + pass return {"changed_files": ch, "method": "git", "current_commit": cur} def _fp_detect(self) -> Optional[Dict[str, Any]]: cached = self.get_all_fingerprints() - if not cached: return None + if not cached: + return None ch, existing = [], set() for dp, dns, fns in os.walk(str(self.repo_path)): - dns[:] = [d for d in dns if not d.startswith(".") and d not in ("node_modules","__pycache__","venv",".venv")] + dns[:] = [ + d + for d in dns + if not d.startswith(".") + and d not in ("node_modules", "__pycache__", "venv", ".venv") + ] rd = Path(dp).relative_to(self.repo_path) for fn in fns: - if Path(fn).suffix.lower() not in self._SRC_EXTS: continue - rp = (rd / fn).as_posix() if rd != Path(".") else fn; existing.add(rp) - cfp = self._hash_file(rp); prev = cached.get(rp) + if Path(fn).suffix.lower() not in self._SRC_EXTS: + continue + rp = (rd / fn).as_posix() if rd != Path(".") else fn + existing.add(rp) + cfp = self._hash_file(rp) + prev = cached.get(rp) if cfp is None: - if prev is not None: ch.append(rp); continue - if prev is None: ch.append(rp); continue - if abs(cfp[0] - prev["mtime"]) > 1.0 or cfp[1] != prev["size"] or cfp[2] != prev["content_hash"]: + if prev is not None: + ch.append(rp) + continue + if prev is None: + ch.append(rp) + continue + if ( + abs(cfp[0] - prev["mtime"]) > 1.0 + or cfp[1] != prev["size"] + or cfp[2] != prev["content_hash"] + ): ch.append(rp) for cp in cached: - if cp not in existing: ch.append(cp) - return {"changed_files": ch, "method": "fingerprint", - "no_changes": True} if not ch else {"changed_files": ch, "method": "fingerprint"} + if cp not in existing: + ch.append(cp) + return ( + {"changed_files": ch, "method": "fingerprint", "no_changes": True} + if not ch + else {"changed_files": ch, "method": "fingerprint"} + ) # -- BM25 search -- # (tokeniser, stopwords and snippet extractor are now module-level; # see _tokenize, _STOPWORDS, _extract_snippet above) def build_search_index(self, output_dir: Path) -> Dict[str, Any]: - od = Path(output_dir); c = self.conn - c.execute("DELETE FROM search_index"); c.execute("DELETE FROM search_token_index") + od = Path(output_dir) + c = self.conn + c.execute("DELETE FROM search_index") + c.execute("DELETE FROM search_token_index") c.execute("DELETE FROM search_stats") from codewiki.src.config import WIKI_SYSTEM_FILES, WIKI_DIR + dc = nc = sc = 0 # Scan wiki/ subdirectories recursively for doc pages wiki_dir = od / WIKI_DIR if wiki_dir.is_dir(): for md in sorted(wiki_dir.rglob("*.md")): - if not md.is_file(): continue - if md.name in WIKI_SYSTEM_FILES: continue - try: ct = md.read_text(encoding="utf-8", errors="replace") - except OSError: continue - if ". " - "Declared docs earn adoption credit which boosts their future " - "ranking (usage.adopted_count)." - ), - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "query": query, + "keywords": keywords, + "search_method": search_method, + **({"query_coverage": coverage} if coverage else {}), + **({"budget_degraded": degraded_count} if degraded_count else {}), + "results": results, + "context_package": context_package, + # P1 A-line: adoption convention reminder — a lower-bound usefulness + # signal. Agents that actually use a result should declare it. + "adoption_hint": ( + "If you actually used any result above, include this single-line " + "comment in your final reply (paths exactly as returned): " + '. ' + "Declared docs earn adoption credit which boosts their future " + "ranking (usage.adopted_count)." + ), + }, + indent=2, + ensure_ascii=False, + ) # ------------------------------------------------------------------ @@ -1840,9 +1944,7 @@ def handle_query_wiki( # ------------------------------------------------------------------ -def _record_retrieval_stats( - output_dir: Path, query: str, results: List[Dict[str, Any]] -) -> None: +def _record_retrieval_stats(output_dir: Path, query: str, results: List[Dict[str, Any]]) -> None: """Record which files were returned by a query_wiki call. T2 (docs/团队知识库支持优化设计方案.md §4.2): the SQLite @@ -1859,6 +1961,7 @@ def _record_retrieval_stats( return try: from codewiki.mcp.tools import telemetry + for r in results: # Prefer 'file' field (relative path); fall back to 'title' file_path = r.get("file") or r.get("title") or r.get("path", "") @@ -1882,6 +1985,7 @@ def handle_wiki_stats( file system to find documents that were never retrieved). """ from codewiki.mcp.tools.workspace_result import resolve_session + session = resolve_session(arguments, store) od = arguments.get("output_dir") @@ -1892,16 +1996,18 @@ def handle_wiki_stats( else: rp = arguments.get("repo_path") if rp: - output_dir = (Path(rp).expanduser().resolve() / "repowiki") + output_dir = Path(rp).expanduser().resolve() / "repowiki" else: return json.dumps({"error": "output_dir is required (or pass repo_path to derive it)."}) from codewiki.mcp.tools import telemetry + usage = telemetry.aggregate_usage(output_dir) if not usage: # P2: aggregation counters stay visible even before any query stats exist. try: from codewiki.mcp.tools import aggregation_state as agg + _agg = agg.aggregation_summary(output_dir) except Exception: _agg = None @@ -1910,12 +2016,14 @@ def handle_wiki_stats( _fresh = _freshness_distribution(output_dir) except Exception: _fresh = None - return json.dumps({ - "error": "No retrieval stats found. Run query_wiki first to generate stats.", - "telemetry_dir": str(output_dir / ".meta" / "telemetry"), - **({"aggregation": _agg} if _agg else {}), - **({"freshness": _fresh} if _fresh else {}), - }) + return json.dumps( + { + "error": "No retrieval stats found. Run query_wiki first to generate stats.", + "telemetry_dir": str(output_dir / ".meta" / "telemetry"), + **({"aggregation": _agg} if _agg else {}), + **({"freshness": _fresh} if _fresh else {}), + } + ) sort_by = arguments.get("sort_by", "hit_count") order = arguments.get("order", "desc") @@ -1941,23 +2049,24 @@ def _sort_value(fp: str): ) # total query count proxy: distinct days on which any hit event was # recorded (the exact query log is gone with the SQLite table). - total_queries = len(set().union( - *(e.get("hit_days") or set() for e in usage.values()) - )) if usage else 0 + total_queries = ( + len(set().union(*(e.get("hit_days") or set() for e in usage.values()))) if usage else 0 + ) stats = [] for fp in eligible[:limit]: e = usage[fp] - stats.append({ - "file_path": fp, - "hit_count": int(e.get("hits", 0)), - "last_hit": e.get("last_hit"), - "first_hit": e.get("first_hit"), - "hit_rate": ( - round(int(e.get("hits", 0)) / total_queries, 4) - if total_queries > 0 else 0 - ), - }) + stats.append( + { + "file_path": fp, + "hit_count": int(e.get("hits", 0)), + "last_hit": e.get("last_hit"), + "first_hit": e.get("first_hit"), + "hit_rate": ( + round(int(e.get("hits", 0)) / total_queries, 4) if total_queries > 0 else 0 + ), + } + ) # Optionally include zero-hit documents (files on disk with no events) zero_hit = [] @@ -1980,6 +2089,7 @@ def _sort_value(fp: str): aggregation = None try: from codewiki.mcp.tools import aggregation_state as agg + aggregation = agg.aggregation_summary(output_dir) except Exception: pass @@ -2008,18 +2118,22 @@ def _sort_value(fp: str): except Exception: promotion = None - return json.dumps({ - "total_distinct_queries": total_queries, - "returned": len(stats), - "sort_by": sort_by, - "order": order, - "stats": stats, - **({"zero_hit_files": zero_hit} if include_zero_hit else {}), - **({"aggregation": aggregation} if aggregation else {}), - **({"freshness": freshness} if freshness else {}), - **({"cold_candidates": cold} if cold else {}), - **({"promotion_candidates": promotion} if promotion else {}), - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "total_distinct_queries": total_queries, + "returned": len(stats), + "sort_by": sort_by, + "order": order, + "stats": stats, + **({"zero_hit_files": zero_hit} if include_zero_hit else {}), + **({"aggregation": aggregation} if aggregation else {}), + **({"freshness": freshness} if freshness else {}), + **({"cold_candidates": cold} if cold else {}), + **({"promotion_candidates": promotion} if promotion else {}), + }, + indent=2, + ensure_ascii=False, + ) def _cold_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: @@ -2030,6 +2144,7 @@ def _cold_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: Returns None when no telemetry data exists or nothing is cold. """ from codewiki.mcp.tools import telemetry + usage = telemetry.aggregate_usage(output_dir) if not usage: return None @@ -2037,6 +2152,7 @@ def _cold_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: cold_days, cold_min_hits = 180, 3 try: from codewiki.mcp.tools.page_router import load_schema + schema = load_schema(str(output_dir)) or {} ur = (schema.get("conventions") or {}).get("usage_ranking") or {} cold_days = int(ur.get("cold_days", cold_days)) @@ -2061,12 +2177,14 @@ def _cold_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: except (TypeError, ValueError): continue if lh_dt < cutoff: - out.append({ - "file_path": fp, - "hit_count": hit_count, - "last_hit": last_hit, - "days_since_last_hit": (today - lh_dt).days, - }) + out.append( + { + "file_path": fp, + "hit_count": hit_count, + "last_hit": last_hit, + "days_since_last_hit": (today - lh_dt).days, + } + ) out.sort(key=lambda x: -x["days_since_last_hit"]) return out @@ -2083,9 +2201,9 @@ def _cold_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: DEFAULT_NOTE_TYPES as _NT_TABLE, ) -_PROMOTION_PAGE_TYPES.update({ - t: str(spec.get("promote_to") or "") for t, spec in _NT_TABLE.items() -}) +_PROMOTION_PAGE_TYPES.update( + {t: str(spec.get("promote_to") or "") for t, spec in _NT_TABLE.items()} +) def _note_age_days(fm: Dict[str, Any], today: datetime) -> int: @@ -2137,6 +2255,7 @@ def _promotion_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: min_adopted, min_age_days = 3, 14 try: from codewiki.mcp.tools.page_router import load_schema + schema = load_schema(str(output_dir)) or {} promo = (schema.get("conventions") or {}).get("promotion") or {} min_adopted = int(promo.get("min_adopted", min_adopted)) @@ -2147,6 +2266,7 @@ def _promotion_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: # A-line adoption counts: missing db/table → {} → nothing can qualify. try: from codewiki.mcp.tools.adoption import load_adoption_counts + adopted_counts = load_adoption_counts(Path(output_dir)) except Exception as e: logger.debug("promotion_candidates adoption load failed: %s", e) @@ -2177,14 +2297,16 @@ def _promotion_candidates(output_dir: Path) -> Optional[List[Dict[str, Any]]]: if age < min_age_days: continue note_type = str(fm.get("type", "")).strip().lower() - out.append({ - "file": rel_path, - "title": fm.get("title", note_file.stem), - "type": note_type, - "adopted_count": adopted, - "age_days": age, - "suggested_page_type": _PROMOTION_PAGE_TYPES.get(note_type, ""), - }) + out.append( + { + "file": rel_path, + "title": fm.get("title", note_file.stem), + "type": note_type, + "adopted_count": adopted, + "age_days": age, + "suggested_page_type": _PROMOTION_PAGE_TYPES.get(note_type, ""), + } + ) out.sort(key=lambda x: -x["adopted_count"]) return out @@ -2224,6 +2346,7 @@ def _legacy_keyword_search( else: # page_type filter: map to directory name for doc source matching from codewiki.src.config import PAGE_TYPE_DIRS + dir_name = PAGE_TYPE_DIRS.get(type_filter, type_filter + "s") allowed_sources = {"doc"} # will filter by path prefix below else: @@ -2235,6 +2358,7 @@ def _legacy_keyword_search( # --- Search docs (recursive: wiki/ subdirs + root level) --- from codewiki.src.config import WIKI_SYSTEM_FILES + for md_file in output_dir.rglob("*.md"): if not md_file.is_file(): continue @@ -2248,6 +2372,7 @@ def _legacy_keyword_search( # Type filter: if type_filter is a page_type, filter by directory if type_filter and type_filter not in ("doc", "note", "source"): from codewiki.src.config import PAGE_TYPE_DIRS + dir_name = PAGE_TYPE_DIRS.get(type_filter, type_filter + "s") if f"wiki/{dir_name}/" not in rel_path: continue @@ -2255,9 +2380,11 @@ def _legacy_keyword_search( # Match by: filename stem, path prefix, or path component (e.g. "modules", "notes") scope_norm = scope.lower().replace(" ", "_").rstrip("/") path_lower = rel_path.lower().replace("\\", "/") - if (file_stem.lower() != scope_norm - and not path_lower.startswith(scope_norm + "/") - and f"/{scope_norm}/" not in f"/{path_lower}"): + if ( + file_stem.lower() != scope_norm + and not path_lower.startswith(scope_norm + "/") + and f"/{scope_norm}/" not in f"/{path_lower}" + ): continue try: content = md_file.read_text(encoding="utf-8") @@ -2361,12 +2488,12 @@ def _extract_frontmatter(content: str, key: str) -> Optional[str]: if line.startswith((" ", "\t")): stripped = line.lstrip() if stripped.startswith(f"{key}:"): - val = stripped[len(key) + 1:].strip().strip('"').strip("'") + val = stripped[len(key) + 1 :].strip().strip('"').strip("'") return val continue in_metadata = False # left the metadata block if line.startswith(f"{key}:"): - val = line[len(key) + 1:].strip().strip('"').strip("'") + val = line[len(key) + 1 :].strip().strip('"').strip("'") return val except (ValueError, IndexError): pass diff --git a/codewiki/mcp/tools/legacy_tools.py b/codewiki/mcp/tools/legacy_tools.py index 9d7d3b1..f94e21b 100644 --- a/codewiki/mcp/tools/legacy_tools.py +++ b/codewiki/mcp/tools/legacy_tools.py @@ -11,7 +11,6 @@ import json import logging from pathlib import Path -from typing import Any from mcp.types import Tool @@ -90,14 +89,14 @@ # Private helpers # ------------------------------------------------------------------ + def _load_config(): """Load CodeWiki configuration from ~/.codewiki/config.json + keyring.""" from codewiki.cli.config_manager import ConfigManager + manager = ConfigManager() if not manager.load(): - raise RuntimeError( - "CodeWiki not configured. Run 'codewiki config set' first." - ) + raise RuntimeError("CodeWiki not configured. Run 'codewiki config set' first.") return manager @@ -105,6 +104,7 @@ def _load_config(): # Handlers # ------------------------------------------------------------------ + async def handle_generate_docs(arguments: dict) -> str: """Legacy generate_docs — requires CodeWiki LLM configuration.""" repo_path = Path(arguments["repo_path"]).expanduser().resolve() @@ -119,19 +119,27 @@ async def handle_generate_docs(arguments: dict) -> str: api_key = manager.get_api_key() from codewiki.src.be.backend import is_caw_provider + caw_mode = bool(config) and is_caw_provider(getattr(config, "provider", "")) if not api_key and not caw_mode: - return json.dumps({"error": "API key not configured. Run 'codewiki config set --api-key '"}) + return json.dumps( + {"error": "API key not configured. Run 'codewiki config set --api-key '"} + ) agent_instructions = {} if arguments.get("doc_type"): agent_instructions["doc_type"] = arguments["doc_type"] if arguments.get("include_patterns"): - agent_instructions["include_patterns"] = [p.strip() for p in arguments["include_patterns"].split(",")] + agent_instructions["include_patterns"] = [ + p.strip() for p in arguments["include_patterns"].split(",") + ] if arguments.get("exclude_patterns"): - agent_instructions["exclude_patterns"] = [p.strip() for p in arguments["exclude_patterns"].split(",")] + agent_instructions["exclude_patterns"] = [ + p.strip() for p in arguments["exclude_patterns"].split(",") + ] from codewiki.src.config import Config as BackendConfig, set_cli_context + set_cli_context(True) backend_config = BackendConfig.from_cli( @@ -150,7 +158,10 @@ async def handle_generate_docs(arguments: dict) -> str: from codewiki.cli.utils.repo_validator import get_git_commit_hash from codewiki.src.be.documentation_generator import DocumentationGenerator - doc_gen = DocumentationGenerator(backend_config, commit_id=get_git_commit_hash(repo_path) or None) + + doc_gen = DocumentationGenerator( + backend_config, commit_id=get_git_commit_hash(repo_path) or None + ) await doc_gen.run() generated_files = [] @@ -174,11 +185,14 @@ async def handle_get_module_tree(arguments: dict, store=None) -> str: output_dir = raw_od.resolve() if raw_od.is_absolute() else (repo_path / raw_od).resolve() from codewiki.src.config import meta_resolve + module_tree_path = Path(meta_resolve(output_dir, "module_tree.json")) if not module_tree_path.exists(): - return json.dumps({ - "error": f"Module tree not found at {module_tree_path}. Run 'codewiki generate' first." - }) + return json.dumps( + { + "error": f"Module tree not found at {module_tree_path}. Run 'codewiki generate' first." + } + ) module_tree = json.loads(module_tree_path.read_text(encoding="utf-8")) diff --git a/codewiki/mcp/tools/note_consolidation.py b/codewiki/mcp/tools/note_consolidation.py index b0b5255..bdb5326 100644 --- a/codewiki/mcp/tools/note_consolidation.py +++ b/codewiki/mcp/tools/note_consolidation.py @@ -112,6 +112,7 @@ def _read_frontmatter(path: Path) -> Optional[Dict[str, Any]]: return None try: import yaml + data = yaml.safe_load(text[3:end]) return data if isinstance(data, dict) else None except Exception: @@ -126,7 +127,7 @@ def _read_body(path: Path) -> str: if text.startswith("---"): end = text.find("---", 3) if end >= 0: - return text[end + 3:].strip() + return text[end + 3 :].strip() return text.strip() @@ -143,6 +144,7 @@ def _update_frontmatter_meta(path: Path, updates: Dict[str, Any]) -> bool: return False try: import yaml + data = yaml.safe_load(text[3:end]) if not isinstance(data, dict): return False @@ -151,9 +153,8 @@ def _update_frontmatter_meta(path: Path, updates: Dict[str, Any]) -> bool: meta = {} meta.update(updates) data["metadata"] = meta - new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, - default_flow_style=False) - path.write_text(f"---\n{new_fm}---{text[end + 3:]}", encoding="utf-8") + new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False) + path.write_text(f"---\n{new_fm}---{text[end + 3 :]}", encoding="utf-8") return True except Exception as e: logger.warning("frontmatter update failed for %s: %s", path, e) @@ -173,6 +174,7 @@ def _append_meta_list(path: Path, key: str, values: List[str]) -> bool: return False try: import yaml + data = yaml.safe_load(text[3:end]) if not isinstance(data, dict): return False @@ -189,9 +191,8 @@ def _append_meta_list(path: Path, key: str, values: List[str]) -> bool: existing.append(v) meta[key] = existing data["metadata"] = meta - new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, - default_flow_style=False) - path.write_text(f"---\n{new_fm}---{text[end + 3:]}", encoding="utf-8") + new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False) + path.write_text(f"---\n{new_fm}---{text[end + 3 :]}", encoding="utf-8") return True except Exception as e: logger.warning("frontmatter list append failed for %s: %s", path, e) @@ -207,6 +208,7 @@ def _norm_rel(p: str, output_dir: Path) -> str: # --------------------------------------------------------------------------- # def _scenarios_dir(output_dir: Path) -> Path: from codewiki.src.config import WIKI_DIR, PAGE_TYPE_DIRS + return Path(output_dir) / WIKI_DIR / PAGE_TYPE_DIRS["scenario"] @@ -225,19 +227,24 @@ def _scan_scenarios(output_dir: Path) -> List[Dict[str, Any]]: heat = int(meta.get("heat") or 0) except (TypeError, ValueError): heat = 0 - out.append({ - "file": _norm_rel(str(p.relative_to(output_dir)), output_dir), - "title": fm.get("title") or p.stem, - "summary": str(meta.get("summary") or "")[:_SUMMARY_CHARS], - "heat": heat, - "updated": str(fm.get("generated", {}).get("at", "")) if isinstance(fm.get("generated"), dict) else "", - }) + out.append( + { + "file": _norm_rel(str(p.relative_to(output_dir)), output_dir), + "title": fm.get("title") or p.stem, + "summary": str(meta.get("summary") or "")[:_SUMMARY_CHARS], + "heat": heat, + "updated": str(fm.get("generated", {}).get("at", "")) + if isinstance(fm.get("generated"), dict) + else "", + } + ) return out def _pending_confirmed_notes(output_dir: Path, limit: int) -> List[Dict[str, Any]]: """Stable notes not yet absorbed into a scene block (no consolidated_into).""" from codewiki.src.config import NOTES_DIR + notes_dir = Path(output_dir) / NOTES_DIR out: List[Dict[str, Any]] = [] if not notes_dir.is_dir(): @@ -254,14 +261,16 @@ def _pending_confirmed_notes(output_dir: Path, limit: int) -> List[Dict[str, Any scene = "" if isinstance(meta.get("scene"), str): scene = meta["scene"] - out.append({ - "file": _norm_rel(str(p.relative_to(output_dir)), output_dir), - "title": fm.get("title") or p.stem, - "note_type": fm.get("type") or "general", - "scene": scene, - "severity": str(meta.get("severity") or ""), - "preview": body[:_SUMMARY_CHARS], - }) + out.append( + { + "file": _norm_rel(str(p.relative_to(output_dir)), output_dir), + "title": fm.get("title") or p.stem, + "note_type": fm.get("type") or "general", + "scene": scene, + "severity": str(meta.get("severity") or ""), + "preview": body[:_SUMMARY_CHARS], + } + ) if len(out) >= limit: break return out @@ -288,6 +297,7 @@ def _cleanup_soft_deleted(output_dir: Path) -> List[str]: # --------------------------------------------------------------------------- # def _capacity(output_dir: Path, live_count: int) -> Dict[str, Any]: from codewiki.mcp.tools.aggregation_state import read_config + max_scenes = read_config(output_dir)["max_scenarios"] if live_count >= max_scenes: warning = "red" @@ -340,30 +350,36 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: pending = _pending_confirmed_notes(output_dir, limit) state = agg.load_state(output_dir) cfg = agg.read_config(output_dir) - return json.dumps({ - "status": "prepared", - "mode": "prepare", - "counters": { - "notes_since_last_consolidation": int(state.get("notes_since_last_consolidation") or 0), - "consolidation_threshold": cfg["consolidation_threshold"], - "last_consolidation_at": state.get("last_consolidation_at"), + return json.dumps( + { + "status": "prepared", + "mode": "prepare", + "counters": { + "notes_since_last_consolidation": int( + state.get("notes_since_last_consolidation") or 0 + ), + "consolidation_threshold": cfg["consolidation_threshold"], + "last_consolidation_at": state.get("last_consolidation_at"), + }, + "capacity": capacity, + "pending_notes": pending, + "pending_total": len(pending), + "scenarios_index": scenarios, + "system_prompt": _CONSOLIDATE_SYSTEM, + "next": ( + "(1) Read pending notes (view_repo_file) — metadata.scene groups " + "related ones; (2) read the scene files you plan to UPDATE/MERGE; " + "(3) write blocks with write_doc_file(page_type='scenario'); obey " + "the capacity warning (red=merge first, orange=update only); " + "(4) reject_note fully-absorbed source notes with " + "reason='consolidated into '; (5) submit the report. " + "If this consolidation was triggered by an aggregation_hint " + "reminder, confirm with the user before starting." + ), }, - "capacity": capacity, - "pending_notes": pending, - "pending_total": len(pending), - "scenarios_index": scenarios, - "system_prompt": _CONSOLIDATE_SYSTEM, - "next": ( - "(1) Read pending notes (view_repo_file) — metadata.scene groups " - "related ones; (2) read the scene files you plan to UPDATE/MERGE; " - "(3) write blocks with write_doc_file(page_type='scenario'); obey " - "the capacity warning (red=merge first, orange=update only); " - "(4) reject_note fully-absorbed source notes with " - "reason='consolidated into '; (5) submit the report. " - "If this consolidation was triggered by an aggregation_hint " - "reminder, confirm with the user before starting." - ), - }, indent=2, ensure_ascii=False) + indent=2, + ensure_ascii=False, + ) # ---- mode == "submit" ---- report = arguments.get("report") @@ -373,13 +389,15 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: except json.JSONDecodeError: return json.dumps({"error": "report must be a JSON object."}) if not isinstance(report, dict): - return json.dumps({ - "error": ( - "mode='submit' requires 'report': {scenarios: [{file, action, " - "source_notes, summary?, heat?}]} with action in " - "created|updated|merged|deleted." - ), - }) + return json.dumps( + { + "error": ( + "mode='submit' requires 'report': {scenarios: [{file, action, " + "source_notes, summary?, heat?}]} with action in " + "created|updated|merged|deleted." + ), + } + ) entries = report.get("scenarios") if not isinstance(entries, list) or not entries: return json.dumps({"error": "report.scenarios must be a non-empty list."}) @@ -409,10 +427,12 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: if action == "deleted": if path.is_file(): if _read_body(path) != _SOFT_DELETE_MARKER: - errors.append({ - "file": rel, - "error": "action=deleted requires the file body to be exactly [DELETED]", - }) + errors.append( + { + "file": rel, + "error": "action=deleted requires the file body to be exactly [DELETED]", + } + ) continue processed.append({"file": rel, "action": "deleted"}) continue @@ -422,23 +442,27 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: continue fm = _read_frontmatter(path) if fm is None or str(fm.get("type", "")).lower() != "scenario": - errors.append({ - "file": rel, - "error": "frontmatter must carry type: Scenario (write via write_doc_file page_type='scenario')", - }) + errors.append( + { + "file": rel, + "error": "frontmatter must carry type: Scenario (write via write_doc_file page_type='scenario')", + } + ) continue # Provenance: scene ← source notes (bidirectional links) source_notes = [ _norm_rel(str(s), output_dir) - for s in (entry.get("source_notes") or []) if str(s).strip() + for s in (entry.get("source_notes") or []) + if str(s).strip() ] if source_notes: _append_meta_list(path, "source_notes", source_notes) from codewiki.src.config import NOTES_DIR + notes_dir = Path(output_dir) / NOTES_DIR for src in source_notes: - note_path = (Path(output_dir) / src) + note_path = Path(output_dir) / src if note_path.is_file(): _append_meta_list(note_path, "consolidated_into", [rel]) elif (notes_dir / Path(src).name).is_file(): @@ -458,41 +482,51 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: if meta_updates: _update_frontmatter_meta(path, meta_updates) - processed.append({ - "file": rel, - "action": action, - "source_notes": len(source_notes), - }) + processed.append( + { + "file": rel, + "action": action, + "source_notes": len(source_notes), + } + ) if errors: - return json.dumps({ - "status": "error", - "mode": "submit", - "errors": errors, - "processed": processed, - "message": ( - f"{len(errors)} report entr(y/ies) failed validation; counters " - "NOT reset. Fix the reported issues and re-submit." - ), - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "error", + "mode": "submit", + "errors": errors, + "processed": processed, + "message": ( + f"{len(errors)} report entr(y/ies) failed validation; counters " + "NOT reset. Fix the reported issues and re-submit." + ), + }, + indent=2, + ensure_ascii=False, + ) # Soft-delete cleanup + capacity enforcement removed = _cleanup_soft_deleted(output_dir) live = _scan_scenarios(output_dir) capacity = _capacity(output_dir, len(live)) if capacity["warning"] == "red": - return json.dumps({ - "status": "capacity_exceeded", - "mode": "submit", - "processed": processed, - "removed_deleted": removed, - "capacity": capacity, - "message": ( - f"Scenario count {capacity['current']} exceeds the cap " - f"{capacity['max']}. MERGE similar scenes (and mark the losers " - "[DELETED]) until below the cap, then re-submit. Counters NOT reset." - ), - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "capacity_exceeded", + "mode": "submit", + "processed": processed, + "removed_deleted": removed, + "capacity": capacity, + "message": ( + f"Scenario count {capacity['current']} exceeds the cap " + f"{capacity['max']}. MERGE similar scenes (and mark the losers " + "[DELETED]) until below the cap, then re-submit. Counters NOT reset." + ), + }, + indent=2, + ensure_ascii=False, + ) state = agg.mark_consolidated(output_dir) @@ -520,24 +554,31 @@ def handle_consolidate_notes(arguments: Dict[str, Any], store: Any) -> str: # Rebuild the search index so scene blocks become queryable immediately. try: from codewiki.mcp.tools.wiki_search import build_full_index + build_full_index(output_dir) except Exception as e: # indexing is best-effort logger.warning("search index rebuild failed after consolidate: %s", e) - return json.dumps({ - "status": "completed", - "mode": "submit", - "processed": processed, - "removed_deleted": removed, - "capacity": capacity, - "counters": { - "notes_since_last_consolidation": int(state.get("notes_since_last_consolidation") or 0), - "notes_since_last_doctrine": int(state.get("notes_since_last_doctrine") or 0), + return json.dumps( + { + "status": "completed", + "mode": "submit", + "processed": processed, + "removed_deleted": removed, + "capacity": capacity, + "counters": { + "notes_since_last_consolidation": int( + state.get("notes_since_last_consolidation") or 0 + ), + "notes_since_last_doctrine": int(state.get("notes_since_last_doctrine") or 0), + }, + **({"doctrine_hint": doctrine_hint} if doctrine_hint else {}), + "message": ( + f"Consolidation recorded: {len(processed)} scene operation(s). " + "Counter reset. Confirm the new/updated scene blocks are reviewed; " + "source notes absorbed into scenes should be retired via reject_note." + ), }, - **({"doctrine_hint": doctrine_hint} if doctrine_hint else {}), - "message": ( - f"Consolidation recorded: {len(processed)} scene operation(s). " - "Counter reset. Confirm the new/updated scene blocks are reviewed; " - "source notes absorbed into scenes should be retired via reject_note." - ), - }, indent=2, ensure_ascii=False) + indent=2, + ensure_ascii=False, + ) diff --git a/codewiki/mcp/tools/note_merge.py b/codewiki/mcp/tools/note_merge.py index f8d7c3f..34b3bfa 100644 --- a/codewiki/mcp/tools/note_merge.py +++ b/codewiki/mcp/tools/note_merge.py @@ -38,7 +38,7 @@ def _split_fm(text: str) -> Tuple[Dict[str, str], str]: continue k, _, v = line.partition(":") fm[k.strip()] = v.strip().strip("'\"") - return fm, text[m.end():] + return fm, text[m.end() :] def _note_age_key(fm: Dict[str, str]) -> str: @@ -67,7 +67,11 @@ def _list_field(fm_text: str, key: str) -> List[str]: return out bl = re.search(rf"^{key}:\s*\n((?:\s+-\s+.*\n?)+)", block, re.MULTILINE) if bl: - out = [v.strip().lstrip("-").strip().strip("'\"") for v in bl.group(1).splitlines() if v.strip()] + out = [ + v.strip().lstrip("-").strip().strip("'\"") + for v in bl.group(1).splitlines() + if v.strip() + ] return out @@ -105,6 +109,7 @@ def merge_notes( note_type = (newest[2].get("type") or newest[2].get("note_type") or "general").lower() from codewiki.mcp.tools.note_types import merge_fields_for + strategies = merge_fields_for(note_type, schema) title = (new_title or newest[2].get("title") or newest[0].rsplit("/", 1)[-1]).strip("'\"") @@ -125,7 +130,9 @@ def merge_notes( tags_all.append(note_type) related = related_new if strategies.get("related_modules") == "replace" else related_all - tags = tags_all if strategies.get("tags", "union") == "union" else _list_field(newest[1], "tags") + tags = ( + tags_all if strategies.get("tags", "union") == "union" else _list_field(newest[1], "tags") + ) # body: append 策略——按树龄升序，每段带来源标记；replace 则只留最新正文。 body_parts: List[str] = [] @@ -160,6 +167,7 @@ def merge_notes( } if write: from codewiki.src.config import NOTES_DIR + notes_dir = od / NOTES_DIR notes_dir.mkdir(parents=True, exist_ok=True) out_path = notes_dir / f"{_slugify(title)}.md" diff --git a/codewiki/mcp/tools/note_types.py b/codewiki/mcp/tools/note_types.py index a2d95e6..d45ae1f 100644 --- a/codewiki/mcp/tools/note_types.py +++ b/codewiki/mcp/tools/note_types.py @@ -83,6 +83,7 @@ def _load_schema(output_dir: Optional[Path]) -> Optional[dict]: return None try: from codewiki.mcp.tools.page_router import load_schema + return load_schema(str(output_dir)) except Exception as e: # missing schema / parser absent — fall back to defaults logger.debug("note_types: schema load skipped (%s)", e) @@ -102,9 +103,7 @@ def load_note_types( """ if schema is None: schema = _load_schema(output_dir) - table: Dict[str, Dict[str, Any]] = { - t: dict(spec) for t, spec in DEFAULT_NOTE_TYPES.items() - } + table: Dict[str, Dict[str, Any]] = {t: dict(spec) for t, spec in DEFAULT_NOTE_TYPES.items()} conv = (schema or {}).get("conventions") or {} custom = conv.get("note_types") if isinstance(custom, dict): @@ -112,10 +111,16 @@ def load_note_types( t = str(raw_key).strip().lower() if not t: continue - base = dict(table.get(t, { - "freshness_days": 180, "promote_to": "", - "merge_fields": dict(_DEFAULT_MERGE_FIELDS), - })) + base = dict( + table.get( + t, + { + "freshness_days": 180, + "promote_to": "", + "merge_fields": dict(_DEFAULT_MERGE_FIELDS), + }, + ) + ) if isinstance(spec, dict): for k, v in spec.items(): base[k] = v @@ -123,9 +128,7 @@ def load_note_types( return table -def valid_note_types( - schema: Optional[dict] = None, output_dir: Optional[Path] = None -) -> Set[str]: +def valid_note_types(schema: Optional[dict] = None, output_dir: Optional[Path] = None) -> Set[str]: """Legal note_type values (MCP inputSchema enum source).""" return set(load_note_types(schema, output_dir)) @@ -182,13 +185,13 @@ def validate_note_types( """ if accepted is None: from codewiki.mcp.tools.distill_conversation import _VALID_NOTE_TYPES + accepted = set(_VALID_NOTE_TYPES) declared = set(load_note_types(schema, output_dir)) errors: List[str] = [] missing = accepted - declared if missing: errors.append( - "note_types: handler accepts %s but the table does not declare them" - % sorted(missing) + "note_types: handler accepts %s but the table does not declare them" % sorted(missing) ) return errors diff --git a/codewiki/mcp/tools/page_router.py b/codewiki/mcp/tools/page_router.py index 05b4e9b..a9781bb 100644 --- a/codewiki/mcp/tools/page_router.py +++ b/codewiki/mcp/tools/page_router.py @@ -20,7 +20,7 @@ import logging import os from pathlib import Path -from typing import Any, Dict, Optional +from typing import Dict import yaml @@ -83,6 +83,7 @@ def invalidate_schema_cache(output_dir: str | Path | None = None) -> None: # Path resolution # --------------------------------------------------------------------------- + def resolve_wiki_paths(output_dir: str | Path, schema: dict | None = None) -> dict: """Return a complete mapping of logical names to filesystem paths. @@ -101,18 +102,18 @@ def resolve_wiki_paths(output_dir: str | Path, schema: dict | None = None) -> di wiki = od / WIKI_DIR paths: Dict[str, Path] = { - "modules": wiki / PAGE_TYPE_DIRS["module"], - "entities": wiki / PAGE_TYPE_DIRS["entity"], - "concepts": wiki / PAGE_TYPE_DIRS["concept"], - "sources": wiki / PAGE_TYPE_DIRS["source"], - "comparisons": wiki / PAGE_TYPE_DIRS["comparison"], - "queries": wiki / PAGE_TYPE_DIRS["query"], - "notes": od / NOTES_DIR, - "raw_sources": od / RAW_SOURCES_DIR, - "index": wiki / INDEX_FILENAME, - "log": wiki / LOG_FILENAME, - "overview": wiki / OVERVIEW_FILENAME, - "schema": od / SCHEMA_FILENAME, + "modules": wiki / PAGE_TYPE_DIRS["module"], + "entities": wiki / PAGE_TYPE_DIRS["entity"], + "concepts": wiki / PAGE_TYPE_DIRS["concept"], + "sources": wiki / PAGE_TYPE_DIRS["source"], + "comparisons": wiki / PAGE_TYPE_DIRS["comparison"], + "queries": wiki / PAGE_TYPE_DIRS["query"], + "notes": od / NOTES_DIR, + "raw_sources": od / RAW_SOURCES_DIR, + "index": wiki / INDEX_FILENAME, + "log": wiki / LOG_FILENAME, + "overview": wiki / OVERVIEW_FILENAME, + "schema": od / SCHEMA_FILENAME, } # Allow schema.page_types to override directory names. @@ -285,6 +286,14 @@ def is_wiki_system_file(path: Path, output_dir: str | Path) -> bool: def ensure_wiki_dirs(output_dir: str | Path, schema: dict | None = None) -> None: """Create all wiki subdirectories if they don't exist yet.""" paths = resolve_wiki_paths(output_dir, schema) - for key in ("modules", "entities", "concepts", "sources", - "comparisons", "queries", "notes", "raw_sources"): + for key in ( + "modules", + "entities", + "concepts", + "sources", + "comparisons", + "queries", + "notes", + "raw_sources", + ): paths[key].mkdir(parents=True, exist_ok=True) diff --git a/codewiki/mcp/tools/prompt_server.py b/codewiki/mcp/tools/prompt_server.py index 532aeb1..6f74365 100644 --- a/codewiki/mcp/tools/prompt_server.py +++ b/codewiki/mcp/tools/prompt_server.py @@ -8,12 +8,13 @@ from __future__ import annotations -import json, logging +import json +import logging from pathlib import Path from typing import Any, Dict, Optional -from codewiki.mcp.session import SessionStore, SessionState -from codewiki.mcp.tools.workspace_result import write_result, _FILE_THRESHOLD +from codewiki.mcp.session import SessionStore +from codewiki.mcp.tools.workspace_result import _FILE_THRESHOLD logger = logging.getLogger(__name__) from codewiki.src.be.prompt_template import ( @@ -23,7 +24,6 @@ format_system_prompt, format_leaf_system_prompt, format_cluster_prompt, - format_user_prompt, ) @@ -41,6 +41,7 @@ def _build_schema_constraints(output_dir: Optional[str]) -> str: return "" try: import yaml + schema = yaml.safe_load(schema_path.read_text(encoding="utf-8")) except Exception: return "" @@ -57,7 +58,9 @@ def _build_schema_constraints(output_dir: Optional[str]) -> str: if isinstance(section, dict): title = section.get("title", "") mermaid = section.get("mermaid_diagram", False) - lines.append(f" - {title}" + (" (must include Mermaid diagram)" if mermaid else "")) + lines.append( + f" - {title}" + (" (must include Mermaid diagram)" if mermaid else "") + ) elif isinstance(section, str): lines.append(f" - {section}") if lines: @@ -94,7 +97,9 @@ def _build_schema_constraints(output_dir: Optional[str]) -> str: # LLM Wiki: extraction granularity granularity = schema.get("extraction_granularity", "") if granularity: - parts.append(f"Extraction granularity: {granularity} (focused=3-7 items, standard=moderate, exhaustive=comprehensive)") + parts.append( + f"Extraction granularity: {granularity} (focused=3-7 items, standard=moderate, exhaustive=comprehensive)" + ) # Project purpose (defined in schema.yaml) purpose_text = schema.get("purpose", "") @@ -146,14 +151,22 @@ def _build_schema_constraints(output_dir: Optional[str]) -> str: # Hardcoded fallback for doc_type hints (used when schema.yaml has no doc_types) _FALLBACK_DOC_TYPE_HINTS = { - "api": {"module": "Focus on API documentation: endpoints, parameters, return types, and usage examples."}, + "api": { + "module": "Focus on API documentation: endpoints, parameters, return types, and usage examples." + }, "architecture": { "module": "Focus on architecture documentation: system design, component relationships, and data flow.", "overview": "Focus on system-level architecture: show how modules relate, data flows between components, and the overall layered design. Include a high-level Mermaid architecture diagram.", }, - "user-guide": {"module": "Focus on user guide documentation: how to use features, step-by-step tutorials."}, - "developer": {"module": "Focus on developer documentation: code structure, contribution guidelines, and implementation details."}, - "business": {"module": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details."}, + "user-guide": { + "module": "Focus on user guide documentation: how to use features, step-by-step tutorials." + }, + "developer": { + "module": "Focus on developer documentation: code structure, contribution guidelines, and implementation details." + }, + "business": { + "module": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details." + }, "design": { "module": "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", "overview": "Focus on system-level architecture: show how modules relate to each other, data flows between components, overall layered design, and key architectural decisions. Provide a high-level view that helps readers understand the system's structural blueprint. Include Mermaid diagrams for the architecture overview.", @@ -366,19 +379,30 @@ def handle_get_prompt( repo_path = arguments.get("repo_path") if output_dir_arg: from pathlib import Path + output_dir = str(Path(output_dir_arg).expanduser().resolve()) # No session/workspace here; large prompts stay inline. session = None elif repo_path: from pathlib import Path - rp = str(Path(repo_path).expanduser().resolve()) if Path(repo_path).is_absolute() else str((Path.cwd() / repo_path).expanduser().resolve()) + + rp = ( + str(Path(repo_path).expanduser().resolve()) + if Path(repo_path).is_absolute() + else str((Path.cwd() / repo_path).expanduser().resolve()) + ) output_dir = str(Path(rp) / "repowiki") # Try to find active session for workspace access session = store.find_or_restore(rp) # Create a lightweight workspace for large prompt writing if no active session if session is None: from codewiki.mcp.workspace import SessionWorkspace - session = type('obj', (object,), {'output_dir': output_dir, 'workspace': SessionWorkspace(Path(rp), 'prompt')})() + + session = type( + "obj", + (object,), + {"output_dir": output_dir, "workspace": SessionWorkspace(Path(rp), "prompt")}, + )() else: session = None output_dir = None @@ -393,16 +417,20 @@ def handle_get_prompt( if prompt_type not in _PROMPT_CATALOG: available = list(_PROMPT_CATALOG.keys()) - return json.dumps({ - "error": f"Unknown prompt_type: {prompt_type}", - "available_types": available, - }) + return json.dumps( + { + "error": f"Unknown prompt_type: {prompt_type}", + "available_types": available, + } + ) catalog_entry = _PROMPT_CATALOG[prompt_type] # Inject schema constraints from schema.yaml into variables for _resolve_prompt schema_constraints = _build_schema_constraints(output_dir) - variables['_has_caller_ci'] = "custom_instructions" in variables and variables["custom_instructions"] + variables["_has_caller_ci"] = ( + "custom_instructions" in variables and variables["custom_instructions"] + ) if schema_constraints: caller_ci = variables.get("custom_instructions") if caller_ci: @@ -413,6 +441,7 @@ def handle_get_prompt( # Inject doc_types config from schema for doc_type hint resolution if output_dir: from codewiki.mcp.tools.page_router import load_schema + try: schema = load_schema(output_dir) variables["_doc_types"] = schema.get("doc_types", {}) @@ -436,19 +465,24 @@ def handle_get_prompt( } # Write to file when content is large and session is available - if (session and getattr(session, "workspace", None) - and len(content.encode("utf-8")) > _FILE_THRESHOLD): - file_path = session.workspace.write_text( - f"prompt_{prompt_type}.txt", content + if ( + session + and getattr(session, "workspace", None) + and len(content.encode("utf-8")) > _FILE_THRESHOLD + ): + file_path = session.workspace.write_text(f"prompt_{prompt_type}.txt", content) + return json.dumps( + { + "prompt_type": prompt_type, + "description": catalog_entry["description"], + "usage_hint": catalog_entry["usage_hint"], + "file": str(file_path), + "content_length": len(content), + "hint": "Read the file for the full prompt content.", + }, + indent=2, + ensure_ascii=False, ) - return json.dumps({ - "prompt_type": prompt_type, - "description": catalog_entry["description"], - "usage_hint": catalog_entry["usage_hint"], - "file": str(file_path), - "content_length": len(content), - "hint": "Read the file for the full prompt content.", - }, indent=2, ensure_ascii=False) return json.dumps(result, indent=2, ensure_ascii=False) @@ -562,7 +596,9 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: """Resolve a prompt template with optional variable substitution.""" if prompt_type == "cluster": - potential_core_components = variables.get("potential_core_components", "") + potential_core_components = variables.get( + "potential_core_components", "" + ) module_tree = variables.get("module_tree", {}) module_name = variables.get("module_name", None) return format_cluster_prompt( @@ -600,10 +636,12 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: # Return the template with placeholders filled as possible return USER_PROMPT.format( module_name=module_name, - module_tree=json.dumps(module_tree, indent=2) if module_tree else "", + module_tree=json.dumps(module_tree, indent=2) + if module_tree + else "", formatted_core_component_codes=variables.get( "formatted_core_component_codes", - "" + "", ), ) @@ -619,10 +657,14 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: custom_instructions = doc_type_hint custom_section = "" if custom_instructions: - custom_section = f"\n\n{custom_instructions}\n" + custom_section = ( + f"\n\n{custom_instructions}\n" + ) return MODULE_OVERVIEW_PROMPT.format( module_name=module_name, - repo_structure=repo_structure if isinstance(repo_structure, str) else json.dumps(repo_structure, indent=4), + repo_structure=repo_structure + if isinstance(repo_structure, str) + else json.dumps(repo_structure, indent=4), custom_instructions=custom_section, ) @@ -638,10 +680,14 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: custom_instructions = doc_type_hint custom_section = "" if custom_instructions: - custom_section = f"\n\n{custom_instructions}\n" + custom_section = ( + f"\n\n{custom_instructions}\n" + ) return REPO_OVERVIEW_PROMPT.format( repo_name=repo_name, - repo_structure=repo_structure if isinstance(repo_structure, str) else json.dumps(repo_structure, indent=4), + repo_structure=repo_structure + if isinstance(repo_structure, str) + else json.dumps(repo_structure, indent=4), custom_instructions=custom_section, ) @@ -649,8 +695,13 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: from codewiki.src.be.prompt_template import WORKSPACE_OVERVIEW_PROMPT workspace_name = variables.get("workspace_name", "WORKSPACE") - services_summary = variables.get("services_summary", "") - cross_service_data = variables.get("cross_service_data", "") + services_summary = variables.get( + "services_summary", "" + ) + cross_service_data = variables.get( + "cross_service_data", + "", + ) custom_instructions = variables.get("custom_instructions", None) doc_type_hint = _resolve_doc_type_hint(variables, "overview") if doc_type_hint: @@ -660,7 +711,9 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: custom_instructions = doc_type_hint custom_section = "" if custom_instructions: - custom_section = f"\n\n{custom_instructions}\n" + custom_section = ( + f"\n\n{custom_instructions}\n" + ) return WORKSPACE_OVERVIEW_PROMPT.format( workspace_name=workspace_name, services_summary=services_summary, @@ -891,7 +944,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "```yaml\n" "---\n" "type: entity\n" - "title: \"\"\n" + 'title: ""\n' "aliases: []\n" "category: \n" "tags: []\n" @@ -908,7 +961,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "- Compiler, not author: factual statements reuse the source document's own sentences, annotated with\n" " `[^src::]`. Light reordering, deduplication, and joining are fine;\n" " do NOT rephrase for style or expand short statements into longer ones.\n" - "- No rhetorical filler: phrases like \"旨在帮助…\", \"该平台致力于…\", \"具有重要意义\" must NOT appear\n" + '- No rhetorical filler: phrases like "旨在帮助…", "该平台致力于…", "具有重要意义" must NOT appear\n' " unless literally present in the source.\n" "- Scope discipline: every statement must be about the page title itself. Reject material that clearly\n" " belongs to a different but related thing.\n" @@ -926,7 +979,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "```yaml\n" "---\n" "type: concept\n" - "title: \"\"\n" + 'title: ""\n' "aliases: []\n" "domain: \n" "tags: []\n" @@ -943,7 +996,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "- Compiler, not author: factual statements reuse the source document's own sentences, annotated with\n" " `[^src::]`. Light reordering, deduplication, and joining are fine;\n" " do NOT rephrase for style or expand short statements into longer ones.\n" - "- No rhetorical filler: phrases like \"旨在帮助…\", \"该平台致力于…\", \"具有重要意义\" must NOT appear\n" + '- No rhetorical filler: phrases like "旨在帮助…", "该平台致力于…", "具有重要意义" must NOT appear\n' " unless literally present in the source.\n" "- Scope discipline: every statement must be about the page title itself. Reject material that clearly\n" " belongs to a different but related concept.\n" @@ -962,10 +1015,10 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "```yaml\n" "---\n" "type: source\n" - "title: \"\"\n" - "origin: \"\"\n" + 'title: ""\n' + 'origin: ""\n' "source_type: \n" - "version: \"\"\n" + 'version: ""\n' "tags: []\n" "---\n" "```\n\n" @@ -980,9 +1033,9 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: " each annotated with `[^src::]`. Do NOT invent, synthesize, or infer information\n" " not explicitly present in the source.\n" "- Stay close to source wording: reuse the source's own sentences; do NOT rephrase for style or pad with\n" - " rhetorical filler (\"旨在帮助…\", \"具有重要意义\" etc.).\n" + ' rhetorical filler ("旨在帮助…", "具有重要意义" etc.).\n' "- Empty content rule: if the source carries no substantive extractable text, say so explicitly\n" - " (\"No textual content was extractable from this document.\"). Do NOT invent a topic or guess from\n" + ' ("No textual content was extractable from this document."). Do NOT invent a topic or guess from\n' " the filename — uploaded files often have uninformative names." ) @@ -995,7 +1048,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "```yaml\n" "---\n" "type: comparison\n" - "title: \" vs \"\n" + 'title: " vs "\n' "subjects: []\n" "tags: []\n" "---\n" @@ -1018,7 +1071,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "```yaml\n" "---\n" "type: query\n" - "title: \"\"\n" + 'title: ""\n' "query_date: \n" "query_status: \n" "tags: []\n" @@ -1053,13 +1106,13 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "### Step 3: Routing\n\n" "| Knowledge type | Write method |\n" "|---|---|\n" - "| Technical choice / trade-off | `ingest_note(note_type=\"decision\")` |\n" - "| Pitfall / gotcha | `ingest_note(note_type=\"pitfall\")` |\n" - "| Lesson learned (debug journey, corrected assumption) | `ingest_note(note_type=\"lesson\")` |\n" - "| Architectural fact discovered | `ingest_note(note_type=\"architecture\")` |\n" - "| Temporary workaround (with recovery condition) | `ingest_note(note_type=\"workaround\")` |\n" - "| Multi-option comparison (with table) | `write_doc_file(page_type=\"comparison\")` |\n" - "| Research conclusion archive | `write_doc_file(page_type=\"query\")` |\n\n" + '| Technical choice / trade-off | `ingest_note(note_type="decision")` |\n' + '| Pitfall / gotcha | `ingest_note(note_type="pitfall")` |\n' + '| Lesson learned (debug journey, corrected assumption) | `ingest_note(note_type="lesson")` |\n' + '| Architectural fact discovered | `ingest_note(note_type="architecture")` |\n' + '| Temporary workaround (with recovery condition) | `ingest_note(note_type="workaround")` |\n' + '| Multi-option comparison (with table) | `write_doc_file(page_type="comparison")` |\n' + '| Research conclusion archive | `write_doc_file(page_type="query")` |\n\n' "### Step 4: Draft Format\n\n" "Present to user for confirmation before writing:\n\n" "```\n" @@ -1091,16 +1144,16 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "### Output format:\n" "```json\n" "{\n" - " \"taxonomy_plan\": {\n" - " \"wiki/modules/\": [],\n" - " \"wiki/entities/\": [],\n" - " \"wiki/concepts/\": [],\n" - " \"wiki/sources/\": [],\n" - " \"wiki/comparisons/\": [],\n" - " \"wiki/queries/\": []\n" + ' "taxonomy_plan": {\n' + ' "wiki/modules/": [],\n' + ' "wiki/entities/": [],\n' + ' "wiki/concepts/": [],\n' + ' "wiki/sources/": [],\n' + ' "wiki/comparisons/": [],\n' + ' "wiki/queries/": []\n' " },\n" - " \"suggested_aliases\": {\n" - " \"\": []\n" + ' "suggested_aliases": {\n' + ' "": []\n' " }\n" "}\n" "```\n\n" @@ -1113,9 +1166,7 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: elif prompt_type == "extraction_scan": granularity = ( - variables.get("granularity") - or variables.get("_schema_granularity") - or "standard" + variables.get("granularity") or variables.get("_schema_granularity") or "standard" ) return ( f"## Extraction Scan Template (granularity: {granularity})\n\n" @@ -1129,31 +1180,31 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: " If unsure, LEAVE IT OUT — a clean focused index beats a comprehensive noisy one.\n" "- **standard**: main subjects PLUS substantively discussed items — those with a dedicated paragraph,\n" " a multi-point list, or at least 2-3 sentences of context. EXCLUDE comma-separated list mentions\n" - " (e.g. \"Tech stack: A, B, C, D\" without individual discussion), one-off mentions, parenthetical references.\n" + ' (e.g. "Tech stack: A, B, C, D" without individual discussion), one-off mentions, parenthetical references.\n' " When in doubt about a marginal item, prefer to EXCLUDE it.\n" "- **exhaustive**: every named entity and recognizable concept, including concrete well-known\n" " technologies/standards/methodologies mentioned even once by name. EXCLUDE only truly generic\n" - " terms (\"server\", \"function\", \"data\") and items appearing only inside URLs or citations.\n\n" + ' terms ("server", "function", "data") and items appearing only inside URLs or citations.\n\n' f"Current granularity: **{granularity}**\n\n" "### Extraction format:\n" "```json\n" "{\n" - " \"items\": [\n" + ' "items": [\n' " {\n" - " \"title\": \"\",\n" - " \"type\": \"\",\n" - " \"summary\": \"<1-2 sentence summary>\",\n" - " \"aliases\": [\"\"],\n" - " \"source_ref\": \"[^src::]\",\n" - " \"target_page\": \"/.md>\"\n" + ' "title": "",\n' + ' "type": "",\n' + ' "summary": "<1-2 sentence summary>",\n' + ' "aliases": [""],\n' + ' "source_ref": "[^src::]",\n' + ' "target_page": "/.md>"\n' " }\n" " ]\n" "}\n" "```\n\n" "### Rules:\n" "- Each item must reference its source location — the line range where the item is SUBSTANTIVELY discussed, not a passing mention\n" - "- aliases only include names for the EXACT same item: official abbreviations (\"IBM\" for \"International Business Machines\"),\n" - " full/short name variants (\"腾讯\" for \"腾讯控股\"), translations (\"Apple\" for \"苹果公司\").\n" + '- aliases only include names for the EXACT same item: official abbreviations ("IBM" for "International Business Machines"),\n' + ' full/short name variants ("腾讯" for "腾讯控股"), translations ("Apple" for "苹果公司").\n' " NEVER include parent categories, related products, generic terms, or broader concepts. Use [] if none.\n" "- Type separation: specific named things (people, orgs, products, services, APIs) go to entity;\n" " abstract ideas (patterns, methodologies, theories, protocols) go to concept.\n" @@ -1175,18 +1226,18 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "2. The match is a **name variation**: abbreviation ↔ full name, translation, or minor spelling difference.\n" "3. Types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.**\n\n" "### Examples of CORRECT merges:\n" - "- \"Acme Corp\" → \"Acme Corporation\" (same company, abbreviation)\n" - "- \"RAG\" → \"Retrieval-Augmented Generation\" (same concept, acronym)\n" - "- \"苹果公司\" → \"Apple Inc.\" (same entity, translation)\n\n" + '- "Acme Corp" → "Acme Corporation" (same company, abbreviation)\n' + '- "RAG" → "Retrieval-Augmented Generation" (same concept, acronym)\n' + '- "苹果公司" → "Apple Inc." (same entity, translation)\n\n' "### Examples of INCORRECT merges — do NOT merge these:\n" - "- \"混元模型\" ≠ \"通义模型\" (competing products in the same category are DIFFERENT entities)\n" - "- \"iPhone 15\" ≠ \"华为 Mate 60\" (different specific products in the same category)\n" - "- \"GPT-4\" ≠ \"GPT-3.5\" (different versions of a product are distinct entities)\n" - "- \"AI 安全\" ≠ \"内容审核机制\" (related topics, but different concepts)\n" - "- \"机器学习\" ≠ \"神经网络\" (neural networks are a subset of ML, not the same concept)\n" - "- \"居民身份证\" ≠ \"工作居住证\" (both government-issued documents but completely different credentials)\n" - "- \"学位证\" ≠ \"毕业证\" (both educational documents but distinct)\n" - "- \"运动员注册\" ≠ \"学历认证\" (both involve verification, but completely different domains)\n\n" + '- "混元模型" ≠ "通义模型" (competing products in the same category are DIFFERENT entities)\n' + '- "iPhone 15" ≠ "华为 Mate 60" (different specific products in the same category)\n' + '- "GPT-4" ≠ "GPT-3.5" (different versions of a product are distinct entities)\n' + '- "AI 安全" ≠ "内容审核机制" (related topics, but different concepts)\n' + '- "机器学习" ≠ "神经网络" (neural networks are a subset of ML, not the same concept)\n' + '- "居民身份证" ≠ "工作居住证" (both government-issued documents but completely different credentials)\n' + '- "学位证" ≠ "毕业证" (both educational documents but distinct)\n' + '- "运动员注册" ≠ "学历认证" (both involve verification, but completely different domains)\n\n' "### Key principle: related ≠ same\n" "Two items sharing a few characters in their name, or belonging to the same domain / category / industry,\n" "is NOT a reason to merge. ABSOLUTELY DO NOT merge different products, different companies, different versions,\n" @@ -1200,12 +1251,12 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: "### Decision output format:\n" "```json\n" "{\n" - " \"decisions\": [\n" + ' "decisions": [\n' " {\n" - " \"title\": \"\",\n" - " \"action\": \"\",\n" - " \"merge_target\": \"\",\n" - " \"reason\": \"\"\n" + ' "title": "",\n' + ' "action": "",\n' + ' "merge_target": "",\n' + ' "reason": ""\n' " }\n" " ]\n" "}\n" diff --git a/codewiki/mcp/tools/reading_guide.py b/codewiki/mcp/tools/reading_guide.py index 4652411..44e65d0 100644 --- a/codewiki/mcp/tools/reading_guide.py +++ b/codewiki/mcp/tools/reading_guide.py @@ -75,10 +75,10 @@ def generate_reading_guide( lines: List[str] = [ "---", "type: Concept", - "title: \"阅读指南\"", + 'title: "阅读指南"', f"generated: {{ by: codewiki/reading_guide.py, at: {generated_at} }}", "stale_after: 2099-12-31", - "description: \"> 基于 PageRank 依赖分析自动生成。排名越靠前的组件被越多模块依赖，建议优先阅读。\"", + 'description: "> 基于 PageRank 依赖分析自动生成。排名越靠前的组件被越多模块依赖，建议优先阅读。"', "---", "# 阅读指南", "", @@ -96,12 +96,18 @@ def generate_reading_guide( meta = components.get(comp_id) name = getattr(meta, "name", comp_id) if meta else comp_id ctype = getattr(meta, "component_type", "?") if meta else "?" - fpath = (getattr(meta, "relative_path", "") or getattr(meta, "file_path", "")) if meta else "" + fpath = ( + (getattr(meta, "relative_path", "") or getattr(meta, "file_path", "")) + if meta + else "" + ) mod = comp_module_idx.get(comp_id, "-") dep_count = len(reverse.get(comp_id, set())) # Truncate long paths for table readability short_path = fpath if len(fpath) <= 50 else "..." + fpath[-47:] - lines.append(f"| {i} | `{name}` | {ctype} | {mod} | {dep_count} | {score:.4f} | {short_path} |") + lines.append( + f"| {i} | `{name}` | {ctype} | {mod} | {dep_count} | {score:.4f} | {short_path} |" + ) # Module-level summary if comp_module_idx: @@ -113,21 +119,25 @@ def generate_reading_guide( top_mods = sorted(mod_scores.items(), key=lambda x: -x[1])[:10] if top_mods: - lines.extend([ - "", - "## 模块重要性排名", - "", - "| # | 模块 | 累计 PageRank |", - "|---|------|---------------|", - ]) + lines.extend( + [ + "", + "## 模块重要性排名", + "", + "| # | 模块 | 累计 PageRank |", + "|---|------|---------------|", + ] + ) for i, (mod, sc) in enumerate(top_mods, 1): lines.append(f"| {i} | {mod} | {sc:.4f} |") - lines.extend([ - "", - "---", - f"*基于 {len(components)} 个组件、{sum(len(d) for d in graph.values())} 条依赖边计算。*", - ]) + lines.extend( + [ + "", + "---", + f"*基于 {len(components)} 个组件、{sum(len(d) for d in graph.values())} 条依赖边计算。*", + ] + ) # Write file wiki_dir = Path(output_dir) / "wiki" diff --git a/codewiki/mcp/tools/review_changes.py b/codewiki/mcp/tools/review_changes.py index 19473a6..f0243f8 100644 --- a/codewiki/mcp/tools/review_changes.py +++ b/codewiki/mcp/tools/review_changes.py @@ -60,6 +60,7 @@ # Source reading (version-aware) + change-line annotation # ------------------------------------------------------------------ + def _read_versioned_lines(git_root: str, rel_path: str, since: Optional[str]) -> List[str]: """File lines from HEAD (``since`` mode) or the working tree.""" if since: @@ -128,14 +129,11 @@ def _build_changed_sources( all_lines = _read_versioned_lines(git_root, rel, None) if start_line > 0 and end_line > 0: - comp_lines = all_lines[max(0, start_line - 1):end_line] + comp_lines = all_lines[max(0, start_line - 1) : end_line] else: comp_lines = all_lines - header = ( - f"### {cid}\n" - f"# file: {rel} lines {start_line}-{end_line}\n" - ) + header = f"### {cid}\n# file: {rel} lines {start_line}-{end_line}\n" body = _annotate_lines(comp_lines, max(1, start_line), changed_lines) by_file.setdefault(rel, []).append(header + body) @@ -216,8 +214,11 @@ def _build_target( except Exception as exc: # pragma: no cover - graph build failure degrades, not fatal logger.warning("Impact computation failed: %s", exc) - suggested = suggest_tests(components, {a["component_id"] for a in affected_list} | start_ids, - repo_path=session.repo_path) + suggested = suggest_tests( + components, + {a["component_id"] for a in affected_list} | start_ids, + repo_path=session.repo_path, + ) return { "diff_summary": { @@ -236,6 +237,7 @@ def _build_target( # Axis evidence collectors (deterministic) # ------------------------------------------------------------------ + def _read_spec_file(path: Path) -> Optional[str]: try: return path.read_text(encoding="utf-8", errors="replace")[:_SPEC_MAX_CHARS] @@ -319,7 +321,9 @@ def _query_wiki(store: SessionStore, session: Any, arguments: Dict[str, Any]) -> try: return json.loads(handle_query_wiki(call, store)) except Exception as exc: - logger.warning("query_wiki failed (%s): %s", arguments.get("query") or arguments.get("mode"), exc) + logger.warning( + "query_wiki failed (%s): %s", arguments.get("query") or arguments.get("mode"), exc + ) return {} @@ -431,6 +435,7 @@ def _collect_general_evidence(repo_path: str, changes: List[FileChange]) -> Dict # Submit: report validation + archiving # ------------------------------------------------------------------ + def _slugify(title: str) -> str: slug = _SLUG_RE.sub("-", title).strip("-") return slug[:40] or "report" @@ -529,6 +534,7 @@ def _handle_submit(arguments: Dict[str, Any], session: Any) -> str: # Main handler # ------------------------------------------------------------------ + def handle_review_changes( arguments: Dict[str, Any], store: SessionStore, @@ -558,13 +564,17 @@ def handle_review_changes( session = resolve_session(arguments, store) if session is None: return json.dumps( - {"error": "Session not found. Provide a valid repo_path pointing to a previously analyzed repository."}, + { + "error": "Session not found. Provide a valid repo_path pointing to a previously analyzed repository." + }, ensure_ascii=False, ) mode = arguments.get("mode", "prepare") if mode not in ("prepare", "submit"): - return json.dumps({"error": f"Invalid mode {mode!r}: expected 'prepare' or 'submit'."}, ensure_ascii=False) + return json.dumps( + {"error": f"Invalid mode {mode!r}: expected 'prepare' or 'submit'."}, ensure_ascii=False + ) if mode == "submit": return _handle_submit(arguments, session) @@ -574,7 +584,10 @@ def handle_review_changes( spec_paths: List[str] = list(arguments.get("spec_paths") or []) focus = arguments.get("focus", "all") if focus not in ("all",) + _AXES: - return json.dumps({"error": f"Invalid focus {focus!r}: expected 'all' or one of {_AXES}."}, ensure_ascii=False) + return json.dumps( + {"error": f"Invalid focus {focus!r}: expected 'all' or one of {_AXES}."}, + ensure_ascii=False, + ) try: git_info = collect_git_changes(session.repo_path, since=since, worktree=True) @@ -595,7 +608,9 @@ def handle_review_changes( "query": {"repo_path": session.repo_path, "since": since, "focus": focus}, "target": { "diff_summary": {"changed_files": 0, "added_lines": 0, "deleted_lines": 0}, - "changed_components": [], "changed_sources": {}, "affected_components": [], + "changed_components": [], + "changed_sources": {}, + "affected_components": [], "suggested_tests": [], }, "evidence": {}, diff --git a/codewiki/mcp/tools/review_checklist.py b/codewiki/mcp/tools/review_checklist.py index 61c19f4..9f6c6a3 100644 --- a/codewiki/mcp/tools/review_checklist.py +++ b/codewiki/mcp/tools/review_checklist.py @@ -155,7 +155,9 @@ { "id": "py-bare-except", "title": "裸 except", - "questions": ["是否使用裸 except / except Exception 吞掉包括 KeyboardInterrupt 在内的异常？"], + "questions": [ + "是否使用裸 except / except Exception 吞掉包括 KeyboardInterrupt 在内的异常？" + ], }, { "id": "py-resource-context", diff --git a/codewiki/mcp/tools/schema_generator.py b/codewiki/mcp/tools/schema_generator.py index 3abeb27..c117f14 100644 --- a/codewiki/mcp/tools/schema_generator.py +++ b/codewiki/mcp/tools/schema_generator.py @@ -8,7 +8,6 @@ from __future__ import annotations import logging -import re from collections import Counter from datetime import datetime from pathlib import Path @@ -61,9 +60,7 @@ # below _DEFAULT_CONVENTIONS to keep the dict literal readable). from codewiki.mcp.tools.note_types import DEFAULT_NOTE_TYPES # noqa: E402 -_DEFAULT_CONVENTIONS["note_types"] = { - t: dict(spec) for t, spec in DEFAULT_NOTE_TYPES.items() -} +_DEFAULT_CONVENTIONS["note_types"] = {t: dict(spec) for t, spec in DEFAULT_NOTE_TYPES.items()} _DEFAULT_REQUIRED_SECTIONS = [ {"title": "Architecture Overview", "mermaid_diagram": True}, @@ -97,14 +94,22 @@ _DEFAULT_DOC_TYPES = { "default": "design", "types": { - "api": {"module": "Focus on API documentation: endpoints, parameters, return types, and usage examples."}, + "api": { + "module": "Focus on API documentation: endpoints, parameters, return types, and usage examples." + }, "architecture": { "module": "Focus on architecture documentation: system design, component relationships, and data flow.", "overview": "Focus on system-level architecture: show how modules relate, data flows between components, and the overall layered design. Include a high-level Mermaid architecture diagram.", }, - "user-guide": {"module": "Focus on user guide documentation: how to use features, step-by-step tutorials."}, - "developer": {"module": "Focus on developer documentation: code structure, contribution guidelines, and implementation details."}, - "business": {"module": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details."}, + "user-guide": { + "module": "Focus on user guide documentation: how to use features, step-by-step tutorials." + }, + "developer": { + "module": "Focus on developer documentation: code structure, contribution guidelines, and implementation details." + }, + "business": { + "module": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details." + }, "design": { "module": "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", "overview": "Focus on system-level architecture: show how modules relate to each other, data flows between components, overall layered design, and key architectural decisions. Provide a high-level view that helps readers understand the system's structural blueprint. Include Mermaid diagrams for the architecture overview.", @@ -115,7 +120,18 @@ # Default code routing rules (Roadmap 2.1) _DEFAULT_CODE_ROUTING = { "boilerplate_patterns": { - "suffix": ["DTO", "VO", "Request", "Response", "Entity", "PO", "Model", "Mapper", "Repository", "Dao"], + "suffix": [ + "DTO", + "VO", + "Request", + "Response", + "Entity", + "PO", + "Model", + "Mapper", + "Repository", + "Dao", + ], "annotation": ["@Data", "@Getter", "@Entity", "@Table", "@Document"], }, "business_patterns": { @@ -140,35 +156,48 @@ "directory": "wiki/entities", "description": "关键类、接口、数据模型、API 端点的独立文档", "required_sections": [ - "职责描述", "公开 API", "使用示例", "依赖关系", + "职责描述", + "公开 API", + "使用示例", + "依赖关系", ], }, "concept": { "directory": "wiki/concepts", "description": "设计模式、架构理念、领域概念的文档", "required_sections": [ - "概念定义", "适用场景", "在本项目中的应用", + "概念定义", + "适用场景", + "在本项目中的应用", ], }, "source": { "directory": "wiki/sources", "description": "第三方文档（SDK/API/框架文档）的摘要", "required_sections": [ - "文档概述", "关键 API/概念", "与本项目相关的部分", + "文档概述", + "关键 API/概念", + "与本项目相关的部分", ], }, "comparison": { "directory": "wiki/comparisons", "description": "方案对比、技术选型分析", "required_sections": [ - "背景与目标", "候选方案", "对比分析", "结论与决策", + "背景与目标", + "候选方案", + "对比分析", + "结论与决策", ], }, "query": { "directory": "wiki/queries", "description": "方案设计决策记录，包含推理过程和权衡", "required_sections": [ - "问题描述", "调研过程", "方案权衡", "决策结论", + "问题描述", + "调研过程", + "方案权衡", + "决策结论", ], }, } diff --git a/codewiki/mcp/tools/source_ingest.py b/codewiki/mcp/tools/source_ingest.py index e584524..cb3fe30 100644 --- a/codewiki/mcp/tools/source_ingest.py +++ b/codewiki/mcp/tools/source_ingest.py @@ -12,9 +12,8 @@ import hashlib import json import logging -import os import shutil -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional @@ -26,6 +25,7 @@ def _load_registry(output_dir: Path) -> Dict[str, Any]: """Load source_registry.json from output_dir/.meta/. Falls back to root for compat.""" from codewiki.src.config import SOURCE_REGISTRY_FILENAME, META_DIR + # Prefer .meta/ location, fallback to root (backward compat) meta_path = output_dir / META_DIR / SOURCE_REGISTRY_FILENAME root_path = output_dir / SOURCE_REGISTRY_FILENAME @@ -34,7 +34,9 @@ def _load_registry(output_dir: Path) -> Dict[str, Any]: return {"sources": {}, "version": 1} try: data = json.loads(reg_path.read_text(encoding="utf-8")) - return data if isinstance(data, dict) and "sources" in data else {"sources": {}, "version": 1} + return ( + data if isinstance(data, dict) and "sources" in data else {"sources": {}, "version": 1} + ) except (json.JSONDecodeError, OSError): return {"sources": {}, "version": 1} @@ -42,6 +44,7 @@ def _load_registry(output_dir: Path) -> Dict[str, Any]: def _save_registry(output_dir: Path, registry: Dict[str, Any]) -> None: """Persist source_registry.json to output_dir/.meta/.""" from codewiki.src.config import SOURCE_REGISTRY_FILENAME, META_DIR + meta_dir = output_dir / META_DIR meta_dir.mkdir(parents=True, exist_ok=True) reg_path = meta_dir / SOURCE_REGISTRY_FILENAME @@ -95,6 +98,7 @@ def _merge_okf_sources_entry(page_path: Path, entry: Dict[str, Any]) -> None: return try: import yaml + data = yaml.safe_load(content[3:end]) if not isinstance(data, dict): return @@ -108,12 +112,14 @@ def _merge_okf_sources_entry(page_path: Path, entry: Dict[str, Any]) -> None: sources.append(entry) data["sources"] = sources new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False) - page_path.write_text(f"---\n{new_fm}---{content[end + 3:]}", encoding="utf-8") + page_path.write_text(f"---\n{new_fm}---{content[end + 3 :]}", encoding="utf-8") except Exception as e: logger.debug("OKF sources merge skipped for %s: %s", page_path, e) -def _ensure_source_frontmatter(dest_path: Path, name: str, description: str, output_dir: Optional[Path] = None) -> None: +def _ensure_source_frontmatter( + dest_path: Path, name: str, description: str, output_dir: Optional[Path] = None +) -> None: """Ensure an ingested raw/sources markdown file carries OKF frontmatter. OKF v0.2 §11 conformance applies to every non-reserved .md in the @@ -132,6 +138,7 @@ def _ensure_source_frontmatter(dest_path: Path, name: str, description: str, out if content.startswith("---"): return # keep whatever the source document already declares from codewiki.src.frontmatter import inject_okf_frontmatter + fm = inject_okf_frontmatter( content, type_="Source", @@ -196,13 +203,17 @@ def _inject_source_refs(output_dir: Path, related_pages: List[str], source_name: # Find end of the list (next non-indented, non-list line) insert_idx = i + 1 while insert_idx < len(lines) and ( - lines[insert_idx].startswith(" ") or lines[insert_idx].strip().startswith("- ") + lines[insert_idx].startswith(" ") + or lines[insert_idx].strip().startswith("- ") ): insert_idx += 1 break if insert_idx is not None: # Avoid duplicate - if f'- "{source_name}"' not in frontmatter and f"- {source_name}" not in frontmatter: + if ( + f'- "{source_name}"' not in frontmatter + and f"- {source_name}" not in frontmatter + ): lines.insert(insert_idx, f' - "{source_name}"') frontmatter = "\n".join(lines) else: @@ -270,17 +281,22 @@ def handle_ingest_source( for existing_name, info in registry.get("sources", {}).items(): if isinstance(info, dict) and info.get("content_hash") == hash_key: if info.get("status") != "retracted": - return json.dumps({ - "status": "duplicate", - "name": name, - "existing_name": existing_name, - "content_hash": f"sha256:{content_hash[:16]}...", - "message": f"Content identical to existing source '{existing_name}'. " - f"Use a different file or retract the existing source first.", - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "duplicate", + "name": name, + "existing_name": existing_name, + "content_hash": f"sha256:{content_hash[:16]}...", + "message": f"Content identical to existing source '{existing_name}'. " + f"Use a different file or retract the existing source first.", + }, + indent=2, + ensure_ascii=False, + ) # Ensure raw/sources/ directory exists from codewiki.src.config import RAW_SOURCES_DIR + raw_sources = output_dir / RAW_SOURCES_DIR raw_sources.mkdir(parents=True, exist_ok=True) # Ensure .meta/ exists for registry and search index @@ -328,26 +344,31 @@ def handle_ingest_source( # LLM Wiki: update log try: from codewiki.mcp.tools.wiki_index import append_log - append_log(str(output_dir), "ingest_source", - f"导入外部文档: {name} ({source_type})") + + append_log(str(output_dir), "ingest_source", f"导入外部文档: {name} ({source_type})") except Exception: pass # Update search index for the new source try: from codewiki.mcp.tools.wiki_search import build_full_index + build_full_index(output_dir, session=session) except Exception as e: logger.warning("Search index rebuild failed (non-fatal): %s", e) - return json.dumps({ - "status": "ingested", - "name": name, - "source_type": source_type, - "stored_at": str(dest_path.relative_to(output_dir)), - "description": description, - "version": version, - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "ingested", + "name": name, + "source_type": source_type, + "stored_at": str(dest_path.relative_to(output_dir)), + "description": description, + "version": version, + }, + indent=2, + ensure_ascii=False, + ) def handle_retract_source( @@ -393,14 +414,18 @@ def handle_retract_source( # dry_run: report what would happen without mutating anything if dry_run: would_clean = _count_source_refs(output_dir, name) if mode == "remove_refs" else 0 - return json.dumps({ - "status": "dry_run", - "name": name, - "mode": mode, - "would_move_to_trash": (mode == "remove_refs" and bool(source_rel_path)), - "source_file": source_rel_path, - "would_clean_refs": would_clean, - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "dry_run", + "name": name, + "mode": mode, + "would_move_to_trash": (mode == "remove_refs" and bool(source_rel_path)), + "source_file": source_rel_path, + "would_clean_refs": would_clean, + }, + indent=2, + ensure_ascii=False, + ) if mode == "remove_refs": # Move the source file to .trash/ instead of permanent deletion @@ -411,7 +436,10 @@ def handle_retract_source( trash_dir.mkdir(parents=True, exist_ok=True) dest = trash_dir / source_abs.name if dest.exists(): - dest = trash_dir / f"{source_abs.stem}_{int(datetime.now().timestamp())}{source_abs.suffix}" + dest = ( + trash_dir + / f"{source_abs.stem}_{int(datetime.now().timestamp())}{source_abs.suffix}" + ) shutil.move(str(source_abs), str(dest)) except OSError as e: logger.warning("Failed to move source file to trash: %s", e) @@ -431,29 +459,35 @@ def handle_retract_source( # LLM Wiki: update log try: from codewiki.mcp.tools.wiki_index import append_log - append_log(str(output_dir), "retract_source", - f"撤回外部文档: {name} (mode={mode})") + + append_log(str(output_dir), "retract_source", f"撤回外部文档: {name} (mode={mode})") except Exception: pass # Rebuild search index try: from codewiki.mcp.tools.wiki_search import build_full_index + build_full_index(output_dir, session=session) except Exception: pass - return json.dumps({ - "status": "retracted", - "name": name, - "mode": mode, - "cleaned_refs": cleaned_refs, - }, indent=2, ensure_ascii=False) + return json.dumps( + { + "status": "retracted", + "name": name, + "mode": mode, + "cleaned_refs": cleaned_refs, + }, + indent=2, + ensure_ascii=False, + ) def _body_ref_patterns(source_name: str): """Regexes for in-body references that retract must clean.""" import re + return ( # Legacy annotation: [^src:name] / [^src:name:range] re.compile(rf"\[\^src:{re.escape(source_name)}(?::[^\]]*)?\]"), @@ -477,6 +511,7 @@ def _frontmatter_mentions(content: str, source_name: str) -> bool: fm = content[3:end] try: import yaml + data = yaml.safe_load(fm) except Exception: data = None @@ -490,14 +525,16 @@ def _frontmatter_mentions(content: str, source_name: str) -> bool: if isinstance(sources, dict): sources = [sources] if isinstance(sources, list) and any( - isinstance(s, dict) and s.get("id") == source_name for s in sources): + isinstance(s, dict) and s.get("id") == source_name for s in sources + ): return True return False # YAML parse failed → permissive textual fallback import re - return bool(re.search( - rf"^source_ref:\s*[\"']?{re.escape(source_name)}[\"']?\s*$", - fm, re.MULTILINE)) + + return bool( + re.search(rf"^source_ref:\s*[\"']?{re.escape(source_name)}[\"']?\s*$", fm, re.MULTILINE) + ) def _count_source_refs(output_dir: Path, source_name: str) -> int: @@ -517,8 +554,9 @@ def _count_source_refs(output_dir: Path, source_name: str) -> int: content = md_file.read_text(encoding="utf-8") except OSError: continue - if (any(p.search(content) for p in patterns) - or _frontmatter_mentions(content, source_name)): + if any(p.search(content) for p in patterns) or _frontmatter_mentions( + content, source_name + ): count += 1 return count @@ -539,6 +577,7 @@ def _strip_okf_sources_entry(md_file: Path, source_name: str) -> bool: return False try: import yaml + data = yaml.safe_load(content[3:end]) if not isinstance(data, dict): return False @@ -547,8 +586,7 @@ def _strip_okf_sources_entry(md_file: Path, source_name: str) -> bool: sources = [sources] if not isinstance(sources, list): return False - kept = [s for s in sources - if not (isinstance(s, dict) and s.get("id") == source_name)] + kept = [s for s in sources if not (isinstance(s, dict) and s.get("id") == source_name)] if len(kept) == len(sources): return False if kept: @@ -556,7 +594,7 @@ def _strip_okf_sources_entry(md_file: Path, source_name: str) -> bool: else: data.pop("sources", None) new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False) - md_file.write_text(f"---\n{new_fm}---{content[end + 3:]}", encoding="utf-8") + md_file.write_text(f"---\n{new_fm}---{content[end + 3 :]}", encoding="utf-8") return True except Exception: return False @@ -580,6 +618,7 @@ def _strip_source_ref_fields(md_file: Path, source_name: str) -> bool: return False try: import yaml + data = yaml.safe_load(content[3:end]) if not isinstance(data, dict): return False @@ -599,7 +638,7 @@ def _strip_source_ref_fields(md_file: Path, source_name: str) -> bool: if not changed: return False new_fm = yaml.safe_dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False) - md_file.write_text(f"---\n{new_fm}---{content[end + 3:]}", encoding="utf-8") + md_file.write_text(f"---\n{new_fm}---{content[end + 3 :]}", encoding="utf-8") return True except Exception: return False diff --git a/codewiki/mcp/tools/task_manager.py b/codewiki/mcp/tools/task_manager.py index 50fa176..2b88880 100644 --- a/codewiki/mcp/tools/task_manager.py +++ b/codewiki/mcp/tools/task_manager.py @@ -456,9 +456,9 @@ def _compaction_needed(total_entries: int, mem_bytes: int) -> bool: # Layered loading (multi-user split design §4.3): warm layer shape constants. -_WARM_RECENT_ENTRIES = 2 # per other author: recent entries injected (Q11) -_WARM_ENTRY_BUDGET = 2048 # per other author: chars before degrading (Q12) -_HINT_LINE_MAX = 60 # degraded hint: first-content-line truncation +_WARM_RECENT_ENTRIES = 2 # per other author: recent entries injected (Q11) +_WARM_ENTRY_BUDGET = 2048 # per other author: chars before degrading (Q12) +_HINT_LINE_MAX = 60 # degraded hint: first-content-line truncation _WARM_SECTION_HEADING = "## 其他成员记忆" @@ -488,15 +488,13 @@ def _parse_memory_file(path: Path) -> Optional[Tuple[str, str, List[str], int]]: return (text, summary, entries, path.stat().st_size) -def _render_warm_author( - owner: str, summary: str, entries: List[str], include_entries: bool -) -> str: +def _render_warm_author(owner: str, summary: str, entries: List[str], include_entries: bool) -> str: """Render one other author's warm block: summary + recent entries / hints.""" parts = [f"### @{owner}"] if summary: # Drop the canonical "## 早期记忆（摘要）" heading line; the @owner # heading above already frames the section. - body = summary[len(_SUMMARY_HEADING):].strip() + body = summary[len(_SUMMARY_HEADING) :].strip() if body: parts.append(body) if include_entries and entries: @@ -907,9 +905,7 @@ def handle_add_task_memory(arguments: Dict[str, Any], store: SessionStore) -> st if _find_by_id(tasks, task_id) is None: return json.dumps({"error": f"Task '{task_id}' does not exist."}) - _append_memory_atomic( - _memories_path_for(output_dir, task_id, _current_user_id()), content - ) + _append_memory_atomic(_memories_path_for(output_dir, task_id, _current_user_id()), content) return json.dumps( { @@ -1213,9 +1209,7 @@ def handle_compact_task_memories(arguments: Dict[str, Any], store: SessionStore) # not cover it. Tolerated: append-only writes are rare mid-compaction and # the entry is preserved verbatim in the archive either way. date = datetime.now().strftime("%Y-%m-%d") - archive_relpaths = ", ".join( - f"{_ARCHIVE_DIRNAME}/{owner}.md" for owner in archive_owners - ) + archive_relpaths = ", ".join(f"{_ARCHIVE_DIRNAME}/{owner}.md" for owner in archive_owners) pointer = f"> 原文归档于 {archive_relpaths}，截至 {date}，共 {len(compress)} 条。" new_text = ( f"{_SUMMARY_HEADING}\n\n{new_summary}\n\n{pointer}\n\n" @@ -1241,7 +1235,9 @@ def handle_compact_task_memories(arguments: Dict[str, Any], store: SessionStore) existing_archive = "" if archive_path.exists(): existing_archive = archive_path.read_text(encoding="utf-8").rstrip("\n") - new_archive = (existing_archive + "\n\n" if existing_archive else "") + "\n\n".join(parts) + "\n" + new_archive = ( + (existing_archive + "\n\n" if existing_archive else "") + "\n\n".join(parts) + "\n" + ) archive_tmp = archive_path.with_suffix(".tmp") archive_tmp.write_text(new_archive, encoding="utf-8") _atomic_replace_with_retry(archive_tmp, archive_path) diff --git a/codewiki/mcp/tools/telemetry.py b/codewiki/mcp/tools/telemetry.py index 2d2a5b3..d176946 100644 --- a/codewiki/mcp/tools/telemetry.py +++ b/codewiki/mcp/tools/telemetry.py @@ -56,6 +56,7 @@ def _meta_dir(output_dir) -> Path: try: from codewiki.src.config import META_DIR + return Path(output_dir) / META_DIR except Exception: return Path(output_dir) / ".meta" @@ -76,12 +77,14 @@ def telemetry_enabled(output_dir) -> bool: """ try: from codewiki.src.config import SCHEMA_FILENAME + name = SCHEMA_FILENAME except Exception: name = "schema.yaml" p = Path(output_dir) / name try: import yaml + with open(p, "r", encoding="utf-8") as fh: data = yaml.safe_load(fh) or {} block = (data.get("conventions") or {}).get("telemetry") or {} @@ -103,6 +106,7 @@ def _user_events_path(output_dir, create: bool = False) -> Path: d.mkdir(parents=True, exist_ok=True) try: from codewiki.src.config import user_id + uid = user_id() except Exception: uid = "local" @@ -127,7 +131,11 @@ def _atomic_write_lines(path: Path, lines: List[str]) -> None: def _read_lines(path: Path) -> List[str]: """Non-empty lines of a jsonl file; missing file → [] (never raises).""" try: - return [line for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()] + return [ + line + for line in path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.strip() + ] except OSError: return [] @@ -136,6 +144,7 @@ def _read_lines(path: Path) -> List[str]: # Write path # --------------------------------------------------------------------------- # + def record_hit(output_dir, doc_path: str, count: int = 1) -> None: """Append (or same-day-merge) a ``hit`` event for *doc_path*. @@ -159,19 +168,23 @@ def record_hit(output_dir, doc_path: str, count: int = 1) -> None: ev = json.loads(lines[i]) except (json.JSONDecodeError, ValueError, TypeError): continue # corrupt line → skip it, keep scanning - if (isinstance(ev, dict) - and ev.get("t") == "hit" - and ev.get("doc") == doc_path - and str(ev.get("at", "")) == today): + if ( + isinstance(ev, dict) + and ev.get("t") == "hit" + and ev.get("doc") == doc_path + and str(ev.get("at", "")) == today + ): ev["n"] = int(ev.get("n", 0) or 0) + int(count) lines[i] = json.dumps(ev, ensure_ascii=False) merged = True break if not merged: - lines.append(json.dumps( - {"t": "hit", "doc": doc_path, "at": today, "n": int(count)}, - ensure_ascii=False, - )) + lines.append( + json.dumps( + {"t": "hit", "doc": doc_path, "at": today, "n": int(count)}, + ensure_ascii=False, + ) + ) _atomic_write_lines(path, lines) @@ -204,10 +217,12 @@ def adopted_docs_for_key(output_dir, capture_key: str) -> Set[str]: ev = json.loads(line) except (json.JSONDecodeError, ValueError): continue - if (isinstance(ev, dict) - and ev.get("t") == "adopted" - and ev.get("key") == capture_key - and isinstance(ev.get("doc"), str)): + if ( + isinstance(ev, dict) + and ev.get("t") == "adopted" + and ev.get("key") == capture_key + and isinstance(ev.get("doc"), str) + ): found.add(ev["doc"]) return found @@ -216,6 +231,7 @@ def adopted_docs_for_key(output_dir, capture_key: str) -> Set[str]: # Aggregation (pure in-memory fold + mtime snapshot cache) # --------------------------------------------------------------------------- # + def _dir_snapshot(dirs: List[Path]) -> tuple: """(dir-name, file-name, mtime_ns) for every *.jsonl in every dir.""" snap = [] @@ -274,10 +290,16 @@ def aggregate_usage(output_dir) -> Dict[str, dict]: doc = ev.get("doc") if not isinstance(doc, str) or not doc: continue - entry = usage.setdefault(doc, { - "hits": 0, "last_hit": None, "first_hit": None, - "adopted_keys": set(), "hit_days": set(), - }) + entry = usage.setdefault( + doc, + { + "hits": 0, + "last_hit": None, + "first_hit": None, + "adopted_keys": set(), + "hit_days": set(), + }, + ) t = ev.get("t") if t == "hit": try: diff --git a/codewiki/mcp/tools/watch.py b/codewiki/mcp/tools/watch.py index 886b57a..935ddf7 100644 --- a/codewiki/mcp/tools/watch.py +++ b/codewiki/mcp/tools/watch.py @@ -176,11 +176,15 @@ def _incremental_refresh( opts = analyze_options or {} _tmp = Path(tempfile.mkdtemp(prefix="codewiki_watch_")) config = Config( - repo_path=str(repo_path), output_dir=str(_tmp), + repo_path=str(repo_path), + output_dir=str(_tmp), dependency_graph_dir=str(_tmp / "dependency_graphs"), - docs_dir=str(output_dir), max_depth=MAX_DEPTH, - llm_base_url="not-needed", llm_api_key="not-needed", - main_model="unused", cluster_model="unused", + docs_dir=str(output_dir), + max_depth=MAX_DEPTH, + llm_base_url="not-needed", + llm_api_key="not-needed", + main_model="unused", + cluster_model="unused", ) ai: Dict[str, Any] = {"doc_type": "design"} if opts.get("include_patterns"): @@ -261,13 +265,21 @@ def _incremental_refresh( metas: Dict[str, ComponentMeta] = {} for comp_id, node in components.items(): metas[comp_id] = ComponentMeta( - id=node.id, name=node.name, component_type=node.component_type, - file_path=node.file_path, relative_path=node.relative_path, - start_line=node.start_line, end_line=node.end_line, - language=(node.language or "").strip() or "unknown", depends_on=node.depends_on, - node_type=node.node_type, base_classes=node.base_classes, - class_name=node.class_name, display_name=node.display_name, - qualified_name=node.qualified_name, has_docstring=node.has_docstring, + id=node.id, + name=node.name, + component_type=node.component_type, + file_path=node.file_path, + relative_path=node.relative_path, + start_line=node.start_line, + end_line=node.end_line, + language=(node.language or "").strip() or "unknown", + depends_on=node.depends_on, + node_type=node.node_type, + base_classes=node.base_classes, + class_name=node.class_name, + display_name=node.display_name, + qualified_name=node.qualified_name, + has_docstring=node.has_docstring, parameters=node.parameters, ) return metas, leaf_nodes, routes diff --git a/codewiki/mcp/tools/wiki_index.py b/codewiki/mcp/tools/wiki_index.py index 21a0e32..34c6b8d 100644 --- a/codewiki/mcp/tools/wiki_index.py +++ b/codewiki/mcp/tools/wiki_index.py @@ -84,9 +84,7 @@ def rebuild_index(output_dir: str | Path) -> None: index_path = wiki_dir / INDEX_FILENAME # --- Collect wiki pages by type --- - type_entries: Dict[str, List[Dict[str, str]]] = { - pt: [] for pt in PAGE_TYPE_DIRS - } + type_entries: Dict[str, List[Dict[str, str]]] = {pt: [] for pt in PAGE_TYPE_DIRS} # Root-level wiki/ files (doctrine.md, reading-guide.md, ...) — not a # subdirectory page type, but real pages that must appear in the index # so they are reachable (and not flagged as orphans). @@ -136,7 +134,9 @@ def rebuild_index(output_dir: str | Path) -> None: { "title": fm.get("title", note_file.stem), "type": fm.get("type", "note"), - "date": str(fm.get("date", "") or (fm.get("metadata") or {}).get("date", "")), + "date": str( + fm.get("date", "") or (fm.get("metadata") or {}).get("date", "") + ), "relpath": f"../{NOTES_DIR}/{note_file.name}", } ) @@ -249,10 +249,7 @@ def append_log( with _log_create_lock: if not log_path.exists(): header = ( - "# 操作日志\n" - "\n" - "> 按日期倒序分组的操作记录，由系统自动维护（OKF v0.2 §9 格式）\n" - "\n" + "# 操作日志\n\n> 按日期倒序分组的操作记录，由系统自动维护（OKF v0.2 §9 格式）\n\n" ) try: log_path.write_text(header, encoding="utf-8") @@ -314,12 +311,20 @@ def _extract_doc_title_and_summary(filepath: Path) -> Tuple[str, str]: fm = _parse_note_frontmatter(filepath) # generic frontmatter parser fm_title = fm.get("title") fm_desc = fm.get("description") - if isinstance(fm_title, str) and fm_title.strip() and \ - isinstance(fm_desc, str) and fm_desc.strip(): + if ( + isinstance(fm_title, str) + and fm_title.strip() + and isinstance(fm_desc, str) + and fm_desc.strip() + ): return fm_title.strip(), fm_desc.strip()[:120] - title: Optional[str] = fm_title.strip() if isinstance(fm_title, str) and fm_title.strip() else None - summary: Optional[str] = fm_desc.strip()[:120] if isinstance(fm_desc, str) and fm_desc.strip() else None + title: Optional[str] = ( + fm_title.strip() if isinstance(fm_title, str) and fm_title.strip() else None + ) + summary: Optional[str] = ( + fm_desc.strip()[:120] if isinstance(fm_desc, str) and fm_desc.strip() else None + ) try: with open(filepath, encoding="utf-8", errors="replace") as f: for i, line in enumerate(f): @@ -429,9 +434,7 @@ def _render_index( parts.append("## 入门指引") parts.append("") for entry in root_entries: - parts.append( - f"* [{entry['title']}]({entry['relpath']}) - {entry['summary']}" - ) + parts.append(f"* [{entry['title']}]({entry['relpath']}) - {entry['summary']}") parts.append("") # Render each page type section (§8 bullet lists) @@ -442,9 +445,7 @@ def _render_index( parts.append(f"## {label}") parts.append("") for entry in entries: - parts.append( - f"* [{entry['title']}]({entry['relpath']}) - {entry['summary']}" - ) + parts.append(f"* [{entry['title']}]({entry['relpath']}) - {entry['summary']}") parts.append("") # Notes section @@ -453,9 +454,7 @@ def _render_index( parts.append("") for entry in note_entries: meta = f" ({entry['type']}, {entry['date']})" if entry.get("date") else "" - parts.append( - f"* [{entry['title']}]({entry['relpath']}) - {entry['type']}{meta}" - ) + parts.append(f"* [{entry['title']}]({entry['relpath']}) - {entry['type']}{meta}") parts.append("") return "\n".join(parts) diff --git a/codewiki/mcp/tools/wiki_lint.py b/codewiki/mcp/tools/wiki_lint.py index 1a5a560..5d936ec 100644 --- a/codewiki/mcp/tools/wiki_lint.py +++ b/codewiki/mcp/tools/wiki_lint.py @@ -23,13 +23,25 @@ # All available check names _ALL_CHECKS = { - "stale_refs", "undocumented", "broken_links", "cycles", "coverage", - "orphan_pages", "no_outlinks", "missing_aliases", "stale_sources", - "superseded_pages", "overview_stale", "unsupported_claims", - "isolated_components", "stale_notes", "note_clusters", + "stale_refs", + "undocumented", + "broken_links", + "cycles", + "coverage", + "orphan_pages", + "no_outlinks", + "missing_aliases", + "stale_sources", + "superseded_pages", + "overview_stale", + "unsupported_claims", + "isolated_components", + "stale_notes", + "note_clusters", "okf_conformance", # P2 (team-memory fusion): L2 scene block hygiene - "scenario_capacity", "scenario_orphan", + "scenario_capacity", + "scenario_orphan", # P1 B-line: hot-but-never-adopted notes (usage utility dimension) "low_adoption", } @@ -50,24 +62,43 @@ # OKF v0.2 §4/§5/§7 standard top-level fields (P2). Producer-private # extensions must live under ``metadata``; anything else at the top level # triggers an okf_conformance warning so new docs don't leak private keys. -_OKF_TOP_LEVEL_KEYS = frozenset({ - "type", "title", "description", "aliases", - "status", "verified", "stale_after", "generated", - "tags", "sources", "metadata", -}) +_OKF_TOP_LEVEL_KEYS = frozenset( + { + "type", + "title", + "description", + "aliases", + "status", + "verified", + "stale_after", + "generated", + "tags", + "sources", + "metadata", + } +) # Legacy top-level extensions that may still appear on older pages. They are # producer-private under OKF §4/§5 and should be folded under ``metadata`` # (migrate_okf.py --fold-private does this). They stay tolerated here — and # line-based consumers (wiki_index note date, lint note_clusters, cache.py # boost) keep reading them via the indented ``key: value`` rows — so folding # remains backwards-compatible. -_OKF_LEGACY_TOP_LEVEL_KEYS = frozenset({ - "severity", "origin", "root_cause", - "source_refs", "chunk_refs", - "related_modules", "related_components", "source_ref", - "summary", "keywords", "date", - "reject_reason", # knowledge_loop reject() 写入的拒绝原因（migrate_okf --fold-private 会折叠进 metadata） -}) +_OKF_LEGACY_TOP_LEVEL_KEYS = frozenset( + { + "severity", + "origin", + "root_cause", + "source_refs", + "chunk_refs", + "related_modules", + "related_components", + "source_ref", + "summary", + "keywords", + "date", + "reject_reason", # knowledge_loop reject() 写入的拒绝原因（migrate_okf --fold-private 会折叠进 metadata） + } +) # Regex patterns for markdown links _WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]\(([^\)]+\.md)\)") @@ -82,9 +113,11 @@ def _strip_code_blocks(content: str) -> str: `` `[text](x.md)` ``) must not be treated as real references by the linter, otherwise they produce false-positive broken-link errors. """ + # Replace fenced blocks with equivalent blank lines to preserve line numbering def _blank_fenced(m: re.Match) -> str: return "\n" * m.group(0).count("\n") + stripped = re.sub(r"(?ms)^[ \t]*(?:```|~~~).*?^[ \t]*(?:```|~~~)", _blank_fenced, content) # Drop inline code spans (`...`). `[^`\n]*` is single-line scoped: a lone # unmatched backtick in prose (e.g. a truncated code ref) must never swallow @@ -125,14 +158,14 @@ def _build_anchor_map(output_dir: Path) -> Dict[str, str]: for line in text[3:end].splitlines(): s = line.strip() if s.startswith("title:"): - t = s.split(":", 1)[1].strip().strip('"\'') + t = s.split(":", 1)[1].strip().strip("\"'") if t: title_to_rel[t.lower()] = rel title_to_rel[t.lower().replace(" ", "-")] = rel elif s.startswith("aliases:"): raw = s.split(":", 1)[1].strip().strip("[]") for a in raw.split(","): - a = a.strip().strip('"\'') + a = a.strip().strip("\"'") if a: title_to_rel[a.lower()] = rel return title_to_rel @@ -202,6 +235,7 @@ def _get_output_dir(session: Optional[SessionState], arguments: Dict) -> Optiona def _load_module_tree(output_dir: Path) -> Optional[dict]: """Load module_tree.json from output directory.""" from codewiki.src.config import meta_resolve + mt_path = Path(meta_resolve(output_dir, "module_tree.json")) if not mt_path.exists(): return None @@ -245,6 +279,7 @@ def _walk(tree: dict): # Individual checks # --------------------------------------------------------------------------- + def _check_stale_refs( output_dir: Path, module_tree: Optional[dict], @@ -254,9 +289,9 @@ def _check_stale_refs( if not module_tree: return issues - valid_modules = _get_all_module_names(module_tree) + _get_all_module_names(module_tree) # Recursively collect all .md files for valid_files set - valid_files = {f.name for f in output_dir.rglob("*.md")} + {f.name for f in output_dir.rglob("*.md")} for md_file in output_dir.rglob("*.md"): if _SCRATCH_DIR_NAMES.intersection(md_file.parts): @@ -283,32 +318,36 @@ def _check_stale_refs( # Resolve relative to source file's directory resolved = (md_file.parent / ref_file).resolve() if not resolved.exists(): - issues.append({ - "check": "stale_refs", - "severity": "error", - "message": f"Reference to non-existent file '{ref_file}' (module '{ref_name}')", - "file": str(md_file.relative_to(output_dir)), - "line": line_no, - "suggestion": f"Remove or update the reference to '{ref_name}'", - }) + issues.append( + { + "check": "stale_refs", + "severity": "error", + "message": f"Reference to non-existent file '{ref_file}' (module '{ref_name}')", + "file": str(md_file.relative_to(output_dir)), + "line": line_no, + "suggestion": f"Remove or update the reference to '{ref_name}'", + } + ) # Check simple [text](file.md) patterns (skip http links) for match in _MD_LINK_RE.finditer(line): - ref_text = match.group(1) + match.group(1) ref_file = match.group(2) if ref_file.startswith(("http://", "https://")): continue # Resolve relative to source file's directory resolved = (md_file.parent / ref_file).resolve() if not resolved.exists(): - issues.append({ - "check": "stale_refs", - "severity": "error", - "message": f"Broken link to '{ref_file}'", - "file": str(md_file.relative_to(output_dir)), - "line": line_no, - "suggestion": f"Update the link target or remove the reference", - }) + issues.append( + { + "check": "stale_refs", + "severity": "error", + "message": f"Broken link to '{ref_file}'", + "file": str(md_file.relative_to(output_dir)), + "line": line_no, + "suggestion": "Update the link target or remove the reference", + } + ) return issues @@ -347,14 +386,16 @@ def _check_broken_links( # Resolve relative to source file's directory target = (md_file.parent / file_part).resolve() if not target.exists(): - issues.append({ - "check": "broken_links", - "severity": "error", - "message": f"Link target '{ref_file}' does not exist", - "file": str(md_file.relative_to(output_dir)), - "line": line_no, - "suggestion": "Fix the link path or create the target file", - }) + issues.append( + { + "check": "broken_links", + "severity": "error", + "message": f"Link target '{ref_file}' does not exist", + "file": str(md_file.relative_to(output_dir)), + "line": line_no, + "suggestion": "Fix the link path or create the target file", + } + ) return issues @@ -382,17 +423,19 @@ def _check_undocumented( if count < threshold: break if comp_id not in documented: - issues.append({ - "check": "undocumented", - "severity": "warning", - "message": ( - f"High-impact component '{comp_id}' " - f"({count} dependents) has no documentation coverage" - ), - "component_id": comp_id, - "depended_by_count": count, - "suggestion": "Add this component to a module or create dedicated documentation", - }) + issues.append( + { + "check": "undocumented", + "severity": "warning", + "message": ( + f"High-impact component '{comp_id}' " + f"({count} dependents) has no documentation coverage" + ), + "component_id": comp_id, + "depended_by_count": count, + "suggestion": "Add this component to a module or create dedicated documentation", + } + ) return issues @@ -410,16 +453,19 @@ def _check_cycles( build_graph_from_components, detect_cycles, ) + graph = build_graph_from_components(components) cycles = detect_cycles(graph) for cycle in cycles[:10]: # cap at 10 cycles - issues.append({ - "check": "cycles", - "severity": "info", - "message": f"Circular dependency detected: {' → '.join(cycle[:5])}{'...' if len(cycle) > 5 else ''}", - "components": cycle, - "suggestion": "Consider refactoring to break the cycle (e.g. via interface or event pattern)", - }) + issues.append( + { + "check": "cycles", + "severity": "info", + "message": f"Circular dependency detected: {' → '.join(cycle[:5])}{'...' if len(cycle) > 5 else ''}", + "components": cycle, + "suggestion": "Consider refactoring to break the cycle (e.g. via interface or event pattern)", + } + ) except Exception as e: logger.warning("Cycle detection skipped: %s", e) @@ -443,6 +489,7 @@ def _check_isolated_components( build_graph_from_components, find_isolated_nodes, ) + graph = build_graph_from_components(components) isolated = find_isolated_nodes(graph) @@ -457,34 +504,40 @@ def _check_isolated_components( by_file: Dict[str, List[str]] = defaultdict(list) for comp_id in reported: meta = components.get(comp_id) - fpath = getattr(meta, "relative_path", "") or getattr(meta, "file_path", "") or "unknown" + fpath = ( + getattr(meta, "relative_path", "") or getattr(meta, "file_path", "") or "unknown" + ) name = getattr(meta, "name", comp_id) if meta else comp_id by_file[fpath].append(name) for fpath, names in sorted(by_file.items()): - issues.append({ - "check": "isolated_components", - "severity": "info", - "message": ( - f"{len(names)} isolated component(s) in {fpath}: " - f"{', '.join(names[:5])}{'...' if len(names) > 5 else ''}" - ), - "file": fpath, - "components": [f"{fpath}::{n}" for n in names], - "suggestion": ( - "These components have no dependency relationships. " - "Verify they are not dead code, or document why they exist " - "(e.g. plugin entry points, scripts, deprecated code)." - ), - }) + issues.append( + { + "check": "isolated_components", + "severity": "info", + "message": ( + f"{len(names)} isolated component(s) in {fpath}: " + f"{', '.join(names[:5])}{'...' if len(names) > 5 else ''}" + ), + "file": fpath, + "components": [f"{fpath}::{n}" for n in names], + "suggestion": ( + "These components have no dependency relationships. " + "Verify they are not dead code, or document why they exist " + "(e.g. plugin entry points, scripts, deprecated code)." + ), + } + ) if total > 20: - issues.append({ - "check": "isolated_components", - "severity": "info", - "message": f"... and {total - 20} more isolated components (showing first 20)", - "suggestion": "Run with component analysis to see the full list.", - }) + issues.append( + { + "check": "isolated_components", + "severity": "info", + "message": f"... and {total - 20} more isolated components (showing first 20)", + "suggestion": "Run with component analysis to see the full list.", + } + ) except Exception as e: logger.warning("Isolated component detection skipped: %s", e) @@ -507,18 +560,17 @@ def _check_coverage( covered = len(documented & set(components.keys())) pct = (covered / total * 100) if total > 0 else 0 - issues.append({ - "check": "coverage", - "severity": "info", - "message": f"Documentation coverage: {covered}/{total} components ({pct:.1f}%)", - "covered": covered, - "total": total, - "percentage": round(pct, 1), - "suggestion": ( - "Coverage is below 50%" if pct < 50 - else "Good coverage" - ), - }) + issues.append( + { + "check": "coverage", + "severity": "info", + "message": f"Documentation coverage: {covered}/{total} components ({pct:.1f}%)", + "covered": covered, + "total": total, + "percentage": round(pct, 1), + "suggestion": ("Coverage is below 50%" if pct < 50 else "Good coverage"), + } + ) # Per-module coverage def _walk(tree: dict): @@ -529,16 +581,18 @@ def _walk(tree: dict): mod_total = len(mod_comps) mod_pct = (mod_covered / mod_total * 100) if mod_total > 0 else 0 if mod_pct < 50: - issues.append({ - "check": "coverage", - "severity": "info", - "message": f"Module '{name}': {mod_covered}/{mod_total} components ({mod_pct:.0f}%)", - "module": name, - "covered": mod_covered, - "total": mod_total, - "percentage": round(mod_pct, 1), - "suggestion": "Consider adding more components to this module's documentation", - }) + issues.append( + { + "check": "coverage", + "severity": "info", + "message": f"Module '{name}': {mod_covered}/{mod_total} components ({mod_pct:.0f}%)", + "module": name, + "covered": mod_covered, + "total": mod_total, + "percentage": round(mod_pct, 1), + "suggestion": "Consider adding more components to this module's documentation", + } + ) children = info.get("children", {}) if isinstance(children, dict): _walk(children) @@ -552,6 +606,7 @@ def _walk(tree: dict): # LLM Wiki checks # --------------------------------------------------------------------------- + def _check_orphan_pages( output_dir: Path, anchor_map: Optional[Dict[str, str]] = None, @@ -605,13 +660,15 @@ def _check_orphan_pages( # Find pages with no incoming links for rel_path, md_file in all_pages.items(): if rel_path not in linked_targets: - issues.append({ - "check": "orphan_pages", - "severity": "warning", - "message": f"Page has no incoming links", - "file": rel_path, - "suggestion": "Add cross-references from related pages", - }) + issues.append( + { + "check": "orphan_pages", + "severity": "warning", + "message": "Page has no incoming links", + "file": rel_path, + "suggestion": "Add cross-references from related pages", + } + ) return issues @@ -646,13 +703,15 @@ def _check_no_outlinks( scan = _strip_code_blocks(content) targets = _collect_linked_targets(scan, md_file, output_dir, anchor_map) if not targets: - issues.append({ - "check": "no_outlinks", - "severity": "info", - "message": "Page has no outgoing links to other wiki pages", - "file": rel_path, - "suggestion": "Add cross-references to related pages for better navigation", - }) + issues.append( + { + "check": "no_outlinks", + "severity": "info", + "message": "Page has no outgoing links to other wiki pages", + "file": rel_path, + "suggestion": "Add cross-references to related pages for better navigation", + } + ) return issues @@ -682,27 +741,26 @@ def _check_missing_aliases( try: end = content.index("---", 3) fm = content[3:end] - has_aliases = any( - line.strip().startswith("aliases:") - for line in fm.splitlines() - ) + has_aliases = any(line.strip().startswith("aliases:") for line in fm.splitlines()) if not has_aliases: missing.append(rel_path) except (ValueError, IndexError): pass if missing: - issues.append({ - "check": "missing_aliases", - "severity": "info", - "message": ( - f"{len(missing)} page(s) lack 'aliases' in frontmatter; " - "adding alternate names improves search discoverability" - ), - "count": len(missing), - "files": missing, - "suggestion": "Add 'aliases: []' to frontmatter when generating docs", - }) + issues.append( + { + "check": "missing_aliases", + "severity": "info", + "message": ( + f"{len(missing)} page(s) lack 'aliases' in frontmatter; " + "adding alternate names improves search discoverability" + ), + "count": len(missing), + "files": missing, + "suggestion": "Add 'aliases: []' to frontmatter when generating docs", + } + ) return issues @@ -726,7 +784,8 @@ def _check_stale_sources( registry = _json.loads(reg_path.read_text(encoding="utf-8")) sources = registry.get("sources", {}) retracted_sources = { - name for name, info in sources.items() + name + for name, info in sources.items() if isinstance(info, dict) and info.get("status") == "retracted" } except (json.JSONDecodeError, OSError): @@ -750,13 +809,15 @@ def _check_stale_sources( for match in _SRC_REF_RE.finditer(content): src_name = match.group(1) if src_name in retracted_sources: - issues.append({ - "check": "stale_sources", - "severity": "warning", - "message": f"References retracted source '{src_name}'", - "file": rel_path, - "suggestion": f"Update or remove the reference to '{src_name}'", - }) + issues.append( + { + "check": "stale_sources", + "severity": "warning", + "message": f"References retracted source '{src_name}'", + "file": rel_path, + "suggestion": f"Update or remove the reference to '{src_name}'", + } + ) return issues @@ -791,21 +852,24 @@ def _check_superseded_pages( rel_path = str(md_file.relative_to(output_dir)) # Try to extract superseded_by superseded_by = "" - m = re.search(r"^superseded_by:\s*[\"']?(.+?)[\"']?\s*$", - fm_text, re.MULTILINE) + m = re.search( + r"^superseded_by:\s*[\"']?(.+?)[\"']?\s*$", fm_text, re.MULTILINE + ) if m: superseded_by = m.group(1) msg = "Page marked as superseded" if superseded_by: msg += f" (replaced by: {superseded_by})" - issues.append({ - "check": "superseded_pages", - "severity": "info", - "message": msg, - "file": rel_path, - "suggestion": "Consider archiving or removing this page" - + (f"; see '{superseded_by}'" if superseded_by else ""), - }) + issues.append( + { + "check": "superseded_pages", + "severity": "info", + "message": msg, + "file": rel_path, + "suggestion": "Consider archiving or removing this page" + + (f"; see '{superseded_by}'" if superseded_by else ""), + } + ) return issues @@ -833,13 +897,15 @@ def _check_overview_stale_lint( pass if overview_stale: - issues.append({ - "check": "overview_stale", - "severity": "warning", - "message": "overview.md references modules that have changed and may need updating", - "file": "overview.md", - "suggestion": "Review and update overview.md to reflect changes in referenced modules", - }) + issues.append( + { + "check": "overview_stale", + "severity": "warning", + "message": "overview.md references modules that have changed and may need updating", + "file": "overview.md", + "suggestion": "Review and update overview.md to reflect changes in referenced modules", + } + ) return issues @@ -905,19 +971,21 @@ def _check_unsupported_claims( ratio = unsupported / total_claims if ratio > threshold: rel_path = str(md_file.relative_to(output_dir)) - issues.append({ - "check": "unsupported_claims", - "severity": "warning", - "message": ( - f"{unsupported}/{total_claims} business assertions lack code evidence " - f"({ratio:.0%} > {threshold:.0%} threshold)" - ), - "file": rel_path, - "suggestion": ( - "Add '> Evidence: `` — ' lines after each assertion, " - "or mark unsupported assertions as [candidate]" - ), - }) + issues.append( + { + "check": "unsupported_claims", + "severity": "warning", + "message": ( + f"{unsupported}/{total_claims} business assertions lack code evidence " + f"({ratio:.0%} > {threshold:.0%} threshold)" + ), + "file": rel_path, + "suggestion": ( + "Add '> Evidence: `` — ' lines after each assertion, " + "or mark unsupported assertions as [candidate]" + ), + } + ) return issues @@ -926,6 +994,7 @@ def _check_unsupported_claims( # Note lifecycle checks (staleness + clustering) # --------------------------------------------------------------------------- + def _parse_note_frontmatter(note_path: Path) -> Optional[Dict[str, Any]]: """Parse a note's YAML frontmatter into a dict. Returns None on failure.""" try: @@ -951,7 +1020,7 @@ def _parse_note_frontmatter(note_path: Path) -> Optional[Dict[str, Any]]: except (json.JSONDecodeError, ValueError): fm[key] = value else: - fm[key] = value.strip('"\'') + fm[key] = value.strip("\"'") return fm @@ -980,11 +1049,12 @@ def _check_stale_notes( > hardcoded defaults (90/60) — dispatch passes no parameters, so bundles with a freshness block now actually get their configured windows. """ - from datetime import datetime, timedelta + from datetime import datetime issues: List[Dict[str, Any]] = [] from codewiki.src.config import NOTES_DIR + notes_dir = output_dir / NOTES_DIR if not notes_dir.is_dir(): return issues @@ -992,6 +1062,7 @@ def _check_stale_notes( # Freshness config from schema.yaml (fallback chain handled inside). try: from codewiki.mcp.tools.page_router import load_schema + schema = load_schema(str(output_dir)) except Exception: schema = {} @@ -1015,6 +1086,7 @@ def _check_stale_notes( hit_count_map: Dict[str, int] = {} # file_path -> hit_count (U2) try: from codewiki.mcp.tools import telemetry + for fp, entry in telemetry.aggregate_usage(output_dir).items(): hit_count_map[str(fp)] = int(entry.get("hits", 0) or 0) lh = entry.get("last_hit") @@ -1038,10 +1110,7 @@ def _check_stale_notes( continue rel_path = str(note_file.relative_to(output_dir)).replace("\\", "/") - last_hit_str = ( - retrieval_map.get(rel_path) - or retrieval_map.get(f"notes/{note_file.name}") - ) + last_hit_str = retrieval_map.get(rel_path) or retrieval_map.get(f"notes/{note_file.name}") verdict = evaluate_note_freshness(fm, cfg, today=today, last_hit=last_hit_str) if verdict["state"] != "due": @@ -1049,16 +1118,12 @@ def _check_stale_notes( due_date = verdict["due_date"] or "?" try: - overdue_days = ( - today - datetime.strptime(due_date, "%Y-%m-%d") - ).days + overdue_days = (today - datetime.strptime(due_date, "%Y-%m-%d")).days except (ValueError, TypeError): overdue_days = 0 title = fm.get("title", note_file.stem) note_type = fm.get("type", "general") - hit_count = hit_count_map.get( - rel_path, hit_count_map.get(f"notes/{note_file.name}", 0) - ) + hit_count = hit_count_map.get(rel_path, hit_count_map.get(f"notes/{note_file.name}", 0)) issue = { "check": "stale_notes", @@ -1072,7 +1137,7 @@ def _check_stale_notes( "file": rel_path, "suggestion": ( f"超过 {overdue_days} 天未验证。确认仍然准确用 " - f"confirm_note(note_file=\"{rel_path}\") 续期" + f'confirm_note(note_file="{rel_path}") 续期' f"（将按类型窗口刷新 stale_after），已过时用 reject_note 退役。" ), } @@ -1123,6 +1188,7 @@ def _check_low_adoption( # Config from schema.yaml (fallback chain handled below). try: from codewiki.mcp.tools.page_router import load_schema + schema = load_schema(str(output_dir)) except Exception: schema = {} @@ -1147,11 +1213,13 @@ def _param(name: str, default: int, override: Optional[int]) -> int: # Cold-start guard: no adopted events anywhere in the bundle → the # adoption signal is not in use yet, skip. from codewiki.mcp.tools.adoption import load_adoption_counts + adoption_counts = load_adoption_counts(output_dir) if not adoption_counts: return issues from codewiki.src.config import NOTES_DIR + notes_dir = output_dir / NOTES_DIR if not notes_dir.is_dir(): return issues @@ -1160,6 +1228,7 @@ def _param(name: str, default: int, override: Optional[int]) -> int: # stale_notes uses — one call carries hit_count / last_hit / adopted). try: from codewiki.mcp.tools import telemetry + usage_agg = telemetry.aggregate_usage(output_dir) except Exception: return issues @@ -1168,7 +1237,8 @@ def _param(name: str, default: int, override: Optional[int]) -> int: hit_map: Dict[str, Tuple[int, str]] = {} # file_path -> (hit_count, last_hit) for fp, entry in usage_agg.items(): hit_map[str(fp).replace("\\", "/")] = ( - int(entry.get("hits", 0) or 0), str(entry.get("last_hit") or ""), + int(entry.get("hits", 0) or 0), + str(entry.get("last_hit") or ""), ) cutoff = datetime.now() - timedelta(days=recent_days) @@ -1206,22 +1276,24 @@ def _param(name: str, default: int, override: Optional[int]) -> int: continue title = fm.get("title", note_file.stem) - issues.append({ - "check": "low_adoption", - "severity": "warning", - "message": ( - f"Note '{title}' was recalled {hit_count} times recently " - f"but adopted {adopted} time(s) — content is likely relevant " - f"but not actionable enough" - ), - "file": rel_path, - "suggestion": ( - "高频召回但零采纳：内容相关但可能不够 actionable。建议重写为更" - "可执行的形式（补充具体步骤/命令/预期结果），可用 " - "distill_conversation 产出草稿后 confirm_note，或用 " - f"edit_doc_file 直接更新 {rel_path}。" - ), - }) + issues.append( + { + "check": "low_adoption", + "severity": "warning", + "message": ( + f"Note '{title}' was recalled {hit_count} times recently " + f"but adopted {adopted} time(s) — content is likely relevant " + f"but not actionable enough" + ), + "file": rel_path, + "suggestion": ( + "高频召回但零采纳：内容相关但可能不够 actionable。建议重写为更" + "可执行的形式（补充具体步骤/命令/预期结果），可用 " + "distill_conversation 产出草稿后 confirm_note，或用 " + f"edit_doc_file 直接更新 {rel_path}。" + ), + } + ) return issues @@ -1239,6 +1311,7 @@ def _check_note_clusters( issues: List[Dict[str, Any]] = [] from codewiki.src.config import NOTES_DIR + notes_dir = output_dir / NOTES_DIR if not notes_dir.is_dir(): return issues @@ -1283,20 +1356,22 @@ def _check_note_clusters( if len(notes) > 5: titles += f" (+{len(notes) - 5} more)" - issues.append({ - "check": "note_clusters", - "severity": "info", - "message": ( - f"Module '{module}' has {len(notes)} {note_type} notes " - f"that may benefit from consolidation: {titles}" - ), - "file": notes[0]["file"], - "suggestion": ( - f"Use get_prompt('consolidate') for guidance on merging " - f"these {len(notes)} {note_type} notes into a single " - f"authoritative note for module '{module}'." - ), - }) + issues.append( + { + "check": "note_clusters", + "severity": "info", + "message": ( + f"Module '{module}' has {len(notes)} {note_type} notes " + f"that may benefit from consolidation: {titles}" + ), + "file": notes[0]["file"], + "suggestion": ( + f"Use get_prompt('consolidate') for guidance on merging " + f"these {len(notes)} {note_type} notes into a single " + f"authoritative note for module '{module}'." + ), + } + ) return issues @@ -1305,6 +1380,7 @@ def _check_note_clusters( # P2: L2 scene block hygiene (team-memory fusion 设计方案 §4.3.4) # --------------------------------------------------------------------------- + def _check_scenario_capacity( output_dir: Path, ) -> List[Dict[str, Any]]: @@ -1318,6 +1394,7 @@ def _check_scenario_capacity( try: from codewiki.mcp.tools.note_consolidation import _scan_scenarios from codewiki.mcp.tools.aggregation_state import read_config + live = _scan_scenarios(output_dir) max_scenes = read_config(output_dir)["max_scenarios"] except Exception: @@ -1325,30 +1402,34 @@ def _check_scenario_capacity( if not live: return issues if len(live) > max_scenes: - issues.append({ - "check": "scenario_capacity", - "severity": "error", - "message": ( - f"Scenario blocks exceed the cap: {len(live)}/{max_scenes}. " - "MERGE similar scenes (mark losers [DELETED]) before adding more." - ), - "file": "wiki/scenarios/", - "suggestion": ( - "Run consolidate_notes(mode='prepare') and follow the RED " - "capacity protocol: merge first, then re-submit." - ), - }) + issues.append( + { + "check": "scenario_capacity", + "severity": "error", + "message": ( + f"Scenario blocks exceed the cap: {len(live)}/{max_scenes}. " + "MERGE similar scenes (mark losers [DELETED]) before adding more." + ), + "file": "wiki/scenarios/", + "suggestion": ( + "Run consolidate_notes(mode='prepare') and follow the RED " + "capacity protocol: merge first, then re-submit." + ), + } + ) elif len(live) == max_scenes: - issues.append({ - "check": "scenario_capacity", - "severity": "warning", - "message": ( - f"Scenario blocks at capacity: {len(live)}/{max_scenes}. " - "Only UPDATE is allowed until a merge frees a slot." - ), - "file": "wiki/scenarios/", - "suggestion": "Prefer UPDATE/MERGE on the next consolidate_notes run.", - }) + issues.append( + { + "check": "scenario_capacity", + "severity": "warning", + "message": ( + f"Scenario blocks at capacity: {len(live)}/{max_scenes}. " + "Only UPDATE is allowed until a merge frees a slot." + ), + "file": "wiki/scenarios/", + "suggestion": "Prefer UPDATE/MERGE on the next consolidate_notes run.", + } + ) return issues @@ -1367,7 +1448,8 @@ def _check_scenario_orphan( issues: List[Dict[str, Any]] = [] try: from codewiki.mcp.tools.note_consolidation import ( - _scan_scenarios, _read_frontmatter, + _scan_scenarios, + _read_frontmatter, ) except Exception: return issues @@ -1380,6 +1462,7 @@ def _check_scenario_orphan( retrieval_map: Dict[str, str] = {} try: from codewiki.mcp.tools import telemetry + for fp, entry in telemetry.aggregate_usage(output_dir).items(): lh = entry.get("last_hit") if lh: @@ -1403,20 +1486,22 @@ def _check_scenario_orphan( recently_used = True # unparseable timestamp: be conservative if recently_used: continue - issues.append({ - "check": "scenario_orphan", - "severity": "info", - "message": ( - f"Scene block '{sc['title']}' has no source_notes provenance and " - f"has not been retrieved for {retrieval_gap_days}+ days — " - "consider reviewing, merging, or retiring it." - ), - "file": sc["file"], - "suggestion": ( - "Verify the block is still valid; retire via [DELETED] on the " - "next consolidate_notes run if superseded." - ), - }) + issues.append( + { + "check": "scenario_orphan", + "severity": "info", + "message": ( + f"Scene block '{sc['title']}' has no source_notes provenance and " + f"has not been retrieved for {retrieval_gap_days}+ days — " + "consider reviewing, merging, or retiring it." + ), + "file": sc["file"], + "suggestion": ( + "Verify the block is still valid; retire via [DELETED] on the " + "next consolidate_notes run if superseded." + ), + } + ) return issues @@ -1424,6 +1509,7 @@ def _check_scenario_orphan( # OKF v0.2 conformance (§11 / §12) # --------------------------------------------------------------------------- + def _check_okf_conformance( output_dir: Path, skip_notes_staleness: bool = False, @@ -1498,16 +1584,18 @@ def _check_okf_conformance( continue if not text.startswith("---") or text.find("---", 3) < 0: - issues.append({ - "check": "okf_conformance", - "severity": "error", - "message": "Missing YAML frontmatter (OKF v0.2 §11 requires a 'type' field)", - "file": rel_path, - "suggestion": ( - "Run `python scripts/migrate_okf.py ` to backfill " - "OKF frontmatter, or regenerate the page." - ), - }) + issues.append( + { + "check": "okf_conformance", + "severity": "error", + "message": "Missing YAML frontmatter (OKF v0.2 §11 requires a 'type' field)", + "file": rel_path, + "suggestion": ( + "Run `python scripts/migrate_okf.py ` to backfill " + "OKF frontmatter, or regenerate the page." + ), + } + ) continue # Prefer real YAML parsing (needed for nested generated/verified/sources), @@ -1516,6 +1604,7 @@ def _check_okf_conformance( fm: Optional[Dict[str, Any]] = None try: import yaml + end = text.find("---", 3) data = yaml.safe_load(text[3:end]) if isinstance(data, dict): @@ -1525,96 +1614,103 @@ def _check_okf_conformance( if fm is None: fm = _parse_note_frontmatter(md_file) if not fm: - issues.append({ - "check": "okf_conformance", - "severity": "error", - "message": "Frontmatter is not parseable YAML (OKF v0.2 §11)", - "file": rel_path, - "suggestion": "Fix the YAML frontmatter block manually or regenerate the page.", - }) + issues.append( + { + "check": "okf_conformance", + "severity": "error", + "message": "Frontmatter is not parseable YAML (OKF v0.2 §11)", + "file": rel_path, + "suggestion": "Fix the YAML frontmatter block manually or regenerate the page.", + } + ) continue # §4: type is the only required field page_type = fm.get("type") if not page_type or not str(page_type).strip(): - issues.append({ - "check": "okf_conformance", - "severity": "error", - "message": "Missing required 'type' field in frontmatter (OKF v0.2 §4)", - "file": rel_path, - "suggestion": ( - "Run `python scripts/migrate_okf.py ` to backfill " - "the type field, or regenerate the page." - ), - }) + issues.append( + { + "check": "okf_conformance", + "severity": "error", + "message": "Missing required 'type' field in frontmatter (OKF v0.2 §4)", + "file": rel_path, + "suggestion": ( + "Run `python scripts/migrate_okf.py ` to backfill " + "the type field, or regenerate the page." + ), + } + ) # P2: producer-private keys must not leak at the top level. OKF §4/§5 # standard fields plus the backward-compat legacy set are allowed; any # other key should be folded under `metadata:`. - _unknown_top = sorted( - set(fm) - _OKF_TOP_LEVEL_KEYS - _OKF_LEGACY_TOP_LEVEL_KEYS - ) + _unknown_top = sorted(set(fm) - _OKF_TOP_LEVEL_KEYS - _OKF_LEGACY_TOP_LEVEL_KEYS) if _unknown_top: - issues.append({ - "check": "okf_conformance", - "severity": "warning", - "message": ( - "Non-OKF top-level frontmatter key(s): " - + ", ".join(_unknown_top) - + " (OKF v0.2 §4/§5 — producer-private fields belong under `metadata:`)" - ), - "file": rel_path, - "suggestion": ( - "Fold these keys under a `metadata:` node, or regenerate " - "the page with the OKF frontmatter helper." - ), - }) - - # §5 status vocabulary - status_raw = fm.get("status") - status = str(status_raw).strip().lower() if status_raw else "" - if status and status not in _OKF_STATUSES: - mapped = _LEGACY_STATUS_MAP.get(status) - if mapped: - issues.append({ + issues.append( + { "check": "okf_conformance", "severity": "warning", "message": ( - f"Legacy status '{status}' — OKF v0.2 uses '{mapped}'" + "Non-OKF top-level frontmatter key(s): " + + ", ".join(_unknown_top) + + " (OKF v0.2 §4/§5 — producer-private fields belong under `metadata:`)" ), "file": rel_path, "suggestion": ( - "Run `python scripts/migrate_okf.py ` to migrate " - "legacy lifecycle statuses to the OKF v0.2 vocabulary." + "Fold these keys under a `metadata:` node, or regenerate " + "the page with the OKF frontmatter helper." ), - }) + } + ) + + # §5 status vocabulary + status_raw = fm.get("status") + status = str(status_raw).strip().lower() if status_raw else "" + if status and status not in _OKF_STATUSES: + mapped = _LEGACY_STATUS_MAP.get(status) + if mapped: + issues.append( + { + "check": "okf_conformance", + "severity": "warning", + "message": (f"Legacy status '{status}' — OKF v0.2 uses '{mapped}'"), + "file": rel_path, + "suggestion": ( + "Run `python scripts/migrate_okf.py ` to migrate " + "legacy lifecycle statuses to the OKF v0.2 vocabulary." + ), + } + ) else: - issues.append({ - "check": "okf_conformance", - "severity": "warning", - "message": ( - f"Unknown status '{status}' — expected one of " - f"draft/stable/deprecated (OKF v0.2 §5)" - ), - "file": rel_path, - "suggestion": "Set status to draft, stable, or deprecated.", - }) + issues.append( + { + "check": "okf_conformance", + "severity": "warning", + "message": ( + f"Unknown status '{status}' — expected one of " + f"draft/stable/deprecated (OKF v0.2 §5)" + ), + "file": rel_path, + "suggestion": "Set status to draft, stable, or deprecated.", + } + ) # §5 verified: mapping or list of mappings ({by, at, note?}) verified = fm.get("verified") if verified is not None: valid = isinstance(verified, dict) or ( - isinstance(verified, list) - and all(isinstance(v, dict) for v in verified) + isinstance(verified, list) and all(isinstance(v, dict) for v in verified) ) if not valid: - issues.append({ - "check": "okf_conformance", - "severity": "warning", - "message": "'verified' must be a mapping or a list of {by, at} mappings (OKF v0.2 §5)", - "file": rel_path, - "suggestion": "Use confirm_note to record verification events correctly.", - }) + issues.append( + { + "check": "okf_conformance", + "severity": "warning", + "message": "'verified' must be a mapping or a list of {by, at} mappings (OKF v0.2 §5)", + "file": rel_path, + "suggestion": "Use confirm_note to record verification events correctly.", + } + ) # §5 stale_after expiry stale_after = fm.get("stale_after") @@ -1628,16 +1724,18 @@ def _check_okf_conformance( if skip_notes_staleness and is_note: pass # handled by _check_stale_notes else: - issues.append({ - "check": "okf_conformance", - "severity": "warning", - "message": f"stale_after ({sa}) has passed — knowledge may be outdated", - "file": rel_path, - "suggestion": ( - "Verify the content is still accurate, then regenerate or " - "update the page to renew stale_after." - ), - }) + issues.append( + { + "check": "okf_conformance", + "severity": "warning", + "message": f"stale_after ({sa}) has passed — knowledge may be outdated", + "file": rel_path, + "suggestion": ( + "Verify the content is still accurate, then regenerate or " + "update the page to renew stale_after." + ), + } + ) # §12: wiki/index.md should declare okf_version index_path = output_dir / WIKI_DIR / INDEX_FILENAME @@ -1652,16 +1750,18 @@ def _check_okf_conformance( if end > 0 and re.search(r"^okf_version:", idx_text[3:end], re.MULTILINE): has_version = True if not has_version: - issues.append({ - "check": "okf_conformance", - "severity": "warning", - "message": "wiki/index.md does not declare okf_version (OKF v0.2 §12)", - "file": "wiki/index.md", - "suggestion": ( - "Regenerate the index, or run `python scripts/migrate_okf.py " - "` to add okf_version." - ), - }) + issues.append( + { + "check": "okf_conformance", + "severity": "warning", + "message": "wiki/index.md does not declare okf_version (OKF v0.2 §12)", + "file": "wiki/index.md", + "suggestion": ( + "Regenerate the index, or run `python scripts/migrate_okf.py " + "` to add okf_version." + ), + } + ) return issues @@ -1670,12 +1770,14 @@ def _check_okf_conformance( # Main handler # --------------------------------------------------------------------------- + def handle_lint_wiki( arguments: Dict[str, Any], store: SessionStore, ) -> str: """Run documentation health checks and return structured results.""" from codewiki.mcp.tools.workspace_result import resolve_session + session = resolve_session(arguments, store) checks = arguments.get("checks", ["all"]) @@ -1720,21 +1822,23 @@ def handle_lint_wiki( try: pre_stale = _check_stale_refs(output_dir, module_tree) if pre_stale and all( - Path(str(i.get("file", ""))).as_posix().endswith("wiki/index.md") - for i in pre_stale + Path(str(i.get("file", ""))).as_posix().endswith("wiki/index.md") for i in pre_stale ): from codewiki.mcp.tools.wiki_index import rebuild_index + rebuild_index(output_dir) module_tree = _load_module_tree(output_dir) except Exception as exc: # keep lint non-fatal - all_issues.append({ - "check": "stale_refs", - "severity": "error", - "message": f"fix=true rebuild failed: {exc}", - "file": "wiki/index.md", - "line": 1, - "suggestion": "Run the rebuild manually and re-lint.", - }) + all_issues.append( + { + "check": "stale_refs", + "severity": "error", + "message": f"fix=true rebuild failed: {exc}", + "file": "wiki/index.md", + "line": 1, + "suggestion": "Run the rebuild manually and re-lint.", + } + ) # Run selected checks if "stale_refs" in checks and output_dir: @@ -1750,12 +1854,11 @@ def handle_lint_wiki( try: import yaml from codewiki.src.config import SCHEMA_FILENAME + schema_path = output_dir / SCHEMA_FILENAME if schema_path.exists(): schema = yaml.safe_load(schema_path.read_text(encoding="utf-8")) - threshold = ( - schema.get("lint", {}).get("high_impact_threshold", 5) - ) + threshold = schema.get("lint", {}).get("high_impact_threshold", 5) except Exception: pass all_issues.extend(_check_undocumented(components, module_tree, threshold)) @@ -1815,10 +1918,12 @@ def handle_lint_wiki( all_issues.extend(_check_scenario_orphan(output_dir)) if "okf_conformance" in checks and output_dir: - all_issues.extend(_check_okf_conformance( - output_dir, - skip_notes_staleness=("stale_notes" in checks), - )) + all_issues.extend( + _check_okf_conformance( + output_dir, + skip_notes_staleness=("stale_notes" in checks), + ) + ) # Deduplicate: if a link is already reported as stale_refs, don't also # report it as broken_links (same file + line = same underlying problem). @@ -1828,7 +1933,8 @@ def handle_lint_wiki( if issue.get("check") == "stale_refs" } all_issues = [ - issue for issue in all_issues + issue + for issue in all_issues if not ( issue.get("check") == "broken_links" and (issue.get("file"), issue.get("line")) in stale_locations @@ -1842,8 +1948,7 @@ def handle_lint_wiki( # Filter by severity filtered = [ - issue for issue in all_issues - if _SEVERITY_ORDER.get(issue["severity"], 2) <= min_severity + issue for issue in all_issues if _SEVERITY_ORDER.get(issue["severity"], 2) <= min_severity ] # Sort: errors first, then warnings, then info @@ -1883,8 +1988,8 @@ def handle_lint_wiki( if output_dir: try: from codewiki.mcp.tools.wiki_index import append_log - append_log(str(output_dir), "lint_wiki", - f"检查完成: {len(filtered)} 个问题") + + append_log(str(output_dir), "lint_wiki", f"检查完成: {len(filtered)} 个问题") except Exception: pass diff --git a/codewiki/mcp/tools/wiki_search.py b/codewiki/mcp/tools/wiki_search.py index 346947c..875a340 100644 --- a/codewiki/mcp/tools/wiki_search.py +++ b/codewiki/mcp/tools/wiki_search.py @@ -18,11 +18,16 @@ from typing import Dict, Optional from codewiki.mcp.cache import ( - _K1, _B, _build_indexable_text, - _tokenize, _extract_snippet, - _load_ontology, _expand_with_ontology, + _K1, + _B, + _build_indexable_text, + _tokenize, + _extract_snippet, + _load_ontology, + _expand_with_ontology, _doc_authority, - compute_usage_heat, _usage_context, + compute_usage_heat, + _usage_context, ) logger = logging.getLogger(__name__) @@ -57,7 +62,7 @@ def _resolve_db_path(output_dir: Path) -> Optional[Path]: if not cand.is_absolute(): # relative → resolve against the repo root (= output_dir.parent # for the standard layout; falls back to output_dir itself) - cand = (od.parent / cand) + cand = od.parent / cand if cand.exists(): return cand except Exception: @@ -82,9 +87,7 @@ def _open_standalone_cache(output_dir: Path, *, readonly: bool = False): repo_path = db_path.parent.parent # .codewiki/analysis_cache.db → repo root cache = AnalysisCache(repo_path, db_path=db_path) # Verify search tables have data - r = cache.conn.execute( - "SELECT value FROM search_stats WHERE key='total_docs'" - ).fetchone() + r = cache.conn.execute("SELECT value FROM search_stats WHERE key='total_docs'").fetchone() if not r or int(r["value"]) == 0: cache.close() return None @@ -92,60 +95,104 @@ def _open_standalone_cache(output_dir: Path, *, readonly: bool = False): except Exception: return None + # ---- Legacy JSON index ---- + class _IndexData: def __init__(self): - self.version = 1; self.total_docs = 0; self.avg_doc_len = 0.0 - self.doc_freq: Dict[str, int] = {}; self.docs: Dict[str, Dict] = {} - self.built_at: float = 0.0 # T1a: build timestamp for mtime-sampling freshness + self.version = 1 + self.total_docs = 0 + self.avg_doc_len = 0.0 + self.doc_freq: Dict[str, int] = {} + self.docs: Dict[str, Dict] = {} + self.built_at: float = 0.0 # T1a: build timestamp for mtime-sampling freshness + def to_dict(self): - return {"version": self.version, "total_docs": self.total_docs, - "avg_doc_len": round(self.avg_doc_len,2), "doc_freq": self.doc_freq, - "docs": self.docs, - "built_at": self.built_at or time.time()} + return { + "version": self.version, + "total_docs": self.total_docs, + "avg_doc_len": round(self.avg_doc_len, 2), + "doc_freq": self.doc_freq, + "docs": self.docs, + "built_at": self.built_at or time.time(), + } + @classmethod def from_dict(cls, d): - i = cls(); i.version = d.get("version",1); i.total_docs = d.get("total_docs",0) - i.avg_doc_len = d.get("avg_doc_len",0.0); i.doc_freq = d.get("doc_freq",{}) - i.docs = d.get("docs",{}); i.built_at = float(d.get("built_at") or 0.0); return i + i = cls() + i.version = d.get("version", 1) + i.total_docs = d.get("total_docs", 0) + i.avg_doc_len = d.get("avg_doc_len", 0.0) + i.doc_freq = d.get("doc_freq", {}) + i.docs = d.get("docs", {}) + i.built_at = float(d.get("built_at") or 0.0) + return i + def _recompute(self): self.total_docs = len(self.docs) - tl = sum(d.get("doc_len",0) for d in self.docs.values()) + tl = sum(d.get("doc_len", 0) for d in self.docs.values()) self.avg_doc_len = tl / self.total_docs if self.total_docs else 0.0 df = {} for di in self.docs.values(): - for t in di.get("term_freq",{}): df[t] = df.get(t,0) + 1 + for t in di.get("term_freq", {}): + df[t] = df.get(t, 0) + 1 self.doc_freq = df + def upsert(self, fk, title, source, content, *, batch=False): tokens = _tokenize(_build_indexable_text(content)) - if not tokens: return + if not tokens: + return tf = {} - for t in tokens: tf[t] = tf.get(t,0) + 1 - self.docs[fk] = {"title": title, "source": source, "doc_len": len(tokens), - "term_freq": tf, "authority": _doc_authority(fk, source, content)} - if not batch: self._recompute() - def finalize(self): self._recompute() + for t in tokens: + tf[t] = tf.get(t, 0) + 1 + self.docs[fk] = { + "title": title, + "source": source, + "doc_len": len(tokens), + "term_freq": tf, + "authority": _doc_authority(fk, source, content), + } + if not batch: + self._recompute() + + def finalize(self): + self._recompute() + def remove(self, fk): - if fk in self.docs: del self.docs[fk]; self._recompute(); return True + if fk in self.docs: + del self.docs[fk] + self._recompute() + return True return False + def _index_path(od): """Search index lives in .meta/ to keep output_dir root clean.""" from codewiki.src.config import META_DIR + meta_path = Path(od) / META_DIR / _SEARCH_INDEX_FILENAME root_path = Path(od) / _SEARCH_INDEX_FILENAME # Prefer .meta/, fallback to root for backward compat (read-only) if meta_path.exists() or not root_path.exists(): return meta_path return root_path + + def _load_index(od): p = _index_path(od) - if not p.exists(): return _IndexData() - try: return _IndexData.from_dict(json.loads(p.read_text(encoding="utf-8"))) - except Exception: logger.warning("Failed to load search index"); return _IndexData() + if not p.exists(): + return _IndexData() + try: + return _IndexData.from_dict(json.loads(p.read_text(encoding="utf-8"))) + except Exception: + logger.warning("Failed to load search index") + return _IndexData() + + def _save_index(od, idx): - p = _index_path(od); tmp = p.with_suffix(".tmp") + p = _index_path(od) + tmp = p.with_suffix(".tmp") try: p.parent.mkdir(parents=True, exist_ok=True) tmp.write_text(json.dumps(idx.to_dict(), ensure_ascii=False), encoding="utf-8") @@ -153,32 +200,46 @@ def _save_index(od, idx): except Exception as e: logger.warning("Failed to save search index: %s", e) if tmp.exists(): - try: tmp.unlink() - except OSError: pass + try: + tmp.unlink() + except OSError: + pass + def _read_doc(fp: Path): try: ct = fp.read_text(encoding="utf-8", errors="replace") - if "", - "", - ]) + lines.extend( + [ + "", + "", + ] + ) # --- Services (compact) --- - lines.extend([ - "## Services", - "", - "| Service | Path | Languages | Components | Wiki |", - "|---------|------|-----------|------------|------|", - ]) + lines.extend( + [ + "## Services", + "", + "| Service | Path | Languages | Components | Wiki |", + "|---------|------|-----------|------------|------|", + ] + ) for r in repo_results: name = r["name"] @@ -227,21 +247,21 @@ def _generate_overview( except ValueError: wiki_rel = r["relative_path"] + "/repowiki" wiki_link = f"[wiki]({wiki_rel}/wiki/)" if r.get("has_overview") else f"[wiki]({wiki_rel}/)" - lines.append( - f"| {name} | `{rel_path}` | {languages} | {components} | {wiki_link} |" - ) + lines.append(f"| {name} | `{rel_path}` | {languages} | {components} | {wiki_link} |") lines.append("") # --- Infra services (ports) --- infra_services = _load_infra_services(output_dir) if infra_services: - lines.extend([ - "## Infrastructure Services", - "", - "| Service | Type | Port(s) |", - "|---------|------|---------|", - ]) + lines.extend( + [ + "## Infrastructure Services", + "", + "| Service | Type | Port(s) |", + "|---------|------|---------|", + ] + ) for svc_name, svc_info in infra_services.items(): svc_type = svc_info.get("type", "unknown") ports = svc_info.get("ports", []) @@ -253,18 +273,22 @@ def _generate_overview( if cross_service_info and cross_service_info.get("cross_service_md"): lines.append(cross_service_info["cross_service_md"]) else: - lines.extend([ - "## Cross-Service Relationships", - "", - "_No cross-service API calls detected automatically._", - "", - ]) + lines.extend( + [ + "## Cross-Service Relationships", + "", + "_No cross-service API calls detected automatically._", + "", + ] + ) # --- Per-repo overview links --- - lines.extend([ - "## Service Overviews", - "", - ]) + lines.extend( + [ + "## Service Overviews", + "", + ] + ) for r in repo_results: name = r["name"] @@ -329,26 +353,30 @@ def _handle_monorepo_fallback( ws_workspace = SessionWorkspace(str(workspace_path), workspace_session.session_id) workspace_session.workspace = ws_workspace - return json.dumps({ - "mode": "monorepo", - "workspace_session_id": workspace_session.session_id, - "workspace_path": str(workspace_path), - "output_dir": str(repo_output_dir), - "explanation": ( - "No sub-repos with individual .git directories were found. " - "The workspace root is itself a git repository (monorepo). " - "Cross-service analysis used analyze_repo's single-repo route " - "detection (sub-service discovery via docker-compose, Dockerfiles, " - "build manifests, convention directories) instead of multi-repo matching." - ), - "analyze_repo_result": result, - "cross_service": { - "total_routes": cross_service.get("total_routes", 0), - "total_links": cross_service.get("total_links", 0), - "total_unmatched": cross_service.get("total_unmatched", 0), - "sub_services": cross_service.get("sub_services", []), + return json.dumps( + { + "mode": "monorepo", + "workspace_session_id": workspace_session.session_id, + "workspace_path": str(workspace_path), + "output_dir": str(repo_output_dir), + "explanation": ( + "No sub-repos with individual .git directories were found. " + "The workspace root is itself a git repository (monorepo). " + "Cross-service analysis used analyze_repo's single-repo route " + "detection (sub-service discovery via docker-compose, Dockerfiles, " + "build manifests, convention directories) instead of multi-repo matching." + ), + "analyze_repo_result": result, + "cross_service": { + "total_routes": cross_service.get("total_routes", 0), + "total_links": cross_service.get("total_links", 0), + "total_unmatched": cross_service.get("total_unmatched", 0), + "sub_services": cross_service.get("sub_services", []), + }, }, - }, indent=2, ensure_ascii=False) + indent=2, + ensure_ascii=False, + ) def handle_analyze_workspace( @@ -386,11 +414,13 @@ def handle_analyze_workspace( # to analyze_repo which already handles sub-service detection. if (workspace_path / ".git").exists(): return _handle_monorepo_fallback(workspace_path, output_dir, store) - return json.dumps({ - "error": f"No git repositories found in {workspace_path}", - "hint": "Make sure each sub-project has its own .git directory, " - "or that the workspace root is itself a git repository (monorepo).", - }) + return json.dumps( + { + "error": f"No git repositories found in {workspace_path}", + "hint": "Make sure each sub-project has its own .git directory, " + "or that the workspace root is itself a git repository (monorepo).", + } + ) # Analyze each repo from codewiki.mcp.tools.analysis import handle_analyze_repo @@ -413,7 +443,9 @@ def handle_analyze_workspace( # Read summary.json for richer info (path comes from analyze_repo result) summary = {} - summary_path = Path(result.get("files", {}).get("summary") or (repo_output_dir / "summary.json")) + summary_path = Path( + result.get("files", {}).get("summary") or (repo_output_dir / "summary.json") + ) if summary_path.exists(): try: summary = json.loads(summary_path.read_text(encoding="utf-8")) @@ -421,17 +453,24 @@ def handle_analyze_workspace( pass stats = result.get("stats") or {} - repo_results.append({ - "name": repo_path.name, - "relative_path": str(repo_path.relative_to(workspace_path)), - "path": str(repo_path), - "output_dir": str(repo_output_dir), - "session_id": result.get("session_id"), - "total_components": stats.get("total_components", summary.get("total_components", 0)), - "total_leaf_nodes": stats.get("total_leaf_nodes", summary.get("total_leaf_nodes", 0)), - "languages": stats.get("languages", summary.get("languages", {})), - "has_overview": (repo_output_dir / "overview.md").exists() or (repo_output_dir / "wiki" / "overview.md").exists(), - }) + repo_results.append( + { + "name": repo_path.name, + "relative_path": str(repo_path.relative_to(workspace_path)), + "path": str(repo_path), + "output_dir": str(repo_output_dir), + "session_id": result.get("session_id"), + "total_components": stats.get( + "total_components", summary.get("total_components", 0) + ), + "total_leaf_nodes": stats.get( + "total_leaf_nodes", summary.get("total_leaf_nodes", 0) + ), + "languages": stats.get("languages", summary.get("languages", {})), + "has_overview": (repo_output_dir / "overview.md").exists() + or (repo_output_dir / "wiki" / "overview.md").exists(), + } + ) except Exception as e: logger.error("Failed to analyze %s: %s", repo_path.name, e) errors.append({"repo": repo_path.name, "error": str(e)}) @@ -440,14 +479,19 @@ def handle_analyze_workspace( cross_service_info = {} try: cross_service_info = _run_cross_service_analysis( - workspace_path, output_dir, repo_results, + workspace_path, + output_dir, + repo_results, ) except Exception as e: logger.warning("Cross-service analysis failed: %s", e) # Generate workspace overview.md (with cross-service topology) overview_path = _generate_overview( - workspace_path.name, output_dir, repo_results, cross_service_info, + workspace_path.name, + output_dir, + repo_results, + cross_service_info, ) # Create lightweight workspace session for ingest_note / query_wiki @@ -460,16 +504,20 @@ def handle_analyze_workspace( ws_workspace = SessionWorkspace(str(workspace_path), workspace_session.session_id) workspace_session.workspace = ws_workspace - return json.dumps({ - "workspace_session_id": workspace_session.session_id, - "workspace_path": str(workspace_path), - "overview_path": str(overview_path), - "repos_analyzed": len(repo_results), - "repos": repo_results, - "cross_service": { - "total_routes": cross_service_info.get("total_routes", 0), - "total_links": cross_service_info.get("total_links", 0), - "total_unmatched": cross_service_info.get("total_unmatched", 0), + return json.dumps( + { + "workspace_session_id": workspace_session.session_id, + "workspace_path": str(workspace_path), + "overview_path": str(overview_path), + "repos_analyzed": len(repo_results), + "repos": repo_results, + "cross_service": { + "total_routes": cross_service_info.get("total_routes", 0), + "total_links": cross_service_info.get("total_links", 0), + "total_unmatched": cross_service_info.get("total_unmatched", 0), + }, + "errors": errors if errors else None, }, - "errors": errors if errors else None, - }, indent=2, ensure_ascii=False) + indent=2, + ensure_ascii=False, + ) diff --git a/codewiki/mcp/tools/workspace_result.py b/codewiki/mcp/tools/workspace_result.py index d8e6cf4..5d48cff 100644 --- a/codewiki/mcp/tools/workspace_result.py +++ b/codewiki/mcp/tools/workspace_result.py @@ -16,7 +16,6 @@ from __future__ import annotations -import json import logging from pathlib import Path from typing import Any, Dict, Optional @@ -51,7 +50,11 @@ def resolve_session( return store.get(session_id) if repo_path: - rp = str(Path(repo_path).expanduser().resolve()) if Path(repo_path).is_absolute() else str((Path.cwd() / repo_path).expanduser().resolve()) + rp = ( + str(Path(repo_path).expanduser().resolve()) + if Path(repo_path).is_absolute() + else str((Path.cwd() / repo_path).expanduser().resolve()) + ) return store.find_or_restore(rp) return None diff --git a/codewiki/mcp/workspace.py b/codewiki/mcp/workspace.py index c79c993..235e3f1 100644 --- a/codewiki/mcp/workspace.py +++ b/codewiki/mcp/workspace.py @@ -76,7 +76,9 @@ def write_json(self, name: str, data: Any, *, compact: bool = False) -> Path: """ p = self.root / name if compact: - p.write_text(json.dumps(data, separators=(",", ":"), ensure_ascii=False), encoding="utf-8") + p.write_text( + json.dumps(data, separators=(",", ":"), ensure_ascii=False), encoding="utf-8" + ) else: p.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") return p @@ -93,7 +95,6 @@ def write_component_source( p.write_text(header + source, encoding="utf-8") return p - def write_text(self, name: str, data: str) -> Path: """Write arbitrary text to a workspace file and return the path.""" p = self.root / name diff --git a/codewiki/run_web_app.py b/codewiki/run_web_app.py index 940a1d8..c88ba2c 100644 --- a/codewiki/run_web_app.py +++ b/codewiki/run_web_app.py @@ -7,10 +7,10 @@ import sys # Add src directory to Python path -src_dir = os.path.join(os.path.dirname(__file__), 'src') +src_dir = os.path.join(os.path.dirname(__file__), "src") sys.path.insert(0, src_dir) from fe.web_app import main if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/codewiki/src/__init__.py b/codewiki/src/__init__.py index 9a6e527..cc5a3b9 100644 --- a/codewiki/src/__init__.py +++ b/codewiki/src/__init__.py @@ -1,2 +1 @@ """CodeWiki backend and frontend modules.""" - diff --git a/codewiki/src/be/__init__.py b/codewiki/src/be/__init__.py index f769618..f85bf77 100644 --- a/codewiki/src/be/__init__.py +++ b/codewiki/src/be/__init__.py @@ -1,2 +1 @@ """CodeWiki backend modules for documentation generation.""" - diff --git a/codewiki/src/be/agent_tools/__init__.py b/codewiki/src/be/agent_tools/__init__.py index b828275..3d2eb71 100644 --- a/codewiki/src/be/agent_tools/__init__.py +++ b/codewiki/src/be/agent_tools/__init__.py @@ -1,2 +1 @@ """Agent tools for backend processing.""" - diff --git a/codewiki/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py index 6f2c469..cd000e3 100644 --- a/codewiki/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -2,6 +2,7 @@ from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.config import Config + @dataclass class CodeWikiDeps: absolute_docs_path: str @@ -14,4 +15,4 @@ class CodeWikiDeps: max_depth: int current_depth: int config: Config # LLM configuration - custom_instructions: str = None \ No newline at end of file + custom_instructions: str = None diff --git a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py index 5c1a738..88f3947 100644 --- a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -9,25 +9,24 @@ from codewiki.src.be.cluster_modules import format_potential_core_components import logging -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) async def generate_sub_module_documentation( - ctx: RunContext[CodeWikiDeps], - sub_module_specs: dict[str, list[str]] + ctx: RunContext[CodeWikiDeps], sub_module_specs: dict[str, list[str]] ) -> str: """Delegate documentation generation of sub-modules to sub-agents. Each sub-module will be documented separately. Args: - sub_module_specs: A dictionary mapping sub-module names to their core component IDs. + sub_module_specs: A dictionary mapping sub-module names to their core component IDs. Example: {"authentication": ["auth_handler.py::AuthHandler", "auth_middleware.py::verify_token"], "database": ["db_client.py::DBClient", "models.py::UserModel"]} Each key is a descriptive sub-module name, and the value is a list of component IDs from the current module's core components that belong to that sub-module. """ deps = ctx.deps previous_module_name = deps.current_module_name - + # Create fallback models from config fallback_models = create_fallback_models(deps.config) @@ -37,31 +36,44 @@ async def generate_sub_module_documentation( value = value[key]["children"] for sub_module_name, core_component_ids in sub_module_specs.items(): value[sub_module_name] = {"components": core_component_ids, "children": {}} - - for sub_module_name, core_component_ids in sub_module_specs.items(): + for sub_module_name, core_component_ids in sub_module_specs.items(): # Create visual indentation for nested modules indent = " " * deps.current_depth arrow = "└─" if deps.current_depth > 0 else "→" logger.info(f"{indent}{arrow} Generating documentation for sub-module: {sub_module_name}") - num_tokens = count_tokens(format_potential_core_components(core_component_ids, ctx.deps.components)[-1]) - - if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module: + num_tokens = count_tokens( + format_potential_core_components(core_component_ids, ctx.deps.components)[-1] + ) + + if ( + is_complex_module(ctx.deps.components, core_component_ids) + and ctx.deps.current_depth < ctx.deps.max_depth + and num_tokens >= ctx.deps.config.max_token_per_leaf_module + ): sub_agent = Agent( model=fallback_models, name=sub_module_name, deps_type=CodeWikiDeps, - system_prompt=SYSTEM_PROMPT.format(module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions), - tools=[read_code_components_tool, str_replace_editor_tool, generate_sub_module_documentation_tool], + system_prompt=SYSTEM_PROMPT.format( + module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions + ), + tools=[ + read_code_components_tool, + str_replace_editor_tool, + generate_sub_module_documentation_tool, + ], ) else: sub_agent = Agent( model=fallback_models, name=sub_module_name, deps_type=CodeWikiDeps, - system_prompt=LEAF_SYSTEM_PROMPT.format(module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions), + system_prompt=LEAF_SYSTEM_PROMPT.format( + module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions + ), tools=[read_code_components_tool, str_replace_editor_tool], ) @@ -71,14 +83,14 @@ async def generate_sub_module_documentation( # log the current module tree # print(f"Current module tree: {json.dumps(deps.module_tree, indent=4)}") - result = await sub_agent.run( + await sub_agent.run( format_user_prompt( module_name=deps.current_module_name, core_component_ids=core_component_ids, components=ctx.deps.components, module_tree=ctx.deps.module_tree, ), - deps=ctx.deps + deps=ctx.deps, ) # remove the sub-module name from the path to current module and the module tree @@ -92,7 +104,7 @@ async def generate_sub_module_documentation( generate_sub_module_documentation_tool = Tool( - function=generate_sub_module_documentation, - name="generate_sub_module_documentation", - takes_ctx=True + function=generate_sub_module_documentation, + name="generate_sub_module_documentation", + takes_ctx=True, ) diff --git a/codewiki/src/be/agent_tools/read_code_components.py b/codewiki/src/be/agent_tools/read_code_components.py index 93c1344..a2494cc 100644 --- a/codewiki/src/be/agent_tools/read_code_components.py +++ b/codewiki/src/be/agent_tools/read_code_components.py @@ -15,8 +15,16 @@ async def read_code_components(ctx: RunContext[CodeWikiDeps], component_ids: lis if component_id not in ctx.deps.components: results.append(f"# Component {component_id} not found") else: - results.append(f"# Component {component_id}:\n{ctx.deps.components[component_id].source_code.strip()}\n\n") + results.append( + f"# Component {component_id}:\n{ctx.deps.components[component_id].source_code.strip()}\n\n" + ) return "\n".join(results) -read_code_components_tool = Tool(function=read_code_components, name="read_code_components", description="Read the code of a given list of component ids", takes_ctx=True) \ No newline at end of file + +read_code_components_tool = Tool( + function=read_code_components, + name="read_code_components", + description="Read the code of a given list of component ids", + takes_ctx=True, +) diff --git a/codewiki/src/be/backend.py b/codewiki/src/be/backend.py index ce1e017..9491355 100644 --- a/codewiki/src/be/backend.py +++ b/codewiki/src/be/backend.py @@ -62,6 +62,8 @@ def get_backend(config) -> "LLMBackend": provider = getattr(config, "provider", "openai-compatible") if is_caw_provider(provider): from codewiki.src.be.caw_backend import CawBackend + return CawBackend(config) from codewiki.src.be.pydantic_ai_backend import PydanticAIBackend + return PydanticAIBackend(config) diff --git a/codewiki/src/be/caw_backend.py b/codewiki/src/be/caw_backend.py index fb939a0..dfd9768 100644 --- a/codewiki/src/be/caw_backend.py +++ b/codewiki/src/be/caw_backend.py @@ -223,7 +223,8 @@ def _run_module_agent_sync( from codewiki.src.be.caw_toolkit import CawToolKit # local import to avoid cycles config = self._config - from codewiki.src.config import MODULE_TREE_FILENAME, meta_resolve + from codewiki.src.config import meta_resolve + module_tree_path = meta_resolve(working_dir, MODULE_TREE_FILENAME) if module_tree is None: module_tree = file_manager.load_json(module_tree_path) @@ -247,16 +248,16 @@ def _run_module_agent_sync( # agent call per sub-spec even when a single leaf write would suffice. # See generate_sub_module_documentation_tool for the pydantic-ai # equivalent. - _, components_with_code = format_potential_core_components( - core_component_ids, components - ) + _, components_with_code = format_potential_core_components(core_component_ids, components) num_tokens = count_tokens(components_with_code) can_delegate = ( is_complex_module(components, core_component_ids) and start_depth < config.max_depth and num_tokens >= config.max_token_per_leaf_module ) - logger.info(f"Module {module_name} can delegate: {can_delegate} - is_complex_module: {is_complex_module(components, core_component_ids)} - start_depth: {start_depth} - num_tokens: {num_tokens} - max_depth: {config.max_depth} - max_token_per_leaf_module: {config.max_token_per_leaf_module}") + logger.info( + f"Module {module_name} can delegate: {can_delegate} - is_complex_module: {is_complex_module(components, core_component_ids)} - start_depth: {start_depth} - num_tokens: {num_tokens} - max_depth: {config.max_depth} - max_token_per_leaf_module: {config.max_token_per_leaf_module}" + ) if can_delegate: system_prompt = format_system_prompt(module_name, custom_instructions) diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py index 37ea4e1..84c0471 100644 --- a/codewiki/src/be/caw_toolkit.py +++ b/codewiki/src/be/caw_toolkit.py @@ -107,8 +107,7 @@ async def read_code_components(self, component_ids: list[str]) -> str: results.append(f"# Component {cid} not found") else: results.append( - f"# Component {cid}:\n" - f"{self._deps.components[cid].source_code.strip()}\n\n" + f"# Component {cid}:\n{self._deps.components[cid].source_code.strip()}\n\n" ) return "\n".join(results) @@ -253,9 +252,7 @@ async def generate_sub_module_documentation( # event loop stays responsive while sub-agents run. A heartbeat task # emits MCP progress notifications so the CLI does not treat the long # tool call as a stalled / cancelled invocation. - work = asyncio.create_task( - asyncio.to_thread(self._run_sub_modules, sub_module_specs) - ) + work = asyncio.create_task(asyncio.to_thread(self._run_sub_modules, sub_module_specs)) heartbeat = asyncio.create_task(_heartbeat(ctx, work)) try: return await work @@ -283,7 +280,9 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: for sub_name, core_ids in sub_module_specs.items(): indent = " " * deps.current_depth arrow = "└─" if deps.current_depth > 0 else "→" - logger.info("%s%s Generating documentation for sub-module: %s", indent, arrow, sub_name) + logger.info( + "%s%s Generating documentation for sub-module: %s", indent, arrow, sub_name + ) deps.current_module_name = sub_name deps.path_to_current_module.append(sub_name) diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py index 61771b2..1bee4a9 100644 --- a/codewiki/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -2,6 +2,7 @@ from collections import defaultdict import logging import traceback + logger = logging.getLogger(__name__) from codewiki.src.be.dependency_analyzer.models.core import Node @@ -13,7 +14,9 @@ Completer = Callable[[str], str] -def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]: +def format_potential_core_components( + leaf_nodes: List[str], components: Dict[str, Node] +) -> tuple[str, str]: """ Format the potential core components into a string that can be used in the prompt. """ @@ -24,8 +27,8 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str valid_leaf_nodes.append(leaf_node) else: logger.warning(f"Skipping invalid leaf node '{leaf_node}' - not found in components") - - #group leaf nodes by file + + # group leaf nodes by file leaf_nodes_by_file = defaultdict(list) for leaf_node in valid_leaf_nodes: leaf_nodes_by_file[components[leaf_node].relative_path].append(leaf_node) @@ -43,9 +46,7 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str return potential_core_components, potential_core_components_with_code -def get_clustering_input_token_count( - leaf_nodes: List[str], components: Dict[str, Node] -) -> int: +def get_clustering_input_token_count(leaf_nodes: List[str], components: Dict[str, Node]) -> int: """Count the tokens used to decide whether a module needs clustering.""" _, potential_core_components_with_code = format_potential_core_components( leaf_nodes, components @@ -97,7 +98,9 @@ def cluster_modules( ) return {} - prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) + prompt = format_cluster_prompt( + potential_core_components, current_module_tree, current_module_name + ) logger.info( "Requesting LLM module clustering for %s because %d tokens exceed the %d-token threshold.", module_label, @@ -109,7 +112,7 @@ def cluster_modules( else: response = call_llm(prompt, config, model=config.cluster_model) - #parse the response + # parse the response try: if "" not in response or "" not in response: logger.warning( @@ -119,14 +122,16 @@ def cluster_modules( response[:200], ) return {} - - response_content = response.split("")[1].split("")[0] + + response_content = response.split("")[1].split("")[ + 0 + ] module_tree = eval(response_content) - + if not isinstance(module_tree, dict): logger.error(f"Invalid module tree format - expected dict, got {type(module_tree)}") return {} - + except Exception as e: logger.warning( "Failed to parse LLM clustering response for %s; falling back to " @@ -166,15 +171,17 @@ def cluster_modules( for module_name, module_info in module_tree.items(): sub_leaf_nodes = module_info.get("components", []) - + # Filter sub_leaf_nodes to ensure they exist in components valid_sub_leaf_nodes = [] for node in sub_leaf_nodes: if node in components: valid_sub_leaf_nodes.append(node) else: - logger.warning(f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components") - + logger.warning( + f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components" + ) + current_module_path.append(module_name) module_info["children"] = {} module_info["children"] = cluster_modules( diff --git a/codewiki/src/be/dependency_analyzer/__init__.py b/codewiki/src/be/dependency_analyzer/__init__.py index 9d1bbf8..1d1dff8 100644 --- a/codewiki/src/be/dependency_analyzer/__init__.py +++ b/codewiki/src/be/dependency_analyzer/__init__.py @@ -10,27 +10,33 @@ """ from codewiki.src.be.dependency_analyzer.models.core import Node -from codewiki.src.be.dependency_analyzer.topo_sort import topological_sort, resolve_cycles, build_graph_from_components, dependency_first_dfs, get_leaf_nodes +from codewiki.src.be.dependency_analyzer.topo_sort import ( + topological_sort, + resolve_cycles, + build_graph_from_components, + dependency_first_dfs, + get_leaf_nodes, +) __all__ = [ - 'Node', - 'topological_sort', - 'resolve_cycles', - 'build_graph_from_components', - 'dependency_first_dfs', - 'get_leaf_nodes', - 'DependencyParser', - 'DependencyGraphBuilder', + "Node", + "topological_sort", + "resolve_cycles", + "build_graph_from_components", + "dependency_first_dfs", + "get_leaf_nodes", + "DependencyParser", + "DependencyGraphBuilder", ] _LAZY_IMPORTS = { - 'DependencyParser': ( - 'codewiki.src.be.dependency_analyzer.ast_parser', - 'DependencyParser', + "DependencyParser": ( + "codewiki.src.be.dependency_analyzer.ast_parser", + "DependencyParser", ), - 'DependencyGraphBuilder': ( - 'codewiki.src.be.dependency_analyzer.dependency_graphs_builder', - 'DependencyGraphBuilder', + "DependencyGraphBuilder": ( + "codewiki.src.be.dependency_analyzer.dependency_graphs_builder", + "DependencyGraphBuilder", ), } diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 3ffceb3..2bddff7 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -13,7 +13,11 @@ from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text, assert_safe_path from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer -from codewiki.src.be.dependency_analyzer.analysis.cloning import clone_repository, cleanup_repository, parse_github_url +from codewiki.src.be.dependency_analyzer.analysis.cloning import ( + clone_repository, + cleanup_repository, + parse_github_url, +) from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult from codewiki.src.be.dependency_analyzer.models.core import Repository @@ -39,56 +43,53 @@ def __init__(self): self._temp_directories = [] def analyze_local_repository( - self, - repo_path: str, - max_files: int = 100, - languages: Optional[List[str]] = None + self, repo_path: str, max_files: int = 100, languages: Optional[List[str]] = None ) -> Dict[str, Any]: """ Analyze a local repository folder. - + Args: repo_path: Path to local repository folder max_files: Maximum number of files to analyze languages: List of languages to include (e.g., ['python', 'javascript']) - + Returns: Dict with analysis results including nodes and relationships """ try: logger.debug(f"Analyzing local repository at {repo_path}") - + # Get repo analyzer to find files repo_analyzer = RepoAnalyzer() structure_result = repo_analyzer.analyze_repository_structure(repo_path) - + # Extract code files code_files = self.call_graph_analyzer.extract_code_files(structure_result["file_tree"]) - + # Filter by languages if specified if languages: code_files = [f for f in code_files if f.get("language") in languages] - + # Limit number of files if len(code_files) > max_files: code_files = code_files[:max_files] logger.debug(f"Limited analysis to {max_files} files") - + logger.debug(f"Analyzing {len(code_files)} files") - + # Analyze files result = self.call_graph_analyzer.analyze_code_files(code_files, repo_path) - + return { "nodes": result.get("functions", {}), "relationships": result.get("relationships", []), "summary": { "total_files": len(code_files), "total_nodes": len(result.get("functions", {})), - "total_relationships": len(result.get("relationships", [])) - } + "total_relationships": len(result.get("relationships", [])), + }, } - + except Exception as e: logger.error(f"Local repository analysis failed: {str(e)}", exc_info=True) raise RuntimeError(f"Analysis failed: {str(e)}") @@ -270,8 +271,9 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: logger.debug("No README file found in repository root.") return None - def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str, - skip_file_paths: Optional[set] = None) -> Dict[str, Any]: + def _analyze_call_graph( + self, file_tree: Dict[str, Any], repo_dir: str, skip_file_paths: Optional[set] = None + ) -> Dict[str, Any]: """ Perform multi-language call graph analysis. @@ -283,11 +285,15 @@ def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str, logger.debug("Extracting code files from file tree...") code_files = self.call_graph_analyzer.extract_code_files(file_tree) - logger.debug(f"Found {len(code_files)} total code files. Filtering for supported languages.") + logger.debug( + f"Found {len(code_files)} total code files. Filtering for supported languages." + ) supported_files = self._filter_supported_languages(code_files) logger.debug(f"Analyzing {len(supported_files)} supported files.") - result = self.call_graph_analyzer.analyze_code_files(supported_files, repo_dir, skip_file_paths) + result = self.call_graph_analyzer.analyze_code_files( + supported_files, repo_dir, skip_file_paths + ) result["call_graph"]["supported_languages"] = self._get_supported_languages() result["call_graph"]["unsupported_files"] = len(code_files) - len(supported_files) @@ -322,7 +328,18 @@ def _filter_supported_languages(self, code_files: List[Dict]) -> List[Dict]: def _get_supported_languages(self) -> List[str]: """Get list of currently supported languages for analysis.""" - return ["python", "javascript", "typescript", "java", "csharp", "c", "cpp", "php", "kotlin", "go"] + return [ + "python", + "javascript", + "typescript", + "java", + "csharp", + "c", + "cpp", + "php", + "kotlin", + "go", + ] def _cleanup_repository(self, temp_dir: str): """Clean up cloned repository.""" diff --git a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index b9ccfc5..da64592 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -35,6 +35,7 @@ class TimeoutError(Exception): """Raised when file parsing exceeds timeout.""" + pass @@ -48,12 +49,12 @@ def timeout(seconds): signal.signal() raises ValueError, so we skip the timeout protection and parse without it instead of failing every file. """ + def signal_handler(signum, frame): raise TimeoutError(f"File parsing exceeded {seconds}s timeout") use_signal = ( - hasattr(signal, "SIGALRM") - and threading.current_thread() is threading.main_thread() + hasattr(signal, "SIGALRM") and threading.current_thread() is threading.main_thread() ) if not use_signal: # Windows / non-main thread: SIGALRM unavailable, skip timeout @@ -80,8 +81,9 @@ def __init__(self): self._python_external_import_roots: set = set() logger.debug("CallGraphAnalyzer initialized.") - def analyze_code_files(self, code_files: List[Dict], base_dir: str, - skip_file_paths: Optional[Set[str]] = None) -> Dict: + def analyze_code_files( + self, code_files: List[Dict], base_dir: str, skip_file_paths: Optional[Set[str]] = None + ) -> Dict: """ Complete analysis: Analyze all files to build complete call graph with all nodes. @@ -98,9 +100,11 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str, """ if skip_file_paths: original_count = len(code_files) - code_files = [f for f in code_files if f.get('path') not in skip_file_paths] - logger.info(f"Incremental mode: skipping {original_count - len(code_files)} unchanged files, " - f"parsing {len(code_files)} changed files") + code_files = [f for f in code_files if f.get("path") not in skip_file_paths] + logger.info( + f"Incremental mode: skipping {original_count - len(code_files)} unchanged files, " + f"parsing {len(code_files)} changed files" + ) logger.debug(f"Starting analysis of {len(code_files)} files") logger.info(f"📊 Parsing {len(code_files)} source files (this may take a few minutes)...") @@ -116,23 +120,27 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str, files_analyzed = 0 files_failed = 0 start_time = time.time() - + for idx, file_info in enumerate(code_files, 1): - file_path = file_info['path'] + file_path = file_info["path"] try: # Log progress every file with elapsed time if idx % max(1, len(code_files) // 10) == 0 or idx <= 5: elapsed = time.time() - start_time rate = idx / elapsed if elapsed > 0 else 0 remaining = (len(code_files) - idx) / rate if rate > 0 else 0 - logger.info(f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)") - + logger.info( + f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)" + ) + self._analyze_code_file(base_dir, file_info) files_analyzed += 1 except Exception as e: files_failed += 1 - logger.warning(f" ⚠️ [{idx}/{len(code_files)}] Failed to analyze {file_path}: {str(e)[:100]}") - + logger.warning( + f" ⚠️ [{idx}/{len(code_files)}] Failed to analyze {file_path}: {str(e)[:100]}" + ) + elapsed_time = time.time() - start_time logger.info( f"✓ Analysis complete: {files_analyzed}/{len(code_files)} files analyzed, " @@ -358,8 +366,9 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): repo_dir: Repository base directory """ try: - - from codewiki.src.be.dependency_analyzer.analyzers.javascript import analyze_javascript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.javascript import ( + analyze_javascript_file_treesitter, + ) functions, relationships = analyze_javascript_file_treesitter( file_path, content, repo_path=repo_dir @@ -376,15 +385,16 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): """ - Analyze TypeScript file using tree-sitter based AST analyzer + Analyze TypeScript file using tree-sitter based AST analyzer Args: file_path: Relative path to the TypeScript file content: File content string """ try: - - from codewiki.src.be.dependency_analyzer.analyzers.typescript import analyze_typescript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.typescript import ( + analyze_typescript_file_treesitter, + ) functions, relationships = analyze_typescript_file_treesitter( file_path, content, repo_path=repo_dir @@ -399,8 +409,6 @@ def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): except Exception as e: logger.error(f"Failed to analyze TypeScript file {file_path}: {e}", exc_info=True) - - def _analyze_c_file(self, file_path: str, content: str, repo_dir: str): """ Analyze C file using tree-sitter based analyzer. @@ -430,9 +438,7 @@ def _analyze_cpp_file(self, file_path: str, content: str, repo_dir: str): """ from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file - functions, relationships = analyze_cpp_file( - file_path, content, repo_path=repo_dir - ) + functions, relationships = analyze_cpp_file(file_path, content, repo_path=repo_dir) for func in functions: func_id = func.id if func.id else f"{file_path}:{func.name}" @@ -622,7 +628,9 @@ def _dotted_project_packages(self) -> Dict[str, set]: packages[func_info.language].add(package) return packages - def _is_external_callee(self, language: Optional[str], callee: str, dotted_packages: Dict[str, set]) -> bool: + def _is_external_callee( + self, language: Optional[str], callee: str, dotted_packages: Dict[str, set] + ) -> bool: """Classify a still-unresolved callee as external, after project resolution has had its chance. @@ -677,6 +685,7 @@ def _build_resolution_indexes(self) -> Dict[str, Dict]: name that is unique within the caller's language resolves even when another language defines the same name, and names made ambiguous only by foreign-language components keep resolving as before.""" + def make() -> Dict[str, Dict[str, List[str]]]: return {"exact": defaultdict(list), "simple": defaultdict(list)} @@ -722,13 +731,17 @@ def add(index: Dict[str, List[str]], key: Optional[str], func_id: str) -> None: "by_lang": dict(by_lang), } - def _resolve_callee(self, relationship: CallRelationship, indexes: Dict[str, Dict]) -> Optional[str]: + def _resolve_callee( + self, relationship: CallRelationship, indexes: Dict[str, Dict] + ) -> Optional[str]: caller = self.functions.get(relationship.caller) caller_language = caller.language if caller else None lang_indexes = indexes["by_lang"].get(caller_language) if caller_language else None if lang_indexes: - match = self._resolve_callee_in(relationship, lang_indexes["exact"], lang_indexes["simple"]) + match = self._resolve_callee_in( + relationship, lang_indexes["exact"], lang_indexes["simple"] + ) if match: return match @@ -850,7 +863,15 @@ def _generate_visualization_data(self) -> Dict: node_classes.append("lang-typescript") elif language == "c": node_classes.append("lang-c") - elif language == "cpp" or file_ext in [".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++"]: + elif language == "cpp" or file_ext in [ + ".cpp", + ".cc", + ".cxx", + ".c++", + ".hpp", + ".hxx", + ".h++", + ]: node_classes.append("lang-cpp") elif file_ext in [".kt", ".kts"]: node_classes.append("lang-kotlin") @@ -967,12 +988,12 @@ def _select_most_connected_nodes(self, target_count: int): selected_func_ids = sorted_func_ids[:target_count] - original_func_count = len(self.functions) + len(self.functions) self.functions = { fid: func for fid, func in self.functions.items() if fid in selected_func_ids } - original_rel_count = len(self.call_relationships) + len(self.call_relationships) self.call_relationships = [ rel for rel in self.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analysis/cloning.py b/codewiki/src/be/dependency_analyzer/analysis/cloning.py index 49d6072..fb2838b 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/cloning.py +++ b/codewiki/src/be/dependency_analyzer/analysis/cloning.py @@ -4,7 +4,6 @@ import subprocess import stat import time -from typing import Optional GIT_EXECUTABLE_PATH = shutil.which("git") @@ -32,7 +31,7 @@ def sanitize_github_url(github_url: str) -> str: if url.startswith("www."): url = url[4:] - parts = url.split("/") + url.split("/") if url.startswith("github.com/"): url_parts = url.split("/") @@ -93,7 +92,7 @@ def clone_repository(github_url: str) -> str: capture_output=True, text=True, ) - except: + except Exception: pass subprocess.run( @@ -149,14 +148,14 @@ def clone_repository(github_url: str) -> str: capture_output=True, text=True, ) - except: + except Exception: pass return temp_dir except subprocess.TimeoutExpired: if os.path.exists(temp_dir): cleanup_repository_safe(temp_dir) raise RuntimeError( - f"Repository cloning timed out after 5 minutes. The repository may be too large or network is slow." + "Repository cloning timed out after 5 minutes. The repository may be too large or network is slow." ) except subprocess.CalledProcessError as e: if os.path.exists(temp_dir): @@ -197,7 +196,7 @@ def handle_remove_readonly(func, path, exc): shutil.rmtree(repo_dir) return True return False - except PermissionError as e: + except PermissionError: try: time.sleep(1) if os.path.exists(repo_dir): @@ -256,4 +255,4 @@ def parse_github_url(github_url: str) -> dict: "name": "unknown", "full_name": "unknown", "url": github_url, - } \ No newline at end of file + } diff --git a/codewiki/src/be/dependency_analyzer/analysis/cross_service_matcher.py b/codewiki/src/be/dependency_analyzer/analysis/cross_service_matcher.py index 124ddf0..70d8e33 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/cross_service_matcher.py +++ b/codewiki/src/be/dependency_analyzer/analysis/cross_service_matcher.py @@ -6,11 +6,12 @@ Phase 3: Channel EMITS/LISTENS_ON matching Phase 4: gRPC / GraphQL / tRPC matching """ + from __future__ import annotations import logging from collections import defaultdict -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Set, Tuple from codewiki.src.be.dependency_analyzer.models.cross_service import ( CrossServiceLink, @@ -86,10 +87,7 @@ def match(self) -> WorkspaceTopology: for link in links: matched_keys.add(link.route_key) - unmatched = [ - r for r in all_routes - if r.route_key not in matched_keys - ] + unmatched = [r for r in all_routes if r.route_key not in matched_keys] return WorkspaceTopology( repos=sorted(self._repo_routes.keys()), @@ -131,19 +129,25 @@ def _match_http_routes(self, matched_keys: Set[str]) -> List[CrossServiceLink]: for srv_repo, srv_route in servers: if srv_repo == repo_name: continue # skip intra-repo - links.append(CrossServiceLink( - route_key=route.route_key, - protocol=RouteProtocol.HTTP, - method=route.method, - path=route.path, - client_repo=repo_name, - client_component_id=route.component_id, - client_function=route.component_id.split("::")[-1] if "::" in route.component_id else "", - server_repo=srv_repo, - server_component_id=srv_route.component_id, - server_function=srv_route.component_id.split("::")[-1] if "::" in srv_route.component_id else "", - confidence=1.0, - )) + links.append( + CrossServiceLink( + route_key=route.route_key, + protocol=RouteProtocol.HTTP, + method=route.method, + path=route.path, + client_repo=repo_name, + client_component_id=route.component_id, + client_function=route.component_id.split("::")[-1] + if "::" in route.component_id + else "", + server_repo=srv_repo, + server_component_id=srv_route.component_id, + server_function=srv_route.component_id.split("::")[-1] + if "::" in srv_route.component_id + else "", + confidence=1.0, + ) + ) continue # Fuzzy template match: try matching concrete path against server templates @@ -185,19 +189,25 @@ def _fuzzy_match( continue # Try matching client path (concrete) against server path (template) if path_matches_template(client_route.path, srv_route.path): - links.append(CrossServiceLink( - route_key=srv_key, - protocol=RouteProtocol.HTTP, - method=method, - path=srv_route.path, - client_repo=client_repo, - client_component_id=client_route.component_id, - client_function=client_route.component_id.split("::")[-1] if "::" in client_route.component_id else "", - server_repo=srv_repo, - server_component_id=srv_route.component_id, - server_function=srv_route.component_id.split("::")[-1] if "::" in srv_route.component_id else "", - confidence=0.8, - )) + links.append( + CrossServiceLink( + route_key=srv_key, + protocol=RouteProtocol.HTTP, + method=method, + path=srv_route.path, + client_repo=client_repo, + client_component_id=client_route.component_id, + client_function=client_route.component_id.split("::")[-1] + if "::" in client_route.component_id + else "", + server_repo=srv_repo, + server_component_id=srv_route.component_id, + server_function=srv_route.component_id.split("::")[-1] + if "::" in srv_route.component_id + else "", + confidence=0.8, + ) + ) return links # ---- Phase 2: MQ ---- @@ -224,18 +234,24 @@ def _match_mq_routes(self, matched_keys: Set[str]) -> List[CrossServiceLink]: for cons_repo, cons_route in consumers: if cons_repo == repo_name: continue - links.append(CrossServiceLink( - route_key=route.route_key, - protocol=RouteProtocol.MQ, - method=None, - path=route.path, - client_repo=repo_name, - client_component_id=route.component_id, - client_function=route.component_id.split("::")[-1] if "::" in route.component_id else "", - server_repo=cons_repo, - server_component_id=cons_route.component_id, - server_function=cons_route.component_id.split("::")[-1] if "::" in cons_route.component_id else "", - confidence=1.0, - )) + links.append( + CrossServiceLink( + route_key=route.route_key, + protocol=RouteProtocol.MQ, + method=None, + path=route.path, + client_repo=repo_name, + client_component_id=route.component_id, + client_function=route.component_id.split("::")[-1] + if "::" in route.component_id + else "", + server_repo=cons_repo, + server_component_id=cons_route.component_id, + server_function=cons_route.component_id.split("::")[-1] + if "::" in cons_route.component_id + else "", + confidence=1.0, + ) + ) return links diff --git a/codewiki/src/be/dependency_analyzer/analysis/infra_scanner.py b/codewiki/src/be/dependency_analyzer/analysis/infra_scanner.py index 139ea1b..e849469 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/infra_scanner.py +++ b/codewiki/src/be/dependency_analyzer/analysis/infra_scanner.py @@ -3,6 +3,7 @@ Parses deployment configuration to discover service names, ports, and inter-service dependencies that complement Route-based matching. """ + from __future__ import annotations import logging @@ -16,9 +17,14 @@ class InfraServiceInfo: """Minimal service info extracted from infrastructure configs.""" - def __init__(self, name: str, ports: List[int] = None, - depends_on: List[str] = None, env_vars: Dict[str, str] = None, - source: str = ""): + def __init__( + self, + name: str, + ports: List[int] = None, + depends_on: List[str] = None, + env_vars: Dict[str, str] = None, + source: str = "", + ): self.name = name self.ports = ports or [] self.depends_on = depends_on or [] @@ -54,14 +60,14 @@ def scan(self) -> Dict[str, InfraServiceInfo]: def _scan_docker_compose(self): """Parse docker-compose.yml / docker-compose.yaml files.""" - for pattern in ("docker-compose.yml", "docker-compose.yaml", - "compose.yml", "compose.yaml"): + for pattern in ("docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"): for f in self.workspace_path.rglob(pattern): self._parse_compose_file(f) def _parse_compose_file(self, path: Path): try: import yaml + data = yaml.safe_load(path.read_text(encoding="utf-8", errors="replace")) except ImportError: logger.debug("PyYAML not available, skipping docker-compose parsing") @@ -119,8 +125,11 @@ def _parse_compose_file(self, path: Path): env_vars = {str(k): str(v) for k, v in raw_env.items()} info = InfraServiceInfo( - name=svc_name, ports=ports, depends_on=depends_on, - env_vars=env_vars, source="docker-compose", + name=svc_name, + ports=ports, + depends_on=depends_on, + env_vars=env_vars, + source="docker-compose", ) self.services[svc_name] = info @@ -150,7 +159,7 @@ def _parse_env_file(self, path: Path): return url_pattern = re.compile( - r'^(\w*(?:SERVICE|API|URL|HOST|ENDPOINT)\w*)\s*=\s*(.+?)$', + r"^(\w*(?:SERVICE|API|URL|HOST|ENDPOINT)\w*)\s*=\s*(.+?)$", re.MULTILINE | re.IGNORECASE, ) for m in url_pattern.finditer(content): @@ -174,11 +183,13 @@ def _extract_service_name_from_key(self, key: str) -> Optional[str]: def _scan_application_yml(self): """Scan application.yml / application.yaml for service URLs.""" - for pattern in ("application.yml", "application.yaml", - "application.properties"): + for pattern in ("application.yml", "application.yaml", "application.properties"): for f in self.workspace_path.rglob(pattern): parts = f.parts - if any(skip in parts for skip in ("node_modules", ".venv", "venv", ".git", "target", "build")): + if any( + skip in parts + for skip in ("node_modules", ".venv", "venv", ".git", "target", "build") + ): continue self._parse_spring_config(f) @@ -186,6 +197,7 @@ def _parse_spring_config(self, path: Path): if path.suffix in (".yml", ".yaml"): try: import yaml + data = yaml.safe_load(path.read_text(encoding="utf-8", errors="replace")) except ImportError: return @@ -199,7 +211,7 @@ def _parse_spring_config(self, path: Path): except OSError: return url_pattern = re.compile( - r'^.*?(?:service|api|url|host|endpoint).*?=(.+?)$', + r"^.*?(?:service|api|url|host|endpoint).*?=(.+?)$", re.MULTILINE | re.IGNORECASE, ) for m in url_pattern.finditer(content): @@ -216,8 +228,12 @@ def _extract_urls_from_dict(self, data: Dict, prefix: str): return for key, value in data.items(): full_key = f"{prefix}.{key}" if prefix else key - if isinstance(value, str) and (value.startswith("http://") or value.startswith("https://")): - svc_name = self._extract_service_from_url(value) or self._extract_service_name_from_key(full_key) + if isinstance(value, str) and ( + value.startswith("http://") or value.startswith("https://") + ): + svc_name = self._extract_service_from_url( + value + ) or self._extract_service_name_from_key(full_key) if svc_name: self.service_urls[svc_name] = value elif isinstance(value, dict): @@ -227,7 +243,7 @@ def _extract_service_from_url(self, url: str) -> Optional[str]: """http://order-service:8080 → order-service.""" for scheme in ("https://", "http://"): if url.startswith(scheme): - rest = url[len(scheme):] + rest = url[len(scheme) :] host = rest.split(":")[0].split("/")[0] if host and not host.replace(".", "").isdigit(): return host diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index d94242c..880c55c 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -5,6 +5,8 @@ detailed file tree representations with filtering capabilities. """ +from __future__ import annotations + import fnmatch import logging import shutil @@ -33,7 +35,7 @@ def __init__(self, repo_dir: Path) -> None: self._ignored_files: set[str] = set() self._ignored_dirs: set[str] = set() self._ignore_all_untracked = False - self._fallback_specs: list[tuple[str, "GitIgnoreSpec"]] = [] + self._fallback_specs: list[tuple[str, "GitIgnoreSpec"]] = [] # noqa: F821 self._using_git = self._load_git_ignored_paths() if not self._using_git: self._load_fallback_specs() @@ -97,7 +99,7 @@ def _load_git_ignored_paths(self) -> bool: if scope_prefix: if not repo_relative.startswith(scope_prefix): continue - relative = repo_relative[len(scope_prefix):] + relative = repo_relative[len(scope_prefix) :] else: relative = repo_relative @@ -162,7 +164,7 @@ def is_ignored(self, relative_path: str, is_dir: bool) -> bool: if normalized == base: local_path = "" elif normalized.startswith(f"{base}/"): - local_path = normalized[len(base) + 1:] + local_path = normalized[len(base) + 1 :] else: continue else: diff --git a/codewiki/src/be/dependency_analyzer/analysis/service_detector.py b/codewiki/src/be/dependency_analyzer/analysis/service_detector.py index 790c37a..72ab97d 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/service_detector.py +++ b/codewiki/src/be/dependency_analyzer/analysis/service_detector.py @@ -8,6 +8,7 @@ Used by ``analyze_repo`` to partition routes by sub-service so that ``CrossServiceMatcher`` can find intra-repo cross-service calls. """ + from __future__ import annotations import logging @@ -20,12 +21,37 @@ # Directories that should never be treated as services _EXCLUDE_DIRS = { - "node_modules", ".venv", "venv", "__pycache__", ".git", ".idea", - ".vscode", "dist", "build", "target", ".tox", ".mypy_cache", - ".pytest_cache", "coverage", ".next", ".nuxt", "vendor", - "test", "tests", "testing", "e2e", "docs", "doc", "scripts", - "migrations", "fixtures", "mocks", "__mocks__", ".codewiki", - "repowiki", "workspace-wiki", + "node_modules", + ".venv", + "venv", + "__pycache__", + ".git", + ".idea", + ".vscode", + "dist", + "build", + "target", + ".tox", + ".mypy_cache", + ".pytest_cache", + "coverage", + ".next", + ".nuxt", + "vendor", + "test", + "tests", + "testing", + "e2e", + "docs", + "doc", + "scripts", + "migrations", + "fixtures", + "mocks", + "__mocks__", + ".codewiki", + "repowiki", + "workspace-wiki", } # Convention directories whose children are likely services @@ -44,7 +70,9 @@ def __init__(self, name: str, relative_path: str, source: str): self.source = source # detection signal: "docker-compose", "dockerfile", etc. def __repr__(self) -> str: - return f"ServiceInfo(name={self.name!r}, path={self.relative_path!r}, source={self.source!r})" + return ( + f"ServiceInfo(name={self.name!r}, path={self.relative_path!r}, source={self.source!r})" + ) def detect_services(repo_path: Path) -> Dict[str, ServiceInfo]: @@ -78,7 +106,8 @@ def detect_services(repo_path: Path) -> Dict[str, ServiceInfo]: if services: logger.info( "Detected %d sub-services in %s: %s", - len(services), repo_path.name, + len(services), + repo_path.name, ", ".join(f"{s.name} ({s.source})" for s in services.values()), ) @@ -106,7 +135,7 @@ def assign_service_label( if not rp.endswith("/"): rp += "/" if fp.startswith(rp): - fp = fp[len(rp):] + fp = fp[len(rp) :] best_name = fallback best_len = 0 @@ -128,8 +157,10 @@ def assign_service_label( # Pruned directory walker (avoids descending into excluded dirs) # --------------------------------------------------------------------------- + def _walk_pruned( - root: Path, max_depth: int = _MAX_DEPTH, + root: Path, + max_depth: int = _MAX_DEPTH, ) -> Iterator[Tuple[Path, List[str], List[str]]]: """os.walk with in-place pruning of excluded directories. @@ -147,10 +178,7 @@ def _walk_pruned( depth = len(rel.replace("\\", "/").split("/")) # Prune excluded dirs in-place - dirnames[:] = [ - d for d in dirnames - if d not in _EXCLUDE_DIRS and not d.startswith(".") - ] + dirnames[:] = [d for d in dirnames if d not in _EXCLUDE_DIRS and not d.startswith(".")] # Stop descending beyond max_depth if depth >= max_depth: @@ -171,6 +199,7 @@ def _find_files(root: Path, name: str, max_depth: int = _MAX_DEPTH) -> List[Path def _find_files_glob(root: Path, pattern: str, max_depth: int = _MAX_DEPTH) -> List[Path]: """Find files matching a glob pattern (e.g. 'Dockerfile.*') with pruned walking.""" import fnmatch + results = [] for dirpath, _, filenames in _walk_pruned(root, max_depth): for fn in filenames: @@ -183,6 +212,7 @@ def _find_files_glob(root: Path, pattern: str, max_depth: int = _MAX_DEPTH) -> L # Service registration (handles name collisions) # --------------------------------------------------------------------------- + def _register_service( services: Dict[str, ServiceInfo], name: str, @@ -201,7 +231,9 @@ def _register_service( qualified = relative_path.replace("/", "-").replace("\\", "-") if qualified not in services: services[qualified] = ServiceInfo( - name=qualified, relative_path=relative_path, source=source, + name=qualified, + relative_path=relative_path, + source=source, ) @@ -209,19 +241,22 @@ def _register_service( # Phase 1: docker-compose # --------------------------------------------------------------------------- + def _detect_from_compose(repo_path: Path, services: Dict[str, ServiceInfo]): """Parse docker-compose files for service definitions with build contexts.""" - for pattern in ("docker-compose.yml", "docker-compose.yaml", - "compose.yml", "compose.yaml"): + for pattern in ("docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"): for f in _find_files(repo_path, pattern): _parse_compose_for_services(f, repo_path, services) def _parse_compose_for_services( - compose_file: Path, repo_path: Path, services: Dict[str, ServiceInfo], + compose_file: Path, + repo_path: Path, + services: Dict[str, ServiceInfo], ): try: import yaml + data = yaml.safe_load(compose_file.read_text(encoding="utf-8", errors="replace")) except ImportError: logger.debug("PyYAML not available, skipping compose parsing") @@ -276,6 +311,7 @@ def _parse_compose_for_services( # Phase 2: Dockerfiles # --------------------------------------------------------------------------- + def _detect_from_dockerfiles(repo_path: Path, services: Dict[str, ServiceInfo]): """Detect services from Dockerfiles in distinct sub-directories.""" dockerfiles: List[Path] = [] @@ -361,7 +397,9 @@ def _detect_from_build_manifests(repo_path: Path, services: Dict[str, ServiceInf if svc_name and rel not in all_dirs: all_dirs.add(rel) _register_service( - services, svc_name, rel, + services, + svc_name, + rel, f"build-manifest:{_BUILD_MANIFESTS[manifest_name]}", ) @@ -370,6 +408,7 @@ def _package_json_is_service(pkg_path: Path) -> bool: """Check if a package.json looks like a runnable service (has start/main).""" try: import json + data = json.loads(pkg_path.read_text(encoding="utf-8", errors="replace")) except Exception: return False @@ -387,6 +426,7 @@ def _package_json_is_service(pkg_path: Path) -> bool: # Phase 4: Convention directories # --------------------------------------------------------------------------- + def _detect_from_convention_dirs(repo_path: Path, services: Dict[str, ServiceInfo]): """Detect services under convention directories like services/, apps/.""" for conv_name in _CONVENTION_DIRS: @@ -417,17 +457,27 @@ def _has_source_files(directory: Path, max_files: int = 200) -> bool: small limit. """ source_exts = { - ".py", ".java", ".js", ".jsx", ".ts", ".tsx", ".go", - ".rs", ".kt", ".kts", ".cs", ".php", ".rb", ".c", ".cpp", + ".py", + ".java", + ".js", + ".jsx", + ".ts", + ".tsx", + ".go", + ".rs", + ".kt", + ".kts", + ".cs", + ".php", + ".rb", + ".c", + ".cpp", } count = 0 try: for dirpath, dirnames, filenames in os.walk(str(directory)): # Prune excluded dirs - dirnames[:] = [ - d for d in dirnames - if d not in _EXCLUDE_DIRS and not d.startswith(".") - ] + dirnames[:] = [d for d in dirnames if d not in _EXCLUDE_DIRS and not d.startswith(".")] for fn in filenames: if Path(fn).suffix.lower() in source_exts: return True @@ -443,6 +493,7 @@ def _has_source_files(directory: Path, max_files: int = 200) -> bool: # Phase 5: Spring Boot application.yml / application.properties # --------------------------------------------------------------------------- + def _detect_from_spring_config(repo_path: Path, services: Dict[str, ServiceInfo]): """Detect Spring Boot services via spring.application.name.""" # YAML configs @@ -460,7 +511,9 @@ def _detect_from_spring_config(repo_path: Path, services: Dict[str, ServiceInfo] def _register_spring_service( - config_file: Path, name: str, repo_path: Path, + config_file: Path, + name: str, + repo_path: Path, services: Dict[str, ServiceInfo], ): """Register a Spring Boot service after finding its app name.""" @@ -478,6 +531,7 @@ def _extract_spring_app_name_yml(yml_path: Path) -> Optional[str]: """Extract spring.application.name from a YAML file.""" try: import yaml + data = yaml.safe_load(yml_path.read_text(encoding="utf-8", errors="replace")) except Exception: return None @@ -501,7 +555,8 @@ def _extract_spring_app_name_properties(prop_path: Path) -> Optional[str]: return None m = re.search( r"^spring\.application\.name\s*=\s*(.+?)$", - content, re.MULTILINE, + content, + re.MULTILINE, ) if m: name = m.group(1).strip() @@ -512,8 +567,16 @@ def _extract_spring_app_name_properties(prop_path: Path) -> Optional[str]: def _find_service_root(start: Path, repo_root: Path) -> Path: """Walk up from start to find the nearest directory with a build manifest.""" - markers = {"pom.xml", "build.gradle", "build.gradle.kts", "go.mod", - "package.json", "pyproject.toml", "setup.py", "Cargo.toml"} + markers = { + "pom.xml", + "build.gradle", + "build.gradle.kts", + "go.mod", + "package.json", + "pyproject.toml", + "setup.py", + "Cargo.toml", + } current = start while current != repo_root and current != current.parent: if any((current / m).exists() for m in markers): @@ -526,6 +589,7 @@ def _find_service_root(start: Path, repo_root: Path) -> Path: # Helpers # --------------------------------------------------------------------------- + def _is_excluded_rel(rel_path: str) -> bool: """Check if a relative path contains excluded directory segments.""" parts = rel_path.replace("\\", "/").split("/") @@ -571,8 +635,7 @@ def _remove_nested_services(services: Dict[str, ServiceInfo]) -> Dict[str, Servi prefix += "/" # Check if this service is nested under an already-kept service is_nested = any( - prefix.startswith(kp if kp.endswith("/") else kp + "/") - for kp in kept_paths + prefix.startswith(kp if kp.endswith("/") else kp + "/") for kp in kept_paths ) if not is_nested: kept[svc.name] = svc diff --git a/codewiki/src/be/dependency_analyzer/analysis/topology_visualizer.py b/codewiki/src/be/dependency_analyzer/analysis/topology_visualizer.py index 3b085d6..dbb4063 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/topology_visualizer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/topology_visualizer.py @@ -3,6 +3,7 @@ Converts a ``WorkspaceTopology`` into human-readable documentation suitable for embedding in workspace ``overview.md``. """ + from __future__ import annotations from typing import List @@ -173,7 +174,5 @@ def generate_unmatched_table(self, topology: WorkspaceTopology) -> str: note = "External API or unimplemented" else: note = "No client detected" - lines.append( - f"| {method} | `{path}` | {route.repo_name} | {role} | {note} |" - ) + lines.append(f"| {method} | `{path}` | {route.repo_name} | {role} | {note} |") return "\n".join(lines) diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index 7c396b6..7b9755e 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -1,7 +1,6 @@ import logging -from typing import List, Optional, Tuple +from typing import List, Tuple from pathlib import Path -import sys import os from tree_sitter import Parser, Language @@ -10,209 +9,233 @@ logger = logging.getLogger(__name__) + class TreeSitterCAnalyzer: - def __init__(self, file_path: str, content: str, repo_path: str = None): - self.file_path = Path(file_path) - self.content = content - self.repo_path = repo_path or "" - self.nodes: List[Node] = [] - self.call_relationships: List[CallRelationship] = [] - self._analyze() - - def _get_module_path(self) -> str: - if self.repo_path: - try: - rel_path = os.path.relpath(str(self.file_path), self.repo_path) - except ValueError: - rel_path = str(self.file_path) - else: - rel_path = str(self.file_path) - - for ext in ['.c', '.h']: - if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] - break - return rel_path.replace('/', '.').replace('\\', '.') - - def _get_relative_path(self) -> str: - if self.repo_path: - try: - # BUG-21: normalize symlinks before computing relative paths - real_file = os.path.realpath(str(self.file_path)) - real_repo = os.path.realpath(self.repo_path) - return os.path.relpath(real_file, real_repo) - except ValueError: - return str(self.file_path) - else: - return str(self.file_path) - - def _get_component_id(self, name: str) -> str: - rel_path = self._get_relative_path() - return f"{rel_path}::{name}" - - def _analyze(self): - language_capsule = tree_sitter_c.language() - c_language = Language(language_capsule) - parser = Parser(c_language) - tree = parser.parse(bytes(self.content, "utf8")) - root = tree.root_node - lines = self.content.splitlines() - - top_level_nodes = {} - - # collect all top-level nodes using recursive traversal - self._extract_nodes(root, top_level_nodes, lines) - - # extract relationships between top-level nodes - self._extract_relationships(root, top_level_nodes) - - def _extract_nodes(self, node, top_level_nodes, lines): - """Recursively extract top-level nodes (functions, structs, and global variables).""" - node_type = None - node_name = None - - if node.type == "function_definition": - node_type = "function" - # look for function_declarator - declarator = next((c for c in node.children if c.type == "function_declarator"), None) - if declarator: - identifier = next((c for c in declarator.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() - elif node.type == "struct_specifier": - # Extract struct definitions: struct Name { ... } - node_type = "struct" - # Find type_identifier that represents the struct name - for child in node.children: - if child.type == "type_identifier": - node_name = child.text.decode() - break - elif node.type == "type_definition": - # Handle typedef struct definitions: typedef struct { ... } Name; - # Check if this typedef contains a struct - struct_spec = next((c for c in node.children if c.type == "struct_specifier"), None) - if struct_spec: - node_type = "struct" - # The typedef name is the type_identifier at the end - type_declarator = next((c for c in node.children if c.type == "type_identifier"), None) - if type_declarator: - node_name = type_declarator.text.decode() - elif node.type == "declaration": - if self._is_global_variable(node): - node_type = "variable" - for child in node.children: - if child.type == "init_declarator": - identifier = next((c for c in child.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() - break - pointer_declarator = next((c for c in child.children if c.type == "pointer_declarator"), None) - if pointer_declarator: - identifier = next((c for c in pointer_declarator.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() - break - elif child.type == "identifier": - node_name = child.text.decode() - break - - if node_type and node_name: - component_id = self._get_component_id(node_name) - relative_path = self._get_relative_path() - node_obj = Node( - id=component_id, - name=node_name, - component_type=node_type, - file_path=str(self.file_path), - relative_path=relative_path, - source_code="\n".join(lines[node.start_point[0]:node.end_point[0]+1]), - start_line=node.start_point[0]+1, - end_line=node.end_point[0]+1, - has_docstring=False, - docstring="", - parameters=None, - node_type=node_type, - base_classes=None, - class_name=None, - display_name=f"{node_type} {node_name}", - component_id=component_id, - language="c", - qualified_name=node_name - ) - - if node_type in ["function", "struct"]: - self.nodes.append(node_obj) - top_level_nodes[node_name] = node_obj - - for child in node.children: - self._extract_nodes(child, top_level_nodes, lines) - - def _is_global_variable(self, node) -> bool: - parent = node.parent - while parent: - if parent.type == "function_definition": - return False - parent = parent.parent - return True - - def _extract_relationships(self, node, top_level_nodes): - """Extract various types of relationships between top-level nodes.""" - - # 1. function calls other functions - if node.type == "call_expression": - containing_function = self._find_containing_function(node, top_level_nodes) - if containing_function: - containing_function_id = self._get_component_id(containing_function) - - # Get called function name. External/libc filtering happens in - # CallGraphAnalyzer after cross-file resolution, so a project - # function that shadows a libc name still gets its edges. - function_node = next((c for c in node.children if c.type == "identifier"), None) - if function_node: - called_function = function_node.text.decode() - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=called_function, # Use simple name for cross-file resolution - call_line=node.start_point[0]+1, - is_resolved=False # Let CallGraphAnalyzer resolve - )) - - # 2. function uses global variables - if node.type == "identifier": - containing_function = self._find_containing_function(node, top_level_nodes) - if containing_function: - var_name = node.text.decode() - # Check if this identifier refers to a global variable - if var_name in top_level_nodes and top_level_nodes[var_name].component_type == "variable": - containing_function_id = self._get_component_id(containing_function) - var_component_id = self._get_component_id(var_name) - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=var_component_id, - call_line=node.start_point[0]+1, - is_resolved=True # Local file relationship - )) - - # Recursively process children - for child in node.children: - self._extract_relationships(child, top_level_nodes) - - def _find_containing_function(self, node, top_level_nodes): - """Find the function that contains this node.""" - current = node.parent - while current: - if current.type == "function_definition": - # Get function name - declarator = next((c for c in current.children if c.type == "function_declarator"), None) - if declarator: - identifier = next((c for c in declarator.children if c.type == "identifier"), None) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name - current = current.parent - return None - -def analyze_c_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: - analyzer = TreeSitterCAnalyzer(file_path, content, repo_path) - return analyzer.nodes, analyzer.call_relationships + def __init__(self, file_path: str, content: str, repo_path: str = None): + self.file_path = Path(file_path) + self.content = content + self.repo_path = repo_path or "" + self.nodes: List[Node] = [] + self.call_relationships: List[CallRelationship] = [] + self._analyze() + + def _get_module_path(self) -> str: + if self.repo_path: + try: + rel_path = os.path.relpath(str(self.file_path), self.repo_path) + except ValueError: + rel_path = str(self.file_path) + else: + rel_path = str(self.file_path) + + for ext in [".c", ".h"]: + if rel_path.endswith(ext): + rel_path = rel_path[: -len(ext)] + break + return rel_path.replace("/", ".").replace("\\", ".") + + def _get_relative_path(self) -> str: + if self.repo_path: + try: + # BUG-21: normalize symlinks before computing relative paths + real_file = os.path.realpath(str(self.file_path)) + real_repo = os.path.realpath(self.repo_path) + return os.path.relpath(real_file, real_repo) + except ValueError: + return str(self.file_path) + else: + return str(self.file_path) + + def _get_component_id(self, name: str) -> str: + rel_path = self._get_relative_path() + return f"{rel_path}::{name}" + + def _analyze(self): + language_capsule = tree_sitter_c.language() + c_language = Language(language_capsule) + parser = Parser(c_language) + tree = parser.parse(bytes(self.content, "utf8")) + root = tree.root_node + lines = self.content.splitlines() + + top_level_nodes = {} + + # collect all top-level nodes using recursive traversal + self._extract_nodes(root, top_level_nodes, lines) + + # extract relationships between top-level nodes + self._extract_relationships(root, top_level_nodes) + + def _extract_nodes(self, node, top_level_nodes, lines): + """Recursively extract top-level nodes (functions, structs, and global variables).""" + node_type = None + node_name = None + + if node.type == "function_definition": + node_type = "function" + # look for function_declarator + declarator = next((c for c in node.children if c.type == "function_declarator"), None) + if declarator: + identifier = next((c for c in declarator.children if c.type == "identifier"), None) + if identifier: + node_name = identifier.text.decode() + elif node.type == "struct_specifier": + # Extract struct definitions: struct Name { ... } + node_type = "struct" + # Find type_identifier that represents the struct name + for child in node.children: + if child.type == "type_identifier": + node_name = child.text.decode() + break + elif node.type == "type_definition": + # Handle typedef struct definitions: typedef struct { ... } Name; + # Check if this typedef contains a struct + struct_spec = next((c for c in node.children if c.type == "struct_specifier"), None) + if struct_spec: + node_type = "struct" + # The typedef name is the type_identifier at the end + type_declarator = next( + (c for c in node.children if c.type == "type_identifier"), None + ) + if type_declarator: + node_name = type_declarator.text.decode() + elif node.type == "declaration": + if self._is_global_variable(node): + node_type = "variable" + for child in node.children: + if child.type == "init_declarator": + identifier = next( + (c for c in child.children if c.type == "identifier"), None + ) + if identifier: + node_name = identifier.text.decode() + break + pointer_declarator = next( + (c for c in child.children if c.type == "pointer_declarator"), None + ) + if pointer_declarator: + identifier = next( + (c for c in pointer_declarator.children if c.type == "identifier"), + None, + ) + if identifier: + node_name = identifier.text.decode() + break + elif child.type == "identifier": + node_name = child.text.decode() + break + + if node_type and node_name: + component_id = self._get_component_id(node_name) + relative_path = self._get_relative_path() + node_obj = Node( + id=component_id, + name=node_name, + component_type=node_type, + file_path=str(self.file_path), + relative_path=relative_path, + source_code="\n".join(lines[node.start_point[0] : node.end_point[0] + 1]), + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + has_docstring=False, + docstring="", + parameters=None, + node_type=node_type, + base_classes=None, + class_name=None, + display_name=f"{node_type} {node_name}", + component_id=component_id, + language="c", + qualified_name=node_name, + ) + + if node_type in ["function", "struct"]: + self.nodes.append(node_obj) + top_level_nodes[node_name] = node_obj + + for child in node.children: + self._extract_nodes(child, top_level_nodes, lines) + + def _is_global_variable(self, node) -> bool: + parent = node.parent + while parent: + if parent.type == "function_definition": + return False + parent = parent.parent + return True + + def _extract_relationships(self, node, top_level_nodes): + """Extract various types of relationships between top-level nodes.""" + + # 1. function calls other functions + if node.type == "call_expression": + containing_function = self._find_containing_function(node, top_level_nodes) + if containing_function: + containing_function_id = self._get_component_id(containing_function) + + # Get called function name. External/libc filtering happens in + # CallGraphAnalyzer after cross-file resolution, so a project + # function that shadows a libc name still gets its edges. + function_node = next((c for c in node.children if c.type == "identifier"), None) + if function_node: + called_function = function_node.text.decode() + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=called_function, # Use simple name for cross-file resolution + call_line=node.start_point[0] + 1, + is_resolved=False, # Let CallGraphAnalyzer resolve + ) + ) + + # 2. function uses global variables + if node.type == "identifier": + containing_function = self._find_containing_function(node, top_level_nodes) + if containing_function: + var_name = node.text.decode() + # Check if this identifier refers to a global variable + if ( + var_name in top_level_nodes + and top_level_nodes[var_name].component_type == "variable" + ): + containing_function_id = self._get_component_id(containing_function) + var_component_id = self._get_component_id(var_name) + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=var_component_id, + call_line=node.start_point[0] + 1, + is_resolved=True, # Local file relationship + ) + ) + + # Recursively process children + for child in node.children: + self._extract_relationships(child, top_level_nodes) + + def _find_containing_function(self, node, top_level_nodes): + """Find the function that contains this node.""" + current = node.parent + while current: + if current.type == "function_definition": + # Get function name + declarator = next( + (c for c in current.children if c.type == "function_declarator"), None + ) + if declarator: + identifier = next( + (c for c in declarator.children if c.type == "identifier"), None + ) + if identifier: + func_name = identifier.text.decode() + if func_name in top_level_nodes: + return func_name + current = current.parent + return None + + +def analyze_c_file( + file_path: str, content: str, repo_path: str = None +) -> Tuple[List[Node], List[CallRelationship]]: + analyzer = TreeSitterCAnalyzer(file_path, content, repo_path) + return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py index d22b622..5637a02 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/cpp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py @@ -1,7 +1,6 @@ import logging from typing import List, Optional, Tuple from pathlib import Path -import sys import os import re @@ -9,8 +8,8 @@ import tree_sitter_cpp from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( - is_external_symbol, - is_macro_name, + is_external_symbol, + is_macro_name, ) logger = logging.getLogger(__name__) @@ -22,658 +21,723 @@ # rather than any specific library's prefix. _SPECIFIER_MACRO_RE = re.compile(r"(^\s*|[{};>,]\s*)([A-Z][A-Z0-9_]*[A-Z0-9])(\s+)(?=[A-Za-z_~])") # Same, but for function-like specifier macros such as `VISIBILITY("default") void f()`. -_SPECIFIER_MACRO_CALL_RE = re.compile(r"(^\s*|[{};>,]\s*)([A-Z][A-Z0-9_]*[A-Z0-9])\s*\([^()]*\)(\s+)(?=[A-Za-z_~])") +_SPECIFIER_MACRO_CALL_RE = re.compile( + r"(^\s*|[{};>,]\s*)([A-Z][A-Z0-9_]*[A-Z0-9])\s*\([^()]*\)(\s+)(?=[A-Za-z_~])" +) # An export/visibility macro between a class-like keyword and the type name, # e.g. `class LIB_API logger {`. Without this the macro is taken as the type # name and the real name is lost. -_KEYWORD_MACRO_RE = re.compile(r"\b(class|struct|union|enum)(\s+)([A-Z][A-Z0-9_]*[A-Z0-9])\s+(?=[A-Za-z_~])") +_KEYWORD_MACRO_RE = re.compile( + r"\b(class|struct|union|enum)(\s+)([A-Z][A-Z0-9_]*[A-Z0-9])\s+(?=[A-Za-z_~])" +) # A line that is nothing but a bare ALL_CAPS macro (optionally a macro call), # e.g. namespace-bracket macros like `LIB_BEGIN_NAMESPACE`. Left in place these # break parsing of the declaration that follows, so the line is blanked. Begin/ # end pairs are both removed, keeping any braces they expand to balanced. _STANDALONE_MACRO_RE = re.compile(r"^\s*([A-Z][A-Z0-9_]*[A-Z0-9])(\s*\([^()]*\))?\s*$") + class TreeSitterCppAnalyzer: - def __init__(self, file_path: str, content: str, repo_path: str = None): - self.file_path = Path(file_path) - self.content = content - self.repo_path = repo_path or "" - self.nodes: List[Node] = [] - self.call_relationships: List[CallRelationship] = [] - self._analyze() - - def _get_module_path(self) -> str: - if self.repo_path: - try: - rel_path = os.path.relpath(str(self.file_path), self.repo_path) - except ValueError: - rel_path = str(self.file_path) - else: - rel_path = str(self.file_path) - - for ext in ['.cpp', '.cc', '.cxx', '.c++', '.hpp', '.hxx', '.h++', '.h']: - if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] - break - return rel_path.replace('/', '.').replace('\\', '.') - - def _get_relative_path(self) -> str: - if self.repo_path: - try: - # BUG-21: normalize symlinks before computing relative paths - real_file = os.path.realpath(str(self.file_path)) - real_repo = os.path.realpath(self.repo_path) - return os.path.relpath(real_file, real_repo) - except ValueError: - return str(self.file_path) - else: - return str(self.file_path) - - def _get_component_id(self, name: str, parent_class: str = None) -> str: - rel_path = self._get_relative_path() - if parent_class: - return f"{rel_path}::{parent_class}.{name}" - return f"{rel_path}::{name}" - - def _analyze(self): - language_capsule = tree_sitter_cpp.language() - cpp_language = Language(language_capsule) - parser = Parser(cpp_language) - root = self._parse_with_macro_recovery(parser) - lines = self.content.splitlines() - - top_level_nodes = {} - - # collect all top-level nodes using recursive traversal - self._extract_nodes(root, top_level_nodes, lines) - - # extract relationships between top-level nodes - self._extract_relationships(root, top_level_nodes) - - def _parse_with_macro_recovery(self, parser): - """Parse the original source; if it has syntax errors, retry with macro - normalization and keep whichever parse has fewer errors. - - Normalization strips ALL_CAPS tokens by naming convention, which is - wrong for code whose *types* are ALL_CAPS (e.g. Win32 `HANDLE`/`DWORD`). - Comparing error counts makes the heuristic self-correcting: clean files - are never touched, and normalization is only kept when it demonstrably - recovers structure. - """ - tree = parser.parse(bytes(self.content, "utf8")) - if not tree.root_node.has_error: - return tree.root_node - - normalized = self._normalize_for_parser(self.content) - if normalized == self.content: - return tree.root_node - - normalized_tree = parser.parse(bytes(normalized, "utf8")) - if self._count_parse_errors(normalized_tree.root_node) < self._count_parse_errors(tree.root_node): - return normalized_tree.root_node - return tree.root_node - - def _count_parse_errors(self, root) -> int: - errors = 0 - stack = [root] - while stack: - node = stack.pop() - if node.is_error or node.is_missing: - errors += 1 - stack.extend(node.children) - return errors - - def _normalize_for_parser(self, content: str) -> str: - """Strip ALL_CAPS attribute/specifier macros that sit in front of a - declaration so tree-sitter can recover the underlying signature. This is - name-agnostic: it keys off the conventional ALL_CAPS macro spelling, not - any specific library's prefix, and only fires in specifier position so - identifiers used in expressions are preserved. Line count is unchanged so - reported line numbers stay accurate. - """ - normalized_lines = [] - for line in content.splitlines(): - updated = line - standalone = _STANDALONE_MACRO_RE.match(updated) - if standalone and is_macro_name(standalone.group(1)): - normalized_lines.append("") - continue - for pattern in (_SPECIFIER_MACRO_CALL_RE, _SPECIFIER_MACRO_RE): - previous = None - while previous != updated: - previous = updated - updated = pattern.sub( - lambda m: (m.group(1) + m.group(3)) if is_macro_name(m.group(2)) else m.group(0), - updated, - ) - updated = _KEYWORD_MACRO_RE.sub( - lambda m: (m.group(1) + m.group(2)) if is_macro_name(m.group(3)) else m.group(0), - updated, - ) - normalized_lines.append(updated) - return "\n".join(normalized_lines) - - def _extract_nodes(self, node, top_level_nodes, lines): - """Recursively extract top-level nodes (classes, functions, global variables).""" - node_type = None - node_name = None - containing_class = None - - if node.type == "class_specifier": - # "class" + type_identifier + { ... } - node_type = "class" - # Find type_identifier that represents the class name - for child in node.children: - if child.type == "type_identifier": - node_name = child.text.decode() - break - elif node.type == "struct_specifier": - # "struct" + type_identifier + { ... } - node_type = "struct" - # Find type_identifier that represents the struct name - for child in node.children: - if child.type == "type_identifier": - node_name = child.text.decode() - break - elif node.type == "function_definition": - # Check if this is inside a class or function - containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) - qualified_parts = self._get_qualified_declarator_parts(declarator) if declarator else [] - if not containing_class and len(qualified_parts) > 1: - containing_class = qualified_parts[-2] - if containing_class: - node_type = "method" - else: - node_type = "function" - - if declarator: - for child in declarator.children: - if child.type == "identifier": - node_name = child.text.decode() - break - elif child.type == "field_identifier": - node_name = child.text.decode() - break - elif child.type == "qualified_identifier": - identifiers = [c for c in child.children if c.type == "identifier"] - if identifiers: - node_name = identifiers[-1].text.decode() - break - elif node.type == "declaration": - containing_class = self._find_containing_class_for_method(node) - declarator = next((c for c in node.children if c.type == "function_declarator"), None) - if containing_class and declarator: - node_type = "method" - node_name = self._get_declarator_name(declarator) - elif self._is_global_variable(node): - node_type = "variable" - for child in node.children: - if child.type == "init_declarator": - identifier = next((c for c in child.children if c.type == "identifier"), None) - if identifier: - node_name = identifier.text.decode() - break - elif child.type == "identifier": - node_name = child.text.decode() - break - elif node.type == "alias_declaration": - # using name = type; — aliases are real API surface (e.g. a - # library's public alias for an internal template), so they are - # extracted as components and can resolve call/type references. - node_type = "type_alias" - for child in node.children: - if child.type == "type_identifier": - node_name = child.text.decode() - break - elif node.type == "type_definition": - # typedef ... name; — the alias name is the trailing type_identifier - node_type = "type_alias" - identifiers = [c for c in node.children if c.type == "type_identifier"] - if identifiers: - node_name = identifiers[-1].text.decode() - elif node.type == "namespace_definition": - node_type = "namespace" - found_namespace_keyword = False - for child in node.children: - if child.type == "namespace": - found_namespace_keyword = True - elif found_namespace_keyword and child.type == "identifier": - node_name = child.text.decode() - break - - if node_type and node_name: - if node_type == "method": - component_id = self._get_component_id(node_name, containing_class) - top_level_key = component_id - else: - component_id = self._get_component_id(node_name) - top_level_key = node_name - - relative_path = self._get_relative_path() - - node_obj = Node( - id=component_id, - name=node_name, - component_type=node_type, - file_path=str(self.file_path), - relative_path=relative_path, - source_code="\n".join(lines[node.start_point[0]:node.end_point[0]+1]), - start_line=node.start_point[0]+1, - end_line=node.end_point[0]+1, - has_docstring=False, - docstring="", - parameters=None, - node_type=node_type, - base_classes=None, - class_name=containing_class if node_type == "method" else None, - display_name=f"{node_type} {node_name}", - component_id=component_id, - language="cpp", - qualified_name=f"{containing_class}.{node_name}" if containing_class else node_name - ) - - top_level_nodes[top_level_key] = node_obj - top_level_nodes[component_id] = node_obj - if node_type == "method" and containing_class: - top_level_nodes[f"{containing_class}.{node_name}"] = node_obj - top_level_nodes.setdefault(node_name, node_obj) - - if node_type in ["class", "struct", "function", "method", "type_alias"]: - self.nodes.append(node_obj) - - # Recursively process children - for child in node.children: - self._extract_nodes(child, top_level_nodes, lines) - - def _is_global_variable(self, node) -> bool: - """Check if a declaration node is a global variable.""" - parent = node.parent - while parent: - if parent.type in ["function_definition", "class_specifier", "struct_specifier"]: - return False - parent = parent.parent - return True - - def _get_declarator_name(self, declarator) -> Optional[str]: - """Extract the declared function or method name from nested declarators.""" - for child in declarator.children: - if child.type in ["identifier", "field_identifier"]: - return child.text.decode() - if child.type == "qualified_identifier": - identifiers = [c for c in child.children if c.type in ["identifier", "field_identifier"]] - if identifiers: - return identifiers[-1].text.decode() - if child.children: - name = self._get_declarator_name(child) - if name: - return name - return None - - def _get_qualified_declarator_parts(self, declarator) -> list[str]: - if declarator is None: - return [] - for child in declarator.children: - if child.type == "qualified_identifier": - return [ - c.text.decode() - for c in child.children - if c.type in ["identifier", "field_identifier", "type_identifier", "namespace_identifier"] - ] - if child.children: - parts = self._get_qualified_declarator_parts(child) - if parts: - return parts - return [] - - def _find_containing_class_for_method(self, node): - """Find the class that contains this method definition.""" - current = node.parent - while current: - if current.type == "class_specifier": - # Get class name - for child in current.children: - if child.type == "type_identifier": - return child.text.decode() - elif current.type == "struct_specifier": - # Get struct name - for child in current.children: - if child.type == "type_identifier": - return child.text.decode() - current = current.parent - return None - - def _extract_relationships(self, node, top_level_nodes): - if node.type == "call_expression": - containing_function_id = self._find_containing_function_or_method(node, top_level_nodes) - if containing_function_id: - - # Get called function name - called_function = None - receiver_name = None - for child in node.children: - if child.type == "identifier": - called_function = child.text.decode() - break - elif child.type == "field_expression": - receiver_name, method_name = self._get_field_call_parts(child) - if method_name: - called_function = method_name - break - - if called_function: - target_method = None - if receiver_name: - receiver_type = self._find_variable_type(node, receiver_name) - if receiver_type: - target_method = self._find_method_component(called_function, top_level_nodes, receiver_type) - if not target_method: - target_method = self._find_method_component(called_function, top_level_nodes) - target_class = self._find_class_containing_method(called_function, top_level_nodes) - - if target_method: - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=target_method, - call_line=node.start_point[0]+1, - is_resolved=True - )) - elif target_class: - target_class_id = self._get_component_id(target_class) - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=target_class_id, - call_line=node.start_point[0]+1, - is_resolved=True - )) - elif called_function in top_level_nodes: - called_function_id = top_level_nodes[called_function].id - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=called_function_id, - call_line=node.start_point[0]+1, - is_resolved=True - )) - elif receiver_name is not None: - # A member call whose receiver type could not be - # resolved: a name matching an STL member here is - # overwhelmingly likely external, so suppress it. - if not self._is_system_function(called_function): - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=called_function, - call_line=node.start_point[0]+1, - is_resolved=False - )) - elif ( - not is_macro_name(called_function) - and called_function not in self._find_template_parameters(node) - ): - # Plain calls are emitted for cross-file resolution; - # external filtering happens centrally after the - # project resolver has had its chance. - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=called_function, - call_line=node.start_point[0]+1, - is_resolved=False - )) - - elif node.type == "base_class_clause": - # Find the containing class - containing_class = self._find_containing_class(node) - if containing_class: - template_params = self._find_template_parameters(node) - # Extract base class names - for child in node.children: - if child.type == "type_identifier": - base_class = child.text.decode() - if base_class in template_params or is_macro_name(base_class): - continue - containing_class_id = self._get_component_id(containing_class) - self.call_relationships.append(CallRelationship( - caller=containing_class_id, - callee=base_class, - call_line=node.start_point[0]+1, - is_resolved=False - )) - - elif node.type == "new_expression": - containing_function_id = self._find_containing_function_or_method(node, top_level_nodes) - if containing_function_id: - - # Get the class being instantiated - for child in node.children: - if child.type == "type_identifier": - class_name = child.text.decode() - if class_name in top_level_nodes: - class_id = self._get_component_id(class_name) - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=class_id, - call_line=node.start_point[0]+1, - is_resolved=True - )) - break - - elif node.type == "identifier": - parent = node.parent - if parent and parent.type not in ["function_definition", "class_specifier", "declaration", "function_declarator"]: - var_name = node.text.decode() - if var_name in top_level_nodes and top_level_nodes[var_name].component_type == "variable": - containing_function_id = self._find_containing_function_or_method(node, top_level_nodes) - if containing_function_id: - self.call_relationships.append(CallRelationship( - caller=containing_function_id, - callee=var_name, - call_line=node.start_point[0]+1, - is_resolved=False - )) - - # Recursively process children - for child in node.children: - self._extract_relationships(child, top_level_nodes) - - def _get_field_call_parts(self, field_expression) -> tuple[Optional[str], Optional[str]]: - receiver_name = None - method_name = None - for child in field_expression.children: - if child.type == "field_identifier": - method_name = child.text.decode() - elif child.type == "identifier" and receiver_name is None: - receiver_name = child.text.decode() - elif child.type == "field_expression" and receiver_name is None: - receiver_name = child.text.decode().split(".")[-1].split("->")[-1] - return receiver_name, method_name - - def _find_variable_type(self, node, variable_name: str) -> Optional[str]: - current = node.parent - while current: - if current.type in ["compound_statement", "field_declaration_list", "translation_unit"]: - found = self._search_variable_declaration(current, variable_name) - if found: - return found - if current.type == "function_definition": - declarator = next((c for c in current.children if c.type == "function_declarator"), None) - found = self._search_parameter_declaration(declarator, variable_name) - if found: - return found - current = current.parent - return None - - def _search_variable_declaration(self, node, variable_name: str) -> Optional[str]: - for child in node.children: - if child.type == "declaration": - type_name = self._get_declaration_type_name(child) - declared_name = self._get_declared_variable_name(child) - if declared_name == variable_name: - return type_name or self._get_constructor_type_name(child) - if child.children and child.type not in ["class_specifier", "struct_specifier", "function_definition"]: - found = self._search_variable_declaration(child, variable_name) - if found: - return found - return None - - def _search_parameter_declaration(self, node, variable_name: str) -> Optional[str]: - if node is None: - return None - if node.type == "parameter_declaration": - type_name = self._get_declaration_type_name(node) - declared_name = self._get_declared_variable_name(node) - if declared_name == variable_name: - return type_name - for child in node.children: - found = self._search_parameter_declaration(child, variable_name) - if found: - return found - return None - - def _get_declaration_type_name(self, node) -> Optional[str]: - for child in node.children: - if child.type in ["type_identifier", "primitive_type", "qualified_identifier"]: - return self._last_type_part(child.text.decode()) - if child.type in ["template_type", "generic_type"]: - return self._last_type_part(child.text.decode().split("<", 1)[0]) - return None - - def _get_declared_variable_name(self, node) -> Optional[str]: - for child in reversed(node.children): - if child.type in ["identifier", "field_identifier"]: - return child.text.decode() - if child.type in ["init_declarator", "pointer_declarator", "reference_declarator", "array_declarator"]: - name = self._get_declared_variable_name(child) - if name: - return name - return None - - def _get_constructor_type_name(self, node) -> Optional[str]: - for child in node.children: - if child.type == "call_expression": - for call_child in child.children: - if call_child.type in ["identifier", "type_identifier"]: - return call_child.text.decode() - if child.children: - found = self._get_constructor_type_name(child) - if found: - return found - return None - - def _last_type_part(self, type_name: str) -> str: - return type_name.strip("&* ").split("::")[-1] - - def _find_containing_function(self, node, top_level_nodes): - """Find the function that contains this node.""" - current = node.parent - while current: - if current.type == "function_definition": - # Get function name - declarator = next((c for c in current.children if c.type == "function_declarator"), None) - if declarator: - identifier = next((c for c in declarator.children if c.type == "identifier"), None) - if identifier: - func_name = identifier.text.decode() - if func_name in top_level_nodes: - return func_name - current = current.parent - return None - - def _find_containing_function_or_method(self, node, top_level_nodes): - """Find the function or method that contains this node.""" - current = node.parent - while current: - if current.type == "function_definition": - declarator = next((c for c in current.children if c.type == "function_declarator"), None) - if declarator: - func_name = self._get_declarator_name(declarator) - if func_name: - containing_class = self._find_containing_class_for_method(current) - qualified_parts = self._get_qualified_declarator_parts(declarator) - if not containing_class and len(qualified_parts) > 1: - containing_class = qualified_parts[-2] - if containing_class: - return self._get_component_id(func_name, containing_class) - return self._get_component_id(func_name) - current = current.parent - return None - - def _get_component_id_for_function(self, func_name, top_level_nodes): - if func_name in top_level_nodes: - node_obj = top_level_nodes[func_name] - if hasattr(node_obj, 'class_name') and node_obj.class_name: - return self._get_component_id(func_name, node_obj.class_name) - else: - return self._get_component_id(func_name) - return self._get_component_id(func_name) - - def _find_containing_class(self, node): - """Find the class that contains this node.""" - current = node.parent - while current: - if current.type == "class_specifier": - # Get class name - for child in current.children: - if child.type == "type_identifier": - return child.text.decode() - current = current.parent - return None - - def _find_template_parameters(self, node) -> set: - """Collect template type-parameter names in scope at this node, so a - reference to `T`/`Char`/... is not reported as an unresolved project - symbol.""" - params = set() - current = node.parent - while current: - if current.type == "template_declaration": - param_list = next( - (c for c in current.children if c.type == "template_parameter_list"), None - ) - if param_list: - for param in param_list.children: - for child in getattr(param, "children", []): - if child.type == "type_identifier": - params.add(child.text.decode()) - current = current.parent - return params - - def _is_system_function(self, func_name: str) -> bool: - """Check if a call target is external rather than a project function. - - Besides the curated standard-library set, an ALL_CAPS callee is treated as - a macro invocation: macros are not extracted as components, so a call to - one can never resolve to a project function and would otherwise pollute the - graph as unresolved noise. This only affects the unresolved fallback — - real components in ALL_CAPS (rare in C++) are matched by the earlier - resolution branches before this check runs. - """ - if is_external_symbol("cpp", func_name): - return True - return is_macro_name(func_name) - - def _find_method_component(self, method_name, top_level_nodes, class_name: str = None): - if class_name: - qualified_key = f"{class_name}.{method_name}" - if qualified_key in top_level_nodes: - return top_level_nodes[qualified_key].id - for node_obj in top_level_nodes.values(): - if node_obj.component_type == "method" and node_obj.name == method_name: - return node_obj.id - return None - - def _find_class_containing_method(self, method_name, top_level_nodes): - for node_name, node_obj in top_level_nodes.items(): - if node_obj.component_type in ["class", "struct"]: - if self._class_has_method(node_obj, method_name): - return node_name - return None - - def _class_has_method(self, class_node, method_name): - lines = class_node.source_code.split('\n') - for line in lines: - if f'{method_name}(' in line and ('void' in line or 'int' in line or 'bool' in line or class_node.name in line): - return True - return False - -def analyze_cpp_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: - analyzer = TreeSitterCppAnalyzer(file_path, content, repo_path) - return analyzer.nodes, analyzer.call_relationships + def __init__(self, file_path: str, content: str, repo_path: str = None): + self.file_path = Path(file_path) + self.content = content + self.repo_path = repo_path or "" + self.nodes: List[Node] = [] + self.call_relationships: List[CallRelationship] = [] + self._analyze() + + def _get_module_path(self) -> str: + if self.repo_path: + try: + rel_path = os.path.relpath(str(self.file_path), self.repo_path) + except ValueError: + rel_path = str(self.file_path) + else: + rel_path = str(self.file_path) + + for ext in [".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++", ".h"]: + if rel_path.endswith(ext): + rel_path = rel_path[: -len(ext)] + break + return rel_path.replace("/", ".").replace("\\", ".") + + def _get_relative_path(self) -> str: + if self.repo_path: + try: + # BUG-21: normalize symlinks before computing relative paths + real_file = os.path.realpath(str(self.file_path)) + real_repo = os.path.realpath(self.repo_path) + return os.path.relpath(real_file, real_repo) + except ValueError: + return str(self.file_path) + else: + return str(self.file_path) + + def _get_component_id(self, name: str, parent_class: str = None) -> str: + rel_path = self._get_relative_path() + if parent_class: + return f"{rel_path}::{parent_class}.{name}" + return f"{rel_path}::{name}" + + def _analyze(self): + language_capsule = tree_sitter_cpp.language() + cpp_language = Language(language_capsule) + parser = Parser(cpp_language) + root = self._parse_with_macro_recovery(parser) + lines = self.content.splitlines() + + top_level_nodes = {} + + # collect all top-level nodes using recursive traversal + self._extract_nodes(root, top_level_nodes, lines) + + # extract relationships between top-level nodes + self._extract_relationships(root, top_level_nodes) + + def _parse_with_macro_recovery(self, parser): + """Parse the original source; if it has syntax errors, retry with macro + normalization and keep whichever parse has fewer errors. + + Normalization strips ALL_CAPS tokens by naming convention, which is + wrong for code whose *types* are ALL_CAPS (e.g. Win32 `HANDLE`/`DWORD`). + Comparing error counts makes the heuristic self-correcting: clean files + are never touched, and normalization is only kept when it demonstrably + recovers structure. + """ + tree = parser.parse(bytes(self.content, "utf8")) + if not tree.root_node.has_error: + return tree.root_node + + normalized = self._normalize_for_parser(self.content) + if normalized == self.content: + return tree.root_node + + normalized_tree = parser.parse(bytes(normalized, "utf8")) + if self._count_parse_errors(normalized_tree.root_node) < self._count_parse_errors( + tree.root_node + ): + return normalized_tree.root_node + return tree.root_node + + def _count_parse_errors(self, root) -> int: + errors = 0 + stack = [root] + while stack: + node = stack.pop() + if node.is_error or node.is_missing: + errors += 1 + stack.extend(node.children) + return errors + + def _normalize_for_parser(self, content: str) -> str: + """Strip ALL_CAPS attribute/specifier macros that sit in front of a + declaration so tree-sitter can recover the underlying signature. This is + name-agnostic: it keys off the conventional ALL_CAPS macro spelling, not + any specific library's prefix, and only fires in specifier position so + identifiers used in expressions are preserved. Line count is unchanged so + reported line numbers stay accurate. + """ + normalized_lines = [] + for line in content.splitlines(): + updated = line + standalone = _STANDALONE_MACRO_RE.match(updated) + if standalone and is_macro_name(standalone.group(1)): + normalized_lines.append("") + continue + for pattern in (_SPECIFIER_MACRO_CALL_RE, _SPECIFIER_MACRO_RE): + previous = None + while previous != updated: + previous = updated + updated = pattern.sub( + lambda m: ( + (m.group(1) + m.group(3)) if is_macro_name(m.group(2)) else m.group(0) + ), + updated, + ) + updated = _KEYWORD_MACRO_RE.sub( + lambda m: (m.group(1) + m.group(2)) if is_macro_name(m.group(3)) else m.group(0), + updated, + ) + normalized_lines.append(updated) + return "\n".join(normalized_lines) + + def _extract_nodes(self, node, top_level_nodes, lines): + """Recursively extract top-level nodes (classes, functions, global variables).""" + node_type = None + node_name = None + containing_class = None + + if node.type == "class_specifier": + # "class" + type_identifier + { ... } + node_type = "class" + # Find type_identifier that represents the class name + for child in node.children: + if child.type == "type_identifier": + node_name = child.text.decode() + break + elif node.type == "struct_specifier": + # "struct" + type_identifier + { ... } + node_type = "struct" + # Find type_identifier that represents the struct name + for child in node.children: + if child.type == "type_identifier": + node_name = child.text.decode() + break + elif node.type == "function_definition": + # Check if this is inside a class or function + containing_class = self._find_containing_class_for_method(node) + declarator = next((c for c in node.children if c.type == "function_declarator"), None) + qualified_parts = self._get_qualified_declarator_parts(declarator) if declarator else [] + if not containing_class and len(qualified_parts) > 1: + containing_class = qualified_parts[-2] + if containing_class: + node_type = "method" + else: + node_type = "function" + + if declarator: + for child in declarator.children: + if child.type == "identifier": + node_name = child.text.decode() + break + elif child.type == "field_identifier": + node_name = child.text.decode() + break + elif child.type == "qualified_identifier": + identifiers = [c for c in child.children if c.type == "identifier"] + if identifiers: + node_name = identifiers[-1].text.decode() + break + elif node.type == "declaration": + containing_class = self._find_containing_class_for_method(node) + declarator = next((c for c in node.children if c.type == "function_declarator"), None) + if containing_class and declarator: + node_type = "method" + node_name = self._get_declarator_name(declarator) + elif self._is_global_variable(node): + node_type = "variable" + for child in node.children: + if child.type == "init_declarator": + identifier = next( + (c for c in child.children if c.type == "identifier"), None + ) + if identifier: + node_name = identifier.text.decode() + break + elif child.type == "identifier": + node_name = child.text.decode() + break + elif node.type == "alias_declaration": + # using name = type; — aliases are real API surface (e.g. a + # library's public alias for an internal template), so they are + # extracted as components and can resolve call/type references. + node_type = "type_alias" + for child in node.children: + if child.type == "type_identifier": + node_name = child.text.decode() + break + elif node.type == "type_definition": + # typedef ... name; — the alias name is the trailing type_identifier + node_type = "type_alias" + identifiers = [c for c in node.children if c.type == "type_identifier"] + if identifiers: + node_name = identifiers[-1].text.decode() + elif node.type == "namespace_definition": + node_type = "namespace" + found_namespace_keyword = False + for child in node.children: + if child.type == "namespace": + found_namespace_keyword = True + elif found_namespace_keyword and child.type == "identifier": + node_name = child.text.decode() + break + + if node_type and node_name: + if node_type == "method": + component_id = self._get_component_id(node_name, containing_class) + top_level_key = component_id + else: + component_id = self._get_component_id(node_name) + top_level_key = node_name + + relative_path = self._get_relative_path() + + node_obj = Node( + id=component_id, + name=node_name, + component_type=node_type, + file_path=str(self.file_path), + relative_path=relative_path, + source_code="\n".join(lines[node.start_point[0] : node.end_point[0] + 1]), + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + has_docstring=False, + docstring="", + parameters=None, + node_type=node_type, + base_classes=None, + class_name=containing_class if node_type == "method" else None, + display_name=f"{node_type} {node_name}", + component_id=component_id, + language="cpp", + qualified_name=f"{containing_class}.{node_name}" if containing_class else node_name, + ) + + top_level_nodes[top_level_key] = node_obj + top_level_nodes[component_id] = node_obj + if node_type == "method" and containing_class: + top_level_nodes[f"{containing_class}.{node_name}"] = node_obj + top_level_nodes.setdefault(node_name, node_obj) + + if node_type in ["class", "struct", "function", "method", "type_alias"]: + self.nodes.append(node_obj) + + # Recursively process children + for child in node.children: + self._extract_nodes(child, top_level_nodes, lines) + + def _is_global_variable(self, node) -> bool: + """Check if a declaration node is a global variable.""" + parent = node.parent + while parent: + if parent.type in ["function_definition", "class_specifier", "struct_specifier"]: + return False + parent = parent.parent + return True + + def _get_declarator_name(self, declarator) -> Optional[str]: + """Extract the declared function or method name from nested declarators.""" + for child in declarator.children: + if child.type in ["identifier", "field_identifier"]: + return child.text.decode() + if child.type == "qualified_identifier": + identifiers = [ + c for c in child.children if c.type in ["identifier", "field_identifier"] + ] + if identifiers: + return identifiers[-1].text.decode() + if child.children: + name = self._get_declarator_name(child) + if name: + return name + return None + + def _get_qualified_declarator_parts(self, declarator) -> list[str]: + if declarator is None: + return [] + for child in declarator.children: + if child.type == "qualified_identifier": + return [ + c.text.decode() + for c in child.children + if c.type + in ["identifier", "field_identifier", "type_identifier", "namespace_identifier"] + ] + if child.children: + parts = self._get_qualified_declarator_parts(child) + if parts: + return parts + return [] + + def _find_containing_class_for_method(self, node): + """Find the class that contains this method definition.""" + current = node.parent + while current: + if current.type == "class_specifier": + # Get class name + for child in current.children: + if child.type == "type_identifier": + return child.text.decode() + elif current.type == "struct_specifier": + # Get struct name + for child in current.children: + if child.type == "type_identifier": + return child.text.decode() + current = current.parent + return None + + def _extract_relationships(self, node, top_level_nodes): + if node.type == "call_expression": + containing_function_id = self._find_containing_function_or_method(node, top_level_nodes) + if containing_function_id: + # Get called function name + called_function = None + receiver_name = None + for child in node.children: + if child.type == "identifier": + called_function = child.text.decode() + break + elif child.type == "field_expression": + receiver_name, method_name = self._get_field_call_parts(child) + if method_name: + called_function = method_name + break + + if called_function: + target_method = None + if receiver_name: + receiver_type = self._find_variable_type(node, receiver_name) + if receiver_type: + target_method = self._find_method_component( + called_function, top_level_nodes, receiver_type + ) + if not target_method: + target_method = self._find_method_component( + called_function, top_level_nodes + ) + target_class = self._find_class_containing_method( + called_function, top_level_nodes + ) + + if target_method: + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=target_method, + call_line=node.start_point[0] + 1, + is_resolved=True, + ) + ) + elif target_class: + target_class_id = self._get_component_id(target_class) + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=target_class_id, + call_line=node.start_point[0] + 1, + is_resolved=True, + ) + ) + elif called_function in top_level_nodes: + called_function_id = top_level_nodes[called_function].id + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=called_function_id, + call_line=node.start_point[0] + 1, + is_resolved=True, + ) + ) + elif receiver_name is not None: + # A member call whose receiver type could not be + # resolved: a name matching an STL member here is + # overwhelmingly likely external, so suppress it. + if not self._is_system_function(called_function): + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=called_function, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + elif not is_macro_name( + called_function + ) and called_function not in self._find_template_parameters(node): + # Plain calls are emitted for cross-file resolution; + # external filtering happens centrally after the + # project resolver has had its chance. + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=called_function, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + elif node.type == "base_class_clause": + # Find the containing class + containing_class = self._find_containing_class(node) + if containing_class: + template_params = self._find_template_parameters(node) + # Extract base class names + for child in node.children: + if child.type == "type_identifier": + base_class = child.text.decode() + if base_class in template_params or is_macro_name(base_class): + continue + containing_class_id = self._get_component_id(containing_class) + self.call_relationships.append( + CallRelationship( + caller=containing_class_id, + callee=base_class, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + elif node.type == "new_expression": + containing_function_id = self._find_containing_function_or_method(node, top_level_nodes) + if containing_function_id: + # Get the class being instantiated + for child in node.children: + if child.type == "type_identifier": + class_name = child.text.decode() + if class_name in top_level_nodes: + class_id = self._get_component_id(class_name) + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=class_id, + call_line=node.start_point[0] + 1, + is_resolved=True, + ) + ) + break + + elif node.type == "identifier": + parent = node.parent + if parent and parent.type not in [ + "function_definition", + "class_specifier", + "declaration", + "function_declarator", + ]: + var_name = node.text.decode() + if ( + var_name in top_level_nodes + and top_level_nodes[var_name].component_type == "variable" + ): + containing_function_id = self._find_containing_function_or_method( + node, top_level_nodes + ) + if containing_function_id: + self.call_relationships.append( + CallRelationship( + caller=containing_function_id, + callee=var_name, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + # Recursively process children + for child in node.children: + self._extract_relationships(child, top_level_nodes) + + def _get_field_call_parts(self, field_expression) -> tuple[Optional[str], Optional[str]]: + receiver_name = None + method_name = None + for child in field_expression.children: + if child.type == "field_identifier": + method_name = child.text.decode() + elif child.type == "identifier" and receiver_name is None: + receiver_name = child.text.decode() + elif child.type == "field_expression" and receiver_name is None: + receiver_name = child.text.decode().split(".")[-1].split("->")[-1] + return receiver_name, method_name + + def _find_variable_type(self, node, variable_name: str) -> Optional[str]: + current = node.parent + while current: + if current.type in ["compound_statement", "field_declaration_list", "translation_unit"]: + found = self._search_variable_declaration(current, variable_name) + if found: + return found + if current.type == "function_definition": + declarator = next( + (c for c in current.children if c.type == "function_declarator"), None + ) + found = self._search_parameter_declaration(declarator, variable_name) + if found: + return found + current = current.parent + return None + + def _search_variable_declaration(self, node, variable_name: str) -> Optional[str]: + for child in node.children: + if child.type == "declaration": + type_name = self._get_declaration_type_name(child) + declared_name = self._get_declared_variable_name(child) + if declared_name == variable_name: + return type_name or self._get_constructor_type_name(child) + if child.children and child.type not in [ + "class_specifier", + "struct_specifier", + "function_definition", + ]: + found = self._search_variable_declaration(child, variable_name) + if found: + return found + return None + + def _search_parameter_declaration(self, node, variable_name: str) -> Optional[str]: + if node is None: + return None + if node.type == "parameter_declaration": + type_name = self._get_declaration_type_name(node) + declared_name = self._get_declared_variable_name(node) + if declared_name == variable_name: + return type_name + for child in node.children: + found = self._search_parameter_declaration(child, variable_name) + if found: + return found + return None + + def _get_declaration_type_name(self, node) -> Optional[str]: + for child in node.children: + if child.type in ["type_identifier", "primitive_type", "qualified_identifier"]: + return self._last_type_part(child.text.decode()) + if child.type in ["template_type", "generic_type"]: + return self._last_type_part(child.text.decode().split("<", 1)[0]) + return None + + def _get_declared_variable_name(self, node) -> Optional[str]: + for child in reversed(node.children): + if child.type in ["identifier", "field_identifier"]: + return child.text.decode() + if child.type in [ + "init_declarator", + "pointer_declarator", + "reference_declarator", + "array_declarator", + ]: + name = self._get_declared_variable_name(child) + if name: + return name + return None + + def _get_constructor_type_name(self, node) -> Optional[str]: + for child in node.children: + if child.type == "call_expression": + for call_child in child.children: + if call_child.type in ["identifier", "type_identifier"]: + return call_child.text.decode() + if child.children: + found = self._get_constructor_type_name(child) + if found: + return found + return None + + def _last_type_part(self, type_name: str) -> str: + return type_name.strip("&* ").split("::")[-1] + + def _find_containing_function(self, node, top_level_nodes): + """Find the function that contains this node.""" + current = node.parent + while current: + if current.type == "function_definition": + # Get function name + declarator = next( + (c for c in current.children if c.type == "function_declarator"), None + ) + if declarator: + identifier = next( + (c for c in declarator.children if c.type == "identifier"), None + ) + if identifier: + func_name = identifier.text.decode() + if func_name in top_level_nodes: + return func_name + current = current.parent + return None + + def _find_containing_function_or_method(self, node, top_level_nodes): + """Find the function or method that contains this node.""" + current = node.parent + while current: + if current.type == "function_definition": + declarator = next( + (c for c in current.children if c.type == "function_declarator"), None + ) + if declarator: + func_name = self._get_declarator_name(declarator) + if func_name: + containing_class = self._find_containing_class_for_method(current) + qualified_parts = self._get_qualified_declarator_parts(declarator) + if not containing_class and len(qualified_parts) > 1: + containing_class = qualified_parts[-2] + if containing_class: + return self._get_component_id(func_name, containing_class) + return self._get_component_id(func_name) + current = current.parent + return None + + def _get_component_id_for_function(self, func_name, top_level_nodes): + if func_name in top_level_nodes: + node_obj = top_level_nodes[func_name] + if hasattr(node_obj, "class_name") and node_obj.class_name: + return self._get_component_id(func_name, node_obj.class_name) + else: + return self._get_component_id(func_name) + return self._get_component_id(func_name) + + def _find_containing_class(self, node): + """Find the class that contains this node.""" + current = node.parent + while current: + if current.type == "class_specifier": + # Get class name + for child in current.children: + if child.type == "type_identifier": + return child.text.decode() + current = current.parent + return None + + def _find_template_parameters(self, node) -> set: + """Collect template type-parameter names in scope at this node, so a + reference to `T`/`Char`/... is not reported as an unresolved project + symbol.""" + params = set() + current = node.parent + while current: + if current.type == "template_declaration": + param_list = next( + (c for c in current.children if c.type == "template_parameter_list"), None + ) + if param_list: + for param in param_list.children: + for child in getattr(param, "children", []): + if child.type == "type_identifier": + params.add(child.text.decode()) + current = current.parent + return params + + def _is_system_function(self, func_name: str) -> bool: + """Check if a call target is external rather than a project function. + + Besides the curated standard-library set, an ALL_CAPS callee is treated as + a macro invocation: macros are not extracted as components, so a call to + one can never resolve to a project function and would otherwise pollute the + graph as unresolved noise. This only affects the unresolved fallback — + real components in ALL_CAPS (rare in C++) are matched by the earlier + resolution branches before this check runs. + """ + if is_external_symbol("cpp", func_name): + return True + return is_macro_name(func_name) + + def _find_method_component(self, method_name, top_level_nodes, class_name: str = None): + if class_name: + qualified_key = f"{class_name}.{method_name}" + if qualified_key in top_level_nodes: + return top_level_nodes[qualified_key].id + for node_obj in top_level_nodes.values(): + if node_obj.component_type == "method" and node_obj.name == method_name: + return node_obj.id + return None + + def _find_class_containing_method(self, method_name, top_level_nodes): + for node_name, node_obj in top_level_nodes.items(): + if node_obj.component_type in ["class", "struct"]: + if self._class_has_method(node_obj, method_name): + return node_name + return None + + def _class_has_method(self, class_node, method_name): + lines = class_node.source_code.split("\n") + for line in lines: + if f"{method_name}(" in line and ( + "void" in line or "int" in line or "bool" in line or class_node.name in line + ): + return True + return False + + +def analyze_cpp_file( + file_path: str, content: str, repo_path: str = None +) -> Tuple[List[Node], List[CallRelationship]]: + analyzer = TreeSitterCppAnalyzer(file_path, content, repo_path) + return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/csharp.py b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py index 1563ba2..e2d3271 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/csharp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py @@ -52,9 +52,26 @@ # C# primitive / contextual keyword types — never project components. _CSHARP_PRIMITIVES = { - "bool", "byte", "sbyte", "char", "decimal", "double", "float", "int", - "uint", "nint", "nuint", "long", "ulong", "short", "ushort", "string", - "object", "void", "var", "dynamic", + "bool", + "byte", + "sbyte", + "char", + "decimal", + "double", + "float", + "int", + "uint", + "nint", + "nuint", + "long", + "ulong", + "short", + "ushort", + "string", + "object", + "void", + "var", + "dynamic", } @@ -157,9 +174,15 @@ def _extract_nodes(self, node, top_level_nodes, lines): class_name = None if node.type == "class_declaration": - is_abstract = any(c.type == "modifier" and c.text.decode() == "abstract" for c in node.children) - is_static = any(c.type == "modifier" and c.text.decode() == "static" for c in node.children) - node_type = "static class" if is_static else ("abstract class" if is_abstract else "class") + is_abstract = any( + c.type == "modifier" and c.text.decode() == "abstract" for c in node.children + ) + is_static = any( + c.type == "modifier" and c.text.decode() == "static" for c in node.children + ) + node_type = ( + "static class" if is_static else ("abstract class" if is_abstract else "class") + ) node_name = self._decl_name(node) qualified_name = self._qualify(node, *self._find_containing_type_names(node), node_name) elif node.type == "interface_declaration": @@ -205,9 +228,9 @@ def _extract_nodes(self, node, top_level_nodes, lines): component_type=node_type, file_path=str(self.file_path), relative_path=self._get_relative_path(), - source_code="\n".join(lines[node.start_point[0]:node.end_point[0]+1]), - start_line=node.start_point[0]+1, - end_line=node.end_point[0]+1, + source_code="\n".join(lines[node.start_point[0] : node.end_point[0] + 1]), + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, has_docstring=has_docstring, docstring=docstring, parameters=None, @@ -265,12 +288,14 @@ def _extract_relationships(self, node, top_level_nodes): caller_id = self._get_component_id(decl_name) for base_type in self._base_list_types(base_list): if not self._skip_type(base_type, node): - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=self._resolve_cs_type(base_type, node, top_level_nodes), - call_line=node.start_point[0]+1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=self._resolve_cs_type(base_type, node, top_level_nodes), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # 2. Field / property / event type use + primary-constructor params. if node.type == "field_declaration": @@ -285,7 +310,9 @@ def _extract_relationships(self, node, top_level_nodes): if param_list: for param in param_list.children: if param.type == "parameter": - self._emit_type_use(param.child_by_field_name("type"), node, top_level_nodes) + self._emit_type_use( + param.child_by_field_name("type"), node, top_level_nodes + ) # 3. Method / function invocations. if node.type == "invocation_expression": @@ -298,12 +325,14 @@ def _extract_relationships(self, node, top_level_nodes): if containing_class and type_node: created_type = self._unwrap_type(type_node) if created_type and not self._skip_type(created_type, node): - self.call_relationships.append(CallRelationship( - caller=containing_class, - callee=self._resolve_cs_type(created_type, node, top_level_nodes), - call_line=node.start_point[0]+1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=containing_class, + callee=self._resolve_cs_type(created_type, node, top_level_nodes), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) for child in node.children: self._extract_relationships(child, top_level_nodes) @@ -316,12 +345,14 @@ def _emit_type_use(self, type_node, context_node, top_level_nodes): return type_name = self._unwrap_type(type_node) if type_name and not self._skip_type(type_name, context_node): - self.call_relationships.append(CallRelationship( - caller=containing_class, - callee=self._resolve_cs_type(type_name, context_node, top_level_nodes), - call_line=context_node.start_point[0]+1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=containing_class, + callee=self._resolve_cs_type(type_name, context_node, top_level_nodes), + call_line=context_node.start_point[0] + 1, + is_resolved=False, + ) + ) def _handle_invocation(self, node, top_level_nodes): containing_class = self._find_containing_class(node, top_level_nodes) @@ -398,12 +429,14 @@ def _handle_invocation(self, node, top_level_nodes): self._add_edge(caller_id, callee, node) def _add_edge(self, caller, callee, node): - self.call_relationships.append(CallRelationship( - caller=caller, - callee=callee, - call_line=node.start_point[0]+1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller, + callee=callee, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) def _enclosing_member_candidates(self, node, member_name): containing_types = self._find_containing_type_names(node) @@ -423,7 +456,11 @@ def _base_list_types(self, base_list): types.append(name) elif child.type == "primary_constructor_base_type": inner = next( - (c for c in child.children if c.type in ("identifier", "qualified_name", "generic_name")), + ( + c + for c in child.children + if c.type in ("identifier", "qualified_name", "generic_name") + ), None, ) name = self._unwrap_type(inner) if inner else None @@ -498,12 +535,21 @@ def _find_type_parameters(self, node) -> set: params = set() current = node while current: - if current.type in (*_TYPE_DECLS, "delegate_declaration", "method_declaration", "local_function_statement"): - type_params = next((c for c in current.children if c.type == "type_parameter_list"), None) + if current.type in ( + *_TYPE_DECLS, + "delegate_declaration", + "method_declaration", + "local_function_statement", + ): + type_params = next( + (c for c in current.children if c.type == "type_parameter_list"), None + ) if type_params: for param in type_params.children: if param.type == "type_parameter": - ident = next((c for c in param.children if c.type == "identifier"), None) + ident = next( + (c for c in param.children if c.type == "identifier"), None + ) if ident: params.add(ident.text.decode()) current = current.parent @@ -595,7 +641,9 @@ def _find_variable_type(self, node, variable_name): # Method / constructor / local-function scope: parameters and locals. method_node = node.parent while method_node and method_node.type not in ( - "method_declaration", "constructor_declaration", "local_function_statement" + "method_declaration", + "constructor_declaration", + "local_function_statement", ): method_node = method_node.parent @@ -624,7 +672,9 @@ def _find_variable_type(self, node, variable_name): if body: for member in body.children: if member.type in ("field_declaration", "event_field_declaration"): - vd = next((c for c in member.children if c.type == "variable_declaration"), None) + vd = next( + (c for c in member.children if c.type == "variable_declaration"), None + ) if vd and self._declares_variable(vd, variable_name): return self._unwrap_type(self._first_type_child(vd)) elif member.type == "property_declaration": @@ -668,15 +718,26 @@ def _search_variable_declaration(self, block_node, variable_name): if type_node is not None and type_node.type != "implicit_type": return self._unwrap_type(type_node) # `var` — recover the type only from a `new T()` initializer. - init = next((c for c in decl.children if c.type == "object_creation_expression"), None) + init = next( + (c for c in decl.children if c.type == "object_creation_expression"), None + ) if init is not None: return self._unwrap_type(init.child_by_field_name("type")) return None elif child.type in ( - "block", "if_statement", "else_clause", "for_statement", - "foreach_statement", "while_statement", "do_statement", - "using_statement", "try_statement", "catch_clause", - "finally_clause", "switch_statement", "lock_statement", + "block", + "if_statement", + "else_clause", + "for_statement", + "foreach_statement", + "while_statement", + "do_statement", + "using_statement", + "try_statement", + "catch_clause", + "finally_clause", + "switch_statement", + "lock_statement", "switch_section", ): result = self._search_variable_declaration(child, variable_name) @@ -685,6 +746,8 @@ def _search_variable_declaration(self, block_node, variable_name): return None -def analyze_csharp_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: +def analyze_csharp_file( + file_path: str, content: str, repo_path: str = None +) -> Tuple[List[Node], List[CallRelationship]]: analyzer = TreeSitterCSharpAnalyzer(file_path, content, repo_path) return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/go.py b/codewiki/src/be/dependency_analyzer/analyzers/go.py index b718fa4..e29a12f 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/go.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/go.py @@ -17,14 +17,41 @@ # Go built-in types and common standard-library types to filter out GO_PRIMITIVE_TYPES: Set[str] = { - "bool", "byte", "rune", "string", "error", - "int", "int8", "int16", "int32", "int64", - "uint", "uint8", "uint16", "uint32", "uint64", "uintptr", - "float32", "float64", "complex64", "complex128", - "any", "comparable", - "Context", "Writer", "Reader", "Handler", "Request", "Response", - "Mutex", "RWMutex", "WaitGroup", "Once", "Pool", - "Buffer", "StringsBuilder", + "bool", + "byte", + "rune", + "string", + "error", + "int", + "int8", + "int16", + "int32", + "int64", + "uint", + "uint8", + "uint16", + "uint32", + "uint64", + "uintptr", + "float32", + "float64", + "complex64", + "complex128", + "any", + "comparable", + "Context", + "Writer", + "Reader", + "Handler", + "Request", + "Response", + "Mutex", + "RWMutex", + "WaitGroup", + "Once", + "Pool", + "Buffer", + "StringsBuilder", } @@ -93,8 +120,7 @@ def _extract_nodes(self, node, top_level_names: dict, lines): (c for c in type_spec.children if c.type == "type_identifier"), None ) type_body = next( - (c for c in type_spec.children - if c.type in ("struct_type", "interface_type")), + (c for c in type_spec.children if c.type in ("struct_type", "interface_type")), None, ) if not name_node or not type_body: @@ -107,9 +133,7 @@ def _extract_nodes(self, node, top_level_names: dict, lines): node_name = tname elif node.type == "function_declaration": - name_node = next( - (c for c in node.children if c.type == "identifier"), None - ) + name_node = next((c for c in node.children if c.type == "identifier"), None) if name_node: node_type = "function" node_name = name_node.text.decode() @@ -180,23 +204,17 @@ def _extract_relationships(self, node, top_level_names: dict): name_node = next( (c for c in type_spec.children if c.type == "type_identifier"), None ) - struct_body = next( - (c for c in type_spec.children if c.type == "struct_type"), None - ) + struct_body = next((c for c in type_spec.children if c.type == "struct_type"), None) if name_node and struct_body: struct_name = name_node.text.decode() - self._extract_struct_dependencies( - struct_body, struct_name, top_level_names - ) + self._extract_struct_dependencies(struct_body, struct_name, top_level_names) iface_body = next( (c for c in type_spec.children if c.type == "interface_type"), None ) if name_node and iface_body: iface_name = name_node.text.decode() - self._extract_interface_dependencies( - iface_body, iface_name, top_level_names - ) + self._extract_interface_dependencies(iface_body, iface_name, top_level_names) if node.type == "call_expression": caller_id = self._find_containing_function(node) @@ -206,18 +224,18 @@ def _extract_relationships(self, node, top_level_names: dict): if node.type == "composite_literal": caller_id = self._find_containing_function(node) if caller_id: - type_node = next( - (c for c in node.children if c.type == "type_identifier"), None - ) + type_node = next((c for c in node.children if c.type == "type_identifier"), None) if type_node: type_name = type_node.text.decode() if not self._is_primitive_type(type_name): - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=self._get_component_id(type_name), - call_line=node.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=self._get_component_id(type_name), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) for child in node.children: self._extract_relationships(child, top_level_names) @@ -235,40 +253,44 @@ def _extract_struct_dependencies(self, struct_body, struct_name: str, top_level_ if len(children) == 1: embedded_type = self._resolve_type_name(children[0]) if embedded_type and not self._is_primitive_type(embedded_type): - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(struct_name), - callee=self._get_component_id(embedded_type), - call_line=field.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(struct_name), + callee=self._get_component_id(embedded_type), + call_line=field.start_point[0] + 1, + is_resolved=False, + ) + ) else: type_node = field.children[-1] if field.children else None if type_node: field_type = self._resolve_type_name(type_node) if field_type and not self._is_primitive_type(field_type): - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(struct_name), - callee=self._get_component_id(field_type), - call_line=field.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(struct_name), + callee=self._get_component_id(field_type), + call_line=field.start_point[0] + 1, + is_resolved=False, + ) + ) def _extract_interface_dependencies(self, iface_body, iface_name: str, top_level_names: dict): - method_list = next( - (c for c in iface_body.children if c.type == "method_spec_list"), None - ) + method_list = next((c for c in iface_body.children if c.type == "method_spec_list"), None) if not method_list: return for spec in method_list.children: if spec.type == "type_identifier": embedded = spec.text.decode() if not self._is_primitive_type(embedded): - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(iface_name), - callee=self._get_component_id(embedded), - call_line=spec.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(iface_name), + callee=self._get_component_id(embedded), + call_line=spec.start_point[0] + 1, + is_resolved=False, + ) + ) def _extract_call_target(self, call_node, caller_id: str, top_level_names: dict): func_node = call_node.children[0] if call_node.children else None @@ -278,21 +300,31 @@ def _extract_call_target(self, call_node, caller_id: str, top_level_names: dict) if func_node.type == "identifier": callee_name = func_node.text.decode() if not self._is_primitive_type(callee_name): - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=self._get_component_id(callee_name), - call_line=call_node.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=self._get_component_id(callee_name), + call_line=call_node.start_point[0] + 1, + is_resolved=False, + ) + ) elif func_node.type == "selector_expression": operand = next( - (c for c in func_node.children if c.type in ("identifier", "selector_expression", "call_expression", "parenthesized_expression")), + ( + c + for c in func_node.children + if c.type + in ( + "identifier", + "selector_expression", + "call_expression", + "parenthesized_expression", + ) + ), None, ) - field = next( - (c for c in func_node.children if c.type == "field_identifier"), None - ) + field = next((c for c in func_node.children if c.type == "field_identifier"), None) if operand and field: method_name = field.text.decode() operand_name = operand.text.decode() @@ -301,28 +333,28 @@ def _extract_call_target(self, call_node, caller_id: str, top_level_names: dict) if operand_name in top_level_names: target_type = operand_name else: - target_type = self._find_variable_type( - call_node, operand_name, top_level_names - ) + target_type = self._find_variable_type(call_node, operand_name, top_level_names) if target_type and not self._is_primitive_type(target_type): - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=self._get_component_id(method_name, target_type), - call_line=call_node.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=self._get_component_id(method_name, target_type), + call_line=call_node.start_point[0] + 1, + is_resolved=False, + ) + ) else: - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=method_name, - call_line=call_node.start_point[0] + 1, - is_resolved=False, - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=method_name, + call_line=call_node.start_point[0] + 1, + is_resolved=False, + ) + ) def _get_receiver_type(self, method_node) -> Optional[str]: - param_list = next( - (c for c in method_node.children if c.type == "parameter_list"), None - ) + param_list = next((c for c in method_node.children if c.type == "parameter_list"), None) if not param_list: return None for param in param_list.children: @@ -336,25 +368,47 @@ def _resolve_type_name(self, node) -> Optional[str]: if node.type == "type_identifier": return node.text.decode() elif node.type == "pointer_type": - inner = next((c for c in node.children if c.type in ("type_identifier", "qualified_type", "pointer_type")), None) + inner = next( + ( + c + for c in node.children + if c.type in ("type_identifier", "qualified_type", "pointer_type") + ), + None, + ) if inner: return self._resolve_type_name(inner) elif node.type == "slice_type": - inner = next((c for c in node.children if c.type in ("type_identifier", "pointer_type", "qualified_type")), None) + inner = next( + ( + c + for c in node.children + if c.type in ("type_identifier", "pointer_type", "qualified_type") + ), + None, + ) if inner: return self._resolve_type_name(inner) elif node.type == "array_type": - inner = next((c for c in node.children if c.type in ("type_identifier", "pointer_type")), None) + inner = next( + (c for c in node.children if c.type in ("type_identifier", "pointer_type")), None + ) if inner: return self._resolve_type_name(inner) elif node.type == "map_type": - children = [c for c in node.children if c.type in ("type_identifier", "pointer_type", "slice_type", "qualified_type")] + children = [ + c + for c in node.children + if c.type in ("type_identifier", "pointer_type", "slice_type", "qualified_type") + ] if len(children) >= 2: return self._resolve_type_name(children[1]) elif len(children) == 1: return self._resolve_type_name(children[0]) elif node.type == "channel_type": - inner = next((c for c in node.children if c.type in ("type_identifier", "pointer_type")), None) + inner = next( + (c for c in node.children if c.type in ("type_identifier", "pointer_type")), None + ) if inner: return self._resolve_type_name(inner) elif node.type == "qualified_type": @@ -368,9 +422,7 @@ def _extract_parameters(self, node) -> Optional[List[str]]: params = [] param_list = None if node.type == "function_declaration": - param_list = next( - (c for c in node.children if c.type == "parameter_list"), None - ) + param_list = next((c for c in node.children if c.type == "parameter_list"), None) elif node.type == "method_declaration": lists = [c for c in node.children if c.type == "parameter_list"] if len(lists) >= 2: @@ -383,11 +435,7 @@ def _extract_parameters(self, node) -> Optional[List[str]]: for param in param_list.children: if param.type == "parameter_declaration": - names = [ - c.text.decode() - for c in param.children - if c.type == "identifier" - ] + names = [c.text.decode() for c in param.children if c.type == "identifier"] type_node = param.children[-1] if param.children else None type_str = self._resolve_type_name(type_node) if type_node else "unknown" if names: @@ -401,9 +449,7 @@ def _find_containing_function(self, node) -> Optional[str]: current = node.parent while current: if current.type == "function_declaration": - name_node = next( - (c for c in current.children if c.type == "identifier"), None - ) + name_node = next((c for c in current.children if c.type == "identifier"), None) if name_node: return self._get_component_id(name_node.text.decode()) elif current.type == "method_declaration": @@ -412,15 +458,11 @@ def _find_containing_function(self, node) -> Optional[str]: ) if method_name_node: recv = self._get_receiver_type(current) - return self._get_component_id( - method_name_node.text.decode(), recv - ) + return self._get_component_id(method_name_node.text.decode(), recv) current = current.parent return None - def _find_variable_type( - self, node, variable_name: str, top_level_names: dict - ) -> Optional[str]: + def _find_variable_type(self, node, variable_name: str, top_level_names: dict) -> Optional[str]: func_node = node.parent while func_node and func_node.type not in ( "function_declaration", @@ -478,17 +520,17 @@ def _find_variable_type( def _search_short_var_decl(self, block_node, variable_name: str) -> Optional[str]: for child in block_node.children: if child.type == "short_var_declaration": - lhs = next( - (c for c in child.children if c.type == "expression_list"), None - ) - rhs_nodes = [ - c for c in child.children if c.type == "expression_list" - ] + lhs = next((c for c in child.children if c.type == "expression_list"), None) + rhs_nodes = [c for c in child.children if c.type == "expression_list"] rhs = rhs_nodes[1] if len(rhs_nodes) >= 2 else None if lhs and rhs: names = [c for c in lhs.children if c.type == "identifier"] - calls = [c for c in rhs.children if c.type in ("call_expression", "composite_literal")] + calls = [ + c + for c in rhs.children + if c.type in ("call_expression", "composite_literal") + ] for i, name_node in enumerate(names): if name_node.text.decode() == variable_name: if i < len(calls): @@ -541,13 +583,47 @@ def _is_primitive_type(type_name: str) -> bool: @staticmethod def _is_stdlib_package(name: str) -> bool: stdlib = { - "fmt", "log", "os", "io", "net", "http", "json", "xml", - "strings", "strconv", "math", "time", "context", "errors", - "sync", "path", "filepath", "regexp", "sort", "bytes", - "bufio", "encoding", "reflect", "runtime", "testing", - "crypto", "database", "sql", "html", "text", "flag", - "exec", "signal", "syscall", "unsafe", "atomic", - "rand", "hash", "compress", "archive", "container", + "fmt", + "log", + "os", + "io", + "net", + "http", + "json", + "xml", + "strings", + "strconv", + "math", + "time", + "context", + "errors", + "sync", + "path", + "filepath", + "regexp", + "sort", + "bytes", + "bufio", + "encoding", + "reflect", + "runtime", + "testing", + "crypto", + "database", + "sql", + "html", + "text", + "flag", + "exec", + "signal", + "syscall", + "unsafe", + "atomic", + "rand", + "hash", + "compress", + "archive", + "container", } return name in stdlib diff --git a/codewiki/src/be/dependency_analyzer/analyzers/java.py b/codewiki/src/be/dependency_analyzer/analyzers/java.py index 0e8367c..072df0f 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/java.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/java.py @@ -1,7 +1,6 @@ import logging -from typing import List, Optional, Tuple +from typing import List, Tuple from pathlib import Path -import sys import os import re @@ -9,533 +8,615 @@ import tree_sitter_java from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( - JAVA_OBJECT_METHODS, - is_external_symbol, + JAVA_OBJECT_METHODS, + is_external_symbol, ) logger = logging.getLogger(__name__) + class TreeSitterJavaAnalyzer: - def __init__(self, file_path: str, content: str, repo_path: str = None): - self.file_path = Path(file_path) - self.content = content - self.repo_path = repo_path or "" - self.nodes: List[Node] = [] - self.call_relationships: List[CallRelationship] = [] - self.package_name = self._extract_package_name() - self.import_map, self.wildcard_imports = self._extract_imports() - self._analyze() - - def _get_module_path(self) -> str: - if self.repo_path: - try: - rel_path = os.path.relpath(str(self.file_path), self.repo_path) - except ValueError: - rel_path = str(self.file_path) - else: - rel_path = str(self.file_path) - - for ext in ['.java']: - if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] - break - return rel_path.replace('/', '.').replace('\\', '.') - - def _get_relative_path(self) -> str: - """Get relative path from repo root.""" - if self.repo_path: - try: - # BUG-21: normalize symlinks before computing relative paths - real_file = os.path.realpath(str(self.file_path)) - real_repo = os.path.realpath(self.repo_path) - return os.path.relpath(real_file, real_repo) - except ValueError: - return str(self.file_path) - else: - return str(self.file_path) - - def _get_component_id(self, name: str, parent_class: str = None) -> str: - rel_path = self._get_relative_path() - if parent_class: - return f"{rel_path}::{parent_class}.{name}" - else: - return f"{rel_path}::{name}" - - def _extract_package_name(self) -> str: - match = re.search(r"^\s*package\s+([\w.]+)\s*;", self.content, re.MULTILINE) - return match.group(1) if match else "" - - def _extract_imports(self) -> tuple[dict[str, str], list[str]]: - import_map: dict[str, str] = {} - wildcards: list[str] = [] - for match in re.finditer(r"^\s*import\s+(?:static\s+)?([\w.]+)(\.\*)?\s*;", self.content, re.MULTILINE): - import_name = match.group(1) - if match.group(2): - wildcards.append(import_name) - else: - import_map[import_name.rsplit(".", 1)[-1]] = import_name - return import_map, wildcards - - def _analyze(self): - language_capsule = tree_sitter_java.language() - java_language = Language(language_capsule) - parser = Parser(java_language) - tree = parser.parse(bytes(self.content, "utf8")) - root = tree.root_node - lines = self.content.splitlines() - - top_level_nodes = {} - - self._extract_nodes(root, top_level_nodes, lines) - - self._extract_relationships(root, top_level_nodes) - - def _extract_nodes(self, node, top_level_nodes, lines): - node_type = None - node_name = None - qualified_name = None - class_name = None - - if node.type == "class_declaration": - is_abstract = any(c.type == "modifier" and c.text.decode() == "abstract" for c in node.children) - node_type = "abstract class" if is_abstract else "class" - name_node = next((c for c in node.children if c.type == "identifier"), None) - node_name = name_node.text.decode() if name_node else None - qualified_name = self._qualified_type_name(node_name, self._find_containing_type_names(node)) - elif node.type == "interface_declaration": - node_type = "interface" - name_node = next((c for c in node.children if c.type == "identifier"), None) - node_name = name_node.text.decode() if name_node else None - qualified_name = self._qualified_type_name(node_name, self._find_containing_type_names(node)) - elif node.type == "enum_declaration": - node_type = "enum" - name_node = next((c for c in node.children if c.type == "identifier"), None) - node_name = name_node.text.decode() if name_node else None - qualified_name = self._qualified_type_name(node_name, self._find_containing_type_names(node)) - elif node.type == "record_declaration": - node_type = "record" - name_node = next((c for c in node.children if c.type == "identifier"), None) - node_name = name_node.text.decode() if name_node else None - qualified_name = self._qualified_type_name(node_name, self._find_containing_type_names(node)) - elif node.type == "annotation_type_declaration": - node_type = "annotation" - name_node = next((c for c in node.children if c.type == "identifier"), None) - node_name = name_node.text.decode() if name_node else None - qualified_name = self._qualified_type_name(node_name, self._find_containing_type_names(node)) - elif node.type == "method_declaration": - node_type = "method" - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - method_name = name_node.text.decode() - containing_types = self._find_containing_type_names(node) - if containing_types: - class_name = containing_types[-1] - node_name = f"{class_name}.{method_name}" - qualified_name = self._qualified_member_name(containing_types, method_name) - else: - node_name = method_name - qualified_name = self._qualify_name(method_name) - - if node_type and node_name: - component_id = self._get_component_id(node_name) - relative_path = self._get_relative_path() - node_obj = Node( - id=component_id, - name=node_name, - component_type=node_type, - file_path=str(self.file_path), - relative_path=relative_path, - source_code="\n".join(lines[node.start_point[0]:node.end_point[0]+1]), - start_line=node.start_point[0]+1, - end_line=node.end_point[0]+1, - has_docstring=False, - docstring="", - parameters=None, - node_type=node_type, - base_classes=None, - class_name=class_name, - display_name=f"{node_type} {node_name}", - component_id=component_id, - language="java", - qualified_name=qualified_name - ) - self.nodes.append(node_obj) - top_level_nodes[node_name] = node_obj - top_level_nodes[component_id] = node_obj - if qualified_name: - top_level_nodes[qualified_name] = node_obj - top_level_nodes.setdefault(qualified_name.split(".")[-1], node_obj) - - # Recursively process children - for child in node.children: - self._extract_nodes(child, top_level_nodes, lines) - - def _extract_relationships(self, node, top_level_nodes): - # 1. Inheritance: Class extends another class - if node.type == "class_declaration": - class_name = self._get_identifier_name(node) - children_types = [c.type for c in node.children] - - extends_node = next((c for c in node.children if c.type == "superclass"), None) - if extends_node: - base_class_name = self._get_type_name(extends_node) - if class_name and base_class_name and not self._skip_type(base_class_name, node): - caller_id = self._get_component_id(class_name) - callee_id = self._resolve_java_type(base_class_name, node, top_level_nodes) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_id, - call_line=node.start_point[0]+1, - is_resolved=False - )) - else: - logger.debug(f" No superclass found for {class_name}") - - # 2. Interface Implementation: Class/enum/record implements interface - if node.type in ["class_declaration", "enum_declaration", "record_declaration"]: - implementer_name = self._get_identifier_name(node) - implements_node = next((c for c in node.children if c.type == "super_interfaces"), None) - if implements_node and implementer_name: - for child in implements_node.children: - if child.type == "type_list": - for type_child in child.children: - if type_child.type in ["type_identifier", "generic_type"]: - interface_name = self._get_type_name(type_child) - if interface_name and not self._skip_type(interface_name, node): - caller_id = self._get_component_id(implementer_name) - callee_id = self._resolve_java_type(interface_name, node, top_level_nodes) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_id, - call_line=node.start_point[0]+1, - is_resolved=False - )) - - # 3. Field Type Use: Class has field of another class/interface type - if node.type == "field_declaration": - containing_class = self._find_containing_class(node, top_level_nodes) - type_node = next((c for c in node.children if c.type in ["type_identifier", "generic_type"]), None) - if containing_class and type_node: - field_type_name = self._get_type_name(type_node) - if field_type_name and not self._skip_type(field_type_name, node): - self.call_relationships.append(CallRelationship( - caller=containing_class, - callee=self._resolve_java_type(field_type_name, node, top_level_nodes), - call_line=node.start_point[0]+1, - is_resolved=False - )) - - # 4. Method Calls: Method calls on objects - if node.type == "method_invocation": - containing_class = self._find_containing_class(node, top_level_nodes) - containing_method = self._find_containing_method(node) - if containing_class: - object_name = None - method_name = None - - identifiers = [child.text.decode() for child in node.children if child.type == "identifier"] - if len(identifiers) >= 2: - object_name = identifiers[0] - method_name = identifiers[1] - elif identifiers: - method_name = identifiers[0] - - if method_name: - target_type = None - - caller_id = containing_method or containing_class - - if object_name and object_name[:1].isupper() and object_name in top_level_nodes: - target_type = object_name - elif object_name: - target_type = self._find_variable_type(node, object_name, top_level_nodes) - if not target_type and object_name in top_level_nodes: - target_type = object_name - if not target_type and object_name[:1].isupper() and not object_name.isupper(): - # CamelCase receiver with no matching variable reads - # as a static call on a type from another file or an - # import; ALL_CAPS receivers are constants, not types. - target_type = object_name - - if target_type and not self._skip_type(target_type, node): - callee = self._resolve_java_member(method_name, node, top_level_nodes, target_type) - if callee not in top_level_nodes and method_name in JAVA_OBJECT_METHODS: - # Inherited java.lang.Object method that the project - # type does not override locally — never a project edge. - callee = None - if callee: - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee, - call_line=node.start_point[0]+1, - is_resolved=False - )) - elif not object_name: - callee = self._resolve_java_member(method_name, node, top_level_nodes) - if callee in top_level_nodes or self.import_map.get(method_name) == callee: - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee, - call_line=node.start_point[0]+1, - is_resolved=False - )) - - # 5. Object Creation - if node.type == "object_creation_expression": - containing_class = self._find_containing_class(node, top_level_nodes) - type_node = next((c for c in node.children if c.type in ["type_identifier", "generic_type"]), None) - if containing_class and type_node: - created_type = self._get_type_name(type_node) - if created_type and not self._skip_type(created_type, node): - self.call_relationships.append(CallRelationship( - caller=containing_class, - callee=self._resolve_java_type(created_type, node, top_level_nodes), - call_line=node.start_point[0]+1, - is_resolved=False - )) - - # Recursively process children - for child in node.children: - self._extract_relationships(child, top_level_nodes) - - def _is_primitive_type(self, type_name: str) -> bool: - """Check if type is a Java primitive or a JDK/runtime type.""" - primitives = { - "boolean", "byte", "char", "double", "float", "int", "long", "short", - "void", "var", - } - simple = self._simple_type_name(type_name) - if simple in primitives: - return True - # Resolve through the import map first so a runtime type written with its - # simple name (imported from a `javax.*`/`java.*` package) is judged by its - # fully-qualified origin. The prefix rules in is_external_symbol then - # filter JDK/runtime packages, while project types — including sibling - # packages like `com.other.Bar` — fall through and resolve cross-file. This - # generalizes JDK filtering to any repository without enumerating types. - # java.lang types (no import to consult) are covered by the curated set - # inside is_external_symbol. - qualified = self.import_map.get(simple) - if qualified is None: - # A wildcard import of a JDK package (`import java.util.*;`) is the - # only way a JDK type outside java.lang appears with no explicit - # import; project wildcard packages fall through to resolution. - for wildcard in self.wildcard_imports: - if is_external_symbol("java", f"{wildcard}.{simple}"): - return True - qualified = simple - return is_external_symbol("java", qualified) - - def _resolve_java_type(self, type_name: str, context_node=None, top_level_nodes=None) -> str: - if not type_name: - return type_name - type_name = self._simple_type_name(type_name) - if "." in type_name: - return type_name - if type_name in self.import_map: - return self.import_map[type_name] - if context_node is not None and top_level_nodes is not None: - containing_types = self._find_containing_type_names(context_node) - for idx in range(len(containing_types), 0, -1): - candidate = self._qualify_name(".".join([*containing_types[:idx], type_name])) - if candidate in top_level_nodes: - return candidate - if self.package_name: - return f"{self.package_name}.{type_name}" - return type_name - - def _resolve_java_member(self, member_name: str, context_node, top_level_nodes, target_type: str = None) -> str: - if target_type: - qualified_type = self._resolve_java_type(target_type, context_node, top_level_nodes) - candidate = f"{qualified_type}.{member_name}" - if candidate in top_level_nodes: - return candidate - simple_type = qualified_type.split(".")[-1] - simple_candidate = f"{simple_type}.{member_name}" - if simple_candidate in top_level_nodes: - return simple_candidate - return candidate - - containing_types = self._find_containing_type_names(context_node) - for idx in range(len(containing_types), 0, -1): - candidate = self._qualified_member_name(containing_types[:idx], member_name) - if candidate in top_level_nodes: - return candidate - # A static import maps the bare call to its declaring type, whether - # project (`com.foo.Util.checkNotNull`) or JDK (`java.util.Objects.requireNonNull`). - if member_name in self.import_map: - return self.import_map[member_name] - return self._qualify_name(member_name) - - def _skip_type(self, type_name: str, context_node) -> bool: - """Types that can never be project components: primitives, JDK/runtime - types, and generic type parameters in scope (e.g. the `K`/`V` of an - enclosing `class Cache`).""" - if self._is_primitive_type(type_name): - return True - return self._simple_type_name(type_name) in self._find_type_parameters(context_node) - - def _find_type_parameters(self, node) -> set: - params = set() - current = node - while current: - if current.type in [ - "class_declaration", - "interface_declaration", - "record_declaration", - "method_declaration", - ]: - type_parameters = next( - (c for c in current.children if c.type == "type_parameters"), None - ) - if type_parameters: - for param in type_parameters.children: - if param.type == "type_parameter": - for child in param.children: - if child.type in ["type_identifier", "identifier"]: - params.add(child.text.decode()) - break - current = current.parent - return params - - def _simple_type_name(self, type_name: str) -> str: - return type_name.strip().split("<", 1)[0].strip() - - def _qualify_name(self, name: str) -> str: - return f"{self.package_name}.{name}" if self.package_name else name - - def _qualified_type_name(self, name: str, containing_types: list[str]) -> str: - parts = [*containing_types, name] if name else containing_types - return self._qualify_name(".".join(parts)) if parts else "" - - def _qualified_member_name(self, containing_types: list[str], member_name: str) -> str: - return self._qualify_name(".".join([*containing_types, member_name])) - - def _get_identifier_name(self, node): - """Get identifier name from a node.""" - name_node = next((c for c in node.children if c.type == "identifier"), None) - return name_node.text.decode() if name_node else None - - def _get_type_name(self, node): - """Get type name from a type node.""" - if node.type == "type_identifier": - return node.text.decode() - elif node.type == "generic_type": - type_node = next((c for c in node.children if c.type == "type_identifier"), None) - return type_node.text.decode() if type_node else None - elif node.type == "superclass": - type_node = next((c for c in node.children if c.type == "type_identifier"), None) - return type_node.text.decode() if type_node else None - return None - - def _find_containing_class(self, node, top_level_nodes): - current = node.parent - while current: - if current.type in ["class_declaration", "interface_declaration", "enum_declaration", "record_declaration", "annotation_type_declaration"]: - class_name = self._get_identifier_name(current) - if class_name and class_name in top_level_nodes: - return self._get_component_id(class_name) - current = current.parent - return None - - def _find_variable_type(self, node, variable_name, top_level_nodes): - method_node = node.parent - while method_node and method_node.type not in ["method_declaration", "constructor_declaration"]: - method_node = method_node.parent - - if method_node: - for child in method_node.children: - if child.type == "block" or child.type == "constructor_body": - variable_type = self._search_variable_declaration(child, variable_name) - if variable_type: - return variable_type - elif child.type == "formal_parameters": - for param in child.children: - if param.type in ["formal_parameter", "spread_parameter"]: - type_node = next( - (c for c in param.children if c.type in ["type_identifier", "generic_type"]), - None, - ) - identifier_node = next( - (c for c in param.children if c.type == "identifier"), None - ) - if ( - type_node - and identifier_node - and identifier_node.text.decode() == variable_name - ): - return self._get_type_name(type_node) - - class_node = node.parent - while class_node and class_node.type != "class_declaration": - class_node = class_node.parent - - if class_node: - for child in class_node.children: - if child.type == "class_body": - for body_child in child.children: - if body_child.type == "field_declaration": - identifier_node = None - type_node = None - for field_child in body_child.children: - if field_child.type in ["type_identifier", "generic_type"]: - type_node = field_child - elif field_child.type == "variable_declarator": - identifier_node = next((c for c in field_child.children if c.type == "identifier"), None) - - if identifier_node and type_node and identifier_node.text.decode() == variable_name: - field_type = self._get_type_name(type_node) - return field_type - - return None - - def _search_variable_declaration(self, block_node, variable_name): - for child in block_node.children: - if child.type == "local_variable_declaration": - type_node = None - identifier_node = None - for decl_child in child.children: - if decl_child.type in ["type_identifier", "generic_type"]: - type_node = decl_child - elif decl_child.type == "variable_declarator": - identifier_node = next((c for c in decl_child.children if c.type == "identifier"), None) - - if identifier_node and type_node and identifier_node.text.decode() == variable_name: - return self._get_type_name(type_node) - - elif child.type == "block": - result = self._search_variable_declaration(child, variable_name) - if result: - return result - - return None - - def _find_containing_class_name(self, node): - names = self._find_containing_type_names(node) - return names[-1] if names else None - - def _find_containing_type_names(self, node) -> list[str]: - names = [] - current = node.parent - while current: - if current.type in ["class_declaration", "interface_declaration", "enum_declaration", "record_declaration", "annotation_type_declaration"]: - name_node = next((c for c in current.children if c.type == "identifier"), None) - if name_node: - names.append(name_node.text.decode()) - current = current.parent - return list(reversed(names)) - - def _find_containing_method(self, node): - current = node.parent - while current: - if current.type == "method_declaration": - method_name = self._get_identifier_name(current) - class_name = self._find_containing_class_name(current) - if method_name and class_name: - return self._get_component_id(f"{class_name}.{method_name}") - current = current.parent - return None - -def analyze_java_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: - analyzer = TreeSitterJavaAnalyzer(file_path, content, repo_path) - return analyzer.nodes, analyzer.call_relationships + def __init__(self, file_path: str, content: str, repo_path: str = None): + self.file_path = Path(file_path) + self.content = content + self.repo_path = repo_path or "" + self.nodes: List[Node] = [] + self.call_relationships: List[CallRelationship] = [] + self.package_name = self._extract_package_name() + self.import_map, self.wildcard_imports = self._extract_imports() + self._analyze() + + def _get_module_path(self) -> str: + if self.repo_path: + try: + rel_path = os.path.relpath(str(self.file_path), self.repo_path) + except ValueError: + rel_path = str(self.file_path) + else: + rel_path = str(self.file_path) + + for ext in [".java"]: + if rel_path.endswith(ext): + rel_path = rel_path[: -len(ext)] + break + return rel_path.replace("/", ".").replace("\\", ".") + + def _get_relative_path(self) -> str: + """Get relative path from repo root.""" + if self.repo_path: + try: + # BUG-21: normalize symlinks before computing relative paths + real_file = os.path.realpath(str(self.file_path)) + real_repo = os.path.realpath(self.repo_path) + return os.path.relpath(real_file, real_repo) + except ValueError: + return str(self.file_path) + else: + return str(self.file_path) + + def _get_component_id(self, name: str, parent_class: str = None) -> str: + rel_path = self._get_relative_path() + if parent_class: + return f"{rel_path}::{parent_class}.{name}" + else: + return f"{rel_path}::{name}" + + def _extract_package_name(self) -> str: + match = re.search(r"^\s*package\s+([\w.]+)\s*;", self.content, re.MULTILINE) + return match.group(1) if match else "" + + def _extract_imports(self) -> tuple[dict[str, str], list[str]]: + import_map: dict[str, str] = {} + wildcards: list[str] = [] + for match in re.finditer( + r"^\s*import\s+(?:static\s+)?([\w.]+)(\.\*)?\s*;", self.content, re.MULTILINE + ): + import_name = match.group(1) + if match.group(2): + wildcards.append(import_name) + else: + import_map[import_name.rsplit(".", 1)[-1]] = import_name + return import_map, wildcards + + def _analyze(self): + language_capsule = tree_sitter_java.language() + java_language = Language(language_capsule) + parser = Parser(java_language) + tree = parser.parse(bytes(self.content, "utf8")) + root = tree.root_node + lines = self.content.splitlines() + + top_level_nodes = {} + + self._extract_nodes(root, top_level_nodes, lines) + + self._extract_relationships(root, top_level_nodes) + + def _extract_nodes(self, node, top_level_nodes, lines): + node_type = None + node_name = None + qualified_name = None + class_name = None + + if node.type == "class_declaration": + is_abstract = any( + c.type == "modifier" and c.text.decode() == "abstract" for c in node.children + ) + node_type = "abstract class" if is_abstract else "class" + name_node = next((c for c in node.children if c.type == "identifier"), None) + node_name = name_node.text.decode() if name_node else None + qualified_name = self._qualified_type_name( + node_name, self._find_containing_type_names(node) + ) + elif node.type == "interface_declaration": + node_type = "interface" + name_node = next((c for c in node.children if c.type == "identifier"), None) + node_name = name_node.text.decode() if name_node else None + qualified_name = self._qualified_type_name( + node_name, self._find_containing_type_names(node) + ) + elif node.type == "enum_declaration": + node_type = "enum" + name_node = next((c for c in node.children if c.type == "identifier"), None) + node_name = name_node.text.decode() if name_node else None + qualified_name = self._qualified_type_name( + node_name, self._find_containing_type_names(node) + ) + elif node.type == "record_declaration": + node_type = "record" + name_node = next((c for c in node.children if c.type == "identifier"), None) + node_name = name_node.text.decode() if name_node else None + qualified_name = self._qualified_type_name( + node_name, self._find_containing_type_names(node) + ) + elif node.type == "annotation_type_declaration": + node_type = "annotation" + name_node = next((c for c in node.children if c.type == "identifier"), None) + node_name = name_node.text.decode() if name_node else None + qualified_name = self._qualified_type_name( + node_name, self._find_containing_type_names(node) + ) + elif node.type == "method_declaration": + node_type = "method" + name_node = next((c for c in node.children if c.type == "identifier"), None) + if name_node: + method_name = name_node.text.decode() + containing_types = self._find_containing_type_names(node) + if containing_types: + class_name = containing_types[-1] + node_name = f"{class_name}.{method_name}" + qualified_name = self._qualified_member_name(containing_types, method_name) + else: + node_name = method_name + qualified_name = self._qualify_name(method_name) + + if node_type and node_name: + component_id = self._get_component_id(node_name) + relative_path = self._get_relative_path() + node_obj = Node( + id=component_id, + name=node_name, + component_type=node_type, + file_path=str(self.file_path), + relative_path=relative_path, + source_code="\n".join(lines[node.start_point[0] : node.end_point[0] + 1]), + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + has_docstring=False, + docstring="", + parameters=None, + node_type=node_type, + base_classes=None, + class_name=class_name, + display_name=f"{node_type} {node_name}", + component_id=component_id, + language="java", + qualified_name=qualified_name, + ) + self.nodes.append(node_obj) + top_level_nodes[node_name] = node_obj + top_level_nodes[component_id] = node_obj + if qualified_name: + top_level_nodes[qualified_name] = node_obj + top_level_nodes.setdefault(qualified_name.split(".")[-1], node_obj) + + # Recursively process children + for child in node.children: + self._extract_nodes(child, top_level_nodes, lines) + + def _extract_relationships(self, node, top_level_nodes): + # 1. Inheritance: Class extends another class + if node.type == "class_declaration": + class_name = self._get_identifier_name(node) + [c.type for c in node.children] + + extends_node = next((c for c in node.children if c.type == "superclass"), None) + if extends_node: + base_class_name = self._get_type_name(extends_node) + if class_name and base_class_name and not self._skip_type(base_class_name, node): + caller_id = self._get_component_id(class_name) + callee_id = self._resolve_java_type(base_class_name, node, top_level_nodes) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_id, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + else: + logger.debug(f" No superclass found for {class_name}") + + # 2. Interface Implementation: Class/enum/record implements interface + if node.type in ["class_declaration", "enum_declaration", "record_declaration"]: + implementer_name = self._get_identifier_name(node) + implements_node = next((c for c in node.children if c.type == "super_interfaces"), None) + if implements_node and implementer_name: + for child in implements_node.children: + if child.type == "type_list": + for type_child in child.children: + if type_child.type in ["type_identifier", "generic_type"]: + interface_name = self._get_type_name(type_child) + if interface_name and not self._skip_type(interface_name, node): + caller_id = self._get_component_id(implementer_name) + callee_id = self._resolve_java_type( + interface_name, node, top_level_nodes + ) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_id, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + # 3. Field Type Use: Class has field of another class/interface type + if node.type == "field_declaration": + containing_class = self._find_containing_class(node, top_level_nodes) + type_node = next( + (c for c in node.children if c.type in ["type_identifier", "generic_type"]), None + ) + if containing_class and type_node: + field_type_name = self._get_type_name(type_node) + if field_type_name and not self._skip_type(field_type_name, node): + self.call_relationships.append( + CallRelationship( + caller=containing_class, + callee=self._resolve_java_type(field_type_name, node, top_level_nodes), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + # 4. Method Calls: Method calls on objects + if node.type == "method_invocation": + containing_class = self._find_containing_class(node, top_level_nodes) + containing_method = self._find_containing_method(node) + if containing_class: + object_name = None + method_name = None + + identifiers = [ + child.text.decode() for child in node.children if child.type == "identifier" + ] + if len(identifiers) >= 2: + object_name = identifiers[0] + method_name = identifiers[1] + elif identifiers: + method_name = identifiers[0] + + if method_name: + target_type = None + + caller_id = containing_method or containing_class + + if object_name and object_name[:1].isupper() and object_name in top_level_nodes: + target_type = object_name + elif object_name: + target_type = self._find_variable_type(node, object_name, top_level_nodes) + if not target_type and object_name in top_level_nodes: + target_type = object_name + if ( + not target_type + and object_name[:1].isupper() + and not object_name.isupper() + ): + # CamelCase receiver with no matching variable reads + # as a static call on a type from another file or an + # import; ALL_CAPS receivers are constants, not types. + target_type = object_name + + if target_type and not self._skip_type(target_type, node): + callee = self._resolve_java_member( + method_name, node, top_level_nodes, target_type + ) + if callee not in top_level_nodes and method_name in JAVA_OBJECT_METHODS: + # Inherited java.lang.Object method that the project + # type does not override locally — never a project edge. + callee = None + if callee: + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + elif not object_name: + callee = self._resolve_java_member(method_name, node, top_level_nodes) + if callee in top_level_nodes or self.import_map.get(method_name) == callee: + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + # 5. Object Creation + if node.type == "object_creation_expression": + containing_class = self._find_containing_class(node, top_level_nodes) + type_node = next( + (c for c in node.children if c.type in ["type_identifier", "generic_type"]), None + ) + if containing_class and type_node: + created_type = self._get_type_name(type_node) + if created_type and not self._skip_type(created_type, node): + self.call_relationships.append( + CallRelationship( + caller=containing_class, + callee=self._resolve_java_type(created_type, node, top_level_nodes), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + + # Recursively process children + for child in node.children: + self._extract_relationships(child, top_level_nodes) + + def _is_primitive_type(self, type_name: str) -> bool: + """Check if type is a Java primitive or a JDK/runtime type.""" + primitives = { + "boolean", + "byte", + "char", + "double", + "float", + "int", + "long", + "short", + "void", + "var", + } + simple = self._simple_type_name(type_name) + if simple in primitives: + return True + # Resolve through the import map first so a runtime type written with its + # simple name (imported from a `javax.*`/`java.*` package) is judged by its + # fully-qualified origin. The prefix rules in is_external_symbol then + # filter JDK/runtime packages, while project types — including sibling + # packages like `com.other.Bar` — fall through and resolve cross-file. This + # generalizes JDK filtering to any repository without enumerating types. + # java.lang types (no import to consult) are covered by the curated set + # inside is_external_symbol. + qualified = self.import_map.get(simple) + if qualified is None: + # A wildcard import of a JDK package (`import java.util.*;`) is the + # only way a JDK type outside java.lang appears with no explicit + # import; project wildcard packages fall through to resolution. + for wildcard in self.wildcard_imports: + if is_external_symbol("java", f"{wildcard}.{simple}"): + return True + qualified = simple + return is_external_symbol("java", qualified) + + def _resolve_java_type(self, type_name: str, context_node=None, top_level_nodes=None) -> str: + if not type_name: + return type_name + type_name = self._simple_type_name(type_name) + if "." in type_name: + return type_name + if type_name in self.import_map: + return self.import_map[type_name] + if context_node is not None and top_level_nodes is not None: + containing_types = self._find_containing_type_names(context_node) + for idx in range(len(containing_types), 0, -1): + candidate = self._qualify_name(".".join([*containing_types[:idx], type_name])) + if candidate in top_level_nodes: + return candidate + if self.package_name: + return f"{self.package_name}.{type_name}" + return type_name + + def _resolve_java_member( + self, member_name: str, context_node, top_level_nodes, target_type: str = None + ) -> str: + if target_type: + qualified_type = self._resolve_java_type(target_type, context_node, top_level_nodes) + candidate = f"{qualified_type}.{member_name}" + if candidate in top_level_nodes: + return candidate + simple_type = qualified_type.split(".")[-1] + simple_candidate = f"{simple_type}.{member_name}" + if simple_candidate in top_level_nodes: + return simple_candidate + return candidate + + containing_types = self._find_containing_type_names(context_node) + for idx in range(len(containing_types), 0, -1): + candidate = self._qualified_member_name(containing_types[:idx], member_name) + if candidate in top_level_nodes: + return candidate + # A static import maps the bare call to its declaring type, whether + # project (`com.foo.Util.checkNotNull`) or JDK (`java.util.Objects.requireNonNull`). + if member_name in self.import_map: + return self.import_map[member_name] + return self._qualify_name(member_name) + + def _skip_type(self, type_name: str, context_node) -> bool: + """Types that can never be project components: primitives, JDK/runtime + types, and generic type parameters in scope (e.g. the `K`/`V` of an + enclosing `class Cache`).""" + if self._is_primitive_type(type_name): + return True + return self._simple_type_name(type_name) in self._find_type_parameters(context_node) + + def _find_type_parameters(self, node) -> set: + params = set() + current = node + while current: + if current.type in [ + "class_declaration", + "interface_declaration", + "record_declaration", + "method_declaration", + ]: + type_parameters = next( + (c for c in current.children if c.type == "type_parameters"), None + ) + if type_parameters: + for param in type_parameters.children: + if param.type == "type_parameter": + for child in param.children: + if child.type in ["type_identifier", "identifier"]: + params.add(child.text.decode()) + break + current = current.parent + return params + + def _simple_type_name(self, type_name: str) -> str: + return type_name.strip().split("<", 1)[0].strip() + + def _qualify_name(self, name: str) -> str: + return f"{self.package_name}.{name}" if self.package_name else name + + def _qualified_type_name(self, name: str, containing_types: list[str]) -> str: + parts = [*containing_types, name] if name else containing_types + return self._qualify_name(".".join(parts)) if parts else "" + + def _qualified_member_name(self, containing_types: list[str], member_name: str) -> str: + return self._qualify_name(".".join([*containing_types, member_name])) + + def _get_identifier_name(self, node): + """Get identifier name from a node.""" + name_node = next((c for c in node.children if c.type == "identifier"), None) + return name_node.text.decode() if name_node else None + + def _get_type_name(self, node): + """Get type name from a type node.""" + if node.type == "type_identifier": + return node.text.decode() + elif node.type == "generic_type": + type_node = next((c for c in node.children if c.type == "type_identifier"), None) + return type_node.text.decode() if type_node else None + elif node.type == "superclass": + type_node = next((c for c in node.children if c.type == "type_identifier"), None) + return type_node.text.decode() if type_node else None + return None + + def _find_containing_class(self, node, top_level_nodes): + current = node.parent + while current: + if current.type in [ + "class_declaration", + "interface_declaration", + "enum_declaration", + "record_declaration", + "annotation_type_declaration", + ]: + class_name = self._get_identifier_name(current) + if class_name and class_name in top_level_nodes: + return self._get_component_id(class_name) + current = current.parent + return None + + def _find_variable_type(self, node, variable_name, top_level_nodes): + method_node = node.parent + while method_node and method_node.type not in [ + "method_declaration", + "constructor_declaration", + ]: + method_node = method_node.parent + + if method_node: + for child in method_node.children: + if child.type == "block" or child.type == "constructor_body": + variable_type = self._search_variable_declaration(child, variable_name) + if variable_type: + return variable_type + elif child.type == "formal_parameters": + for param in child.children: + if param.type in ["formal_parameter", "spread_parameter"]: + type_node = next( + ( + c + for c in param.children + if c.type in ["type_identifier", "generic_type"] + ), + None, + ) + identifier_node = next( + (c for c in param.children if c.type == "identifier"), None + ) + if ( + type_node + and identifier_node + and identifier_node.text.decode() == variable_name + ): + return self._get_type_name(type_node) + + class_node = node.parent + while class_node and class_node.type != "class_declaration": + class_node = class_node.parent + + if class_node: + for child in class_node.children: + if child.type == "class_body": + for body_child in child.children: + if body_child.type == "field_declaration": + identifier_node = None + type_node = None + for field_child in body_child.children: + if field_child.type in ["type_identifier", "generic_type"]: + type_node = field_child + elif field_child.type == "variable_declarator": + identifier_node = next( + (c for c in field_child.children if c.type == "identifier"), + None, + ) + + if ( + identifier_node + and type_node + and identifier_node.text.decode() == variable_name + ): + field_type = self._get_type_name(type_node) + return field_type + + return None + + def _search_variable_declaration(self, block_node, variable_name): + for child in block_node.children: + if child.type == "local_variable_declaration": + type_node = None + identifier_node = None + for decl_child in child.children: + if decl_child.type in ["type_identifier", "generic_type"]: + type_node = decl_child + elif decl_child.type == "variable_declarator": + identifier_node = next( + (c for c in decl_child.children if c.type == "identifier"), None + ) + + if identifier_node and type_node and identifier_node.text.decode() == variable_name: + return self._get_type_name(type_node) + + elif child.type == "block": + result = self._search_variable_declaration(child, variable_name) + if result: + return result + + return None + + def _find_containing_class_name(self, node): + names = self._find_containing_type_names(node) + return names[-1] if names else None + + def _find_containing_type_names(self, node) -> list[str]: + names = [] + current = node.parent + while current: + if current.type in [ + "class_declaration", + "interface_declaration", + "enum_declaration", + "record_declaration", + "annotation_type_declaration", + ]: + name_node = next((c for c in current.children if c.type == "identifier"), None) + if name_node: + names.append(name_node.text.decode()) + current = current.parent + return list(reversed(names)) + + def _find_containing_method(self, node): + current = node.parent + while current: + if current.type == "method_declaration": + method_name = self._get_identifier_name(current) + class_name = self._find_containing_class_name(current) + if method_name and class_name: + return self._get_component_id(f"{class_name}.{method_name}") + current = current.parent + return None + + +def analyze_java_file( + file_path: str, content: str, repo_path: str = None +) -> Tuple[List[Node], List[CallRelationship]]: + analyzer = TreeSitterJavaAnalyzer(file_path, content, repo_path) + return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/javascript.py b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py index 95cd6d5..39bdfac 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/javascript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py @@ -1,14 +1,11 @@ import logging import os import traceback -from typing import List, Set, Optional, Tuple +from typing import List, Optional, Tuple from pathlib import Path -import sys -import os from tree_sitter import Parser, Language import tree_sitter_javascript -import tree_sitter_typescript from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( @@ -25,9 +22,9 @@ def __init__(self, file_path: str, content: str, repo_path: str = None): self.repo_path = repo_path or "" self.nodes: List[Node] = [] self.call_relationships: List[CallRelationship] = [] - + self.top_level_nodes = {} - + self.seen_relationships = set() try: @@ -41,10 +38,9 @@ def __init__(self, file_path: str, content: str, repo_path: str = None): self.parser = None self.js_language = None - def _add_relationship(self, relationship: CallRelationship) -> bool: rel_key = (relationship.caller, relationship.callee, relationship.call_line) - + if rel_key not in self.seen_relationships: self.seen_relationships.add(rel_key) self.call_relationships.append(relationship) @@ -80,13 +76,13 @@ def _get_module_path(self) -> str: rel_path = str(self.file_path) else: rel_path = str(self.file_path) - - for ext in ['.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs']: + + for ext in [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs"]: if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] + rel_path = rel_path[: -len(ext)] break - return rel_path.replace('/', '.').replace('\\', '.') - + return rel_path.replace("/", ".").replace("\\", ".") + def _get_relative_path(self) -> str: if self.repo_path: try: @@ -112,7 +108,11 @@ def _get_component_id(self, name: str, class_name: str = None, is_method: bool = def _find_containing_class(self, node) -> Optional[str]: parent = node.parent while parent: - if parent.type in ["class_declaration", "abstract_class_declaration", "interface_declaration"]: + if parent.type in [ + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + ]: name_node = self._find_child_by_type(parent, "type_identifier") if not name_node: name_node = self._find_child_by_type(parent, "identifier") @@ -126,14 +126,18 @@ def _extract_functions(self, node) -> None: self.nodes.sort(key=lambda n: n.start_line) def _traverse_for_functions(self, node) -> None: - if node.type in ["class_declaration", "abstract_class_declaration", "interface_declaration"]: + if node.type in [ + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + ]: cls = self._extract_class_declaration(node) if cls: self.nodes.append(cls) self.top_level_nodes[cls.name] = cls - + self._extract_methods_from_class(node, cls.name) - + elif node.type == "function_declaration": containing_class = self._find_containing_class(node) if containing_class is None: @@ -171,7 +175,7 @@ def _traverse_for_functions(self, node) -> None: elif node.type == "expression_statement": # Handle CommonJS exports: module.exports = ..., exports.name = ... self._extract_commonjs_exports(node) - + for child in node.children: self._traverse_for_functions(child) @@ -179,7 +183,7 @@ def _extract_methods_from_class(self, class_node, class_name: str) -> None: class_body = self._find_child_by_type(class_node, "class_body") if not class_body: return - + for child in class_body.children: if child.type == "method_definition": method_name = self._get_method_name(child) @@ -202,7 +206,7 @@ def _get_method_name(self, method_node) -> Optional[str]: """Get method name from method_definition node.""" if method_node.type != "method_definition": return None - + for child in method_node.children: if child.type == "property_identifier": return self._get_node_text(child) @@ -212,7 +216,7 @@ def _get_field_name(self, field_node) -> Optional[str]: """Get field name from field_definition node.""" if field_node.type != "field_definition": return None - + for child in field_node.children: if child.type == "property_identifier": return self._get_node_text(child) @@ -277,7 +281,7 @@ def _extract_class_declaration(self, node) -> Optional[Node]: if child.type in ["identifier", "type_identifier"]: base_classes.append(self._get_node_text(child)) code_snippet = "\n".join(self.content.splitlines()[line_start - 1 : line_end]) - + if node.type == "abstract_class_declaration": node_type = "abstract class" display_name = f"abstract class {name}" @@ -287,10 +291,10 @@ def _extract_class_declaration(self, node) -> Optional[Node]: else: node_type = "class" display_name = f"class {name}" - + component_id = self._get_component_id(name, is_method=False) relative_path = self._get_relative_path() - + return Node( id=component_id, name=name, @@ -327,7 +331,7 @@ def _extract_function_declaration(self, node) -> Optional[Node]: # Check for async and generator from code snippet is_async = "async function" in code_snippet is_generator = "function*" in code_snippet or "*" in func_name - + if is_async and is_generator: display_name = f"async generator {func_name}" elif is_async: @@ -361,6 +365,7 @@ def _extract_function_declaration(self, node) -> Optional[Node]: except Exception as e: logger.debug(f"Error extracting function declaration: {e}") return None + def _extract_exported_function(self, node) -> Optional[Node]: """Extract export function or export default function""" try: @@ -471,7 +476,11 @@ def _extract_exports_value(self, value_node) -> None: if child.type == "pair": key_node = child.child_by_field_name("key") val_node = child.child_by_field_name("value") - if key_node and val_node and val_node.type in ("arrow_function", "function_expression", "function"): + if ( + key_node + and val_node + and val_node.type in ("arrow_function", "function_expression", "function") + ): name = self._get_node_text(key_node) self._create_export_component(val_node, name) elif child.type == "shorthand_property_identifier": @@ -552,12 +561,18 @@ def _extract_call_relationships(self, node) -> None: def _traverse_for_calls(self, node, current_top_level) -> None: if current_top_level: self._extract_jsdoc_type_dependencies(node, current_top_level) - - if node.type in ["class_declaration", "abstract_class_declaration", "interface_declaration"]: - name_node = self._find_child_by_type(node, "type_identifier") or self._find_child_by_type(node, "identifier") + + if node.type in [ + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + ]: + name_node = self._find_child_by_type( + node, "type_identifier" + ) or self._find_child_by_type(node, "identifier") if name_node: current_top_level = self._get_node_text(name_node) - + heritage_node = self._find_child_by_type(node, "class_heritage") if heritage_node: for child in heritage_node.children: @@ -565,17 +580,23 @@ def _traverse_for_calls(self, node, current_top_level) -> None: base_class = self._get_node_text(child) caller_id = self._get_component_id(current_top_level) resolved = base_class in self.top_level_nodes - callee_id = self._get_component_id(base_class) if resolved else base_class + callee_id = ( + self._get_component_id(base_class) if resolved else base_class + ) inheritance_rel = CallRelationship( caller=caller_id, callee=callee_id, call_line=node.start_point[0] + 1, - is_resolved=resolved + is_resolved=resolved, ) self._add_relationship(inheritance_rel) elif node.type in ("method_definition", "field_definition"): - name = self._get_method_name(node) if node.type == "method_definition" else self._get_field_name(node) + name = ( + self._get_method_name(node) + if node.type == "method_definition" + else self._get_field_name(node) + ) containing_class = self._find_containing_class(node) if name and containing_class and f"{containing_class}.{name}" in self.top_level_nodes: current_top_level = f"{containing_class}.{name}" @@ -592,14 +613,18 @@ def _traverse_for_calls(self, node, current_top_level) -> None: for child in node.children: if child.type == "variable_declarator": name_node = self._find_child_by_type(child, "identifier") - func_node = self._find_child_by_type(child, "arrow_function") or self._find_child_by_type(child, "function_expression") + func_node = self._find_child_by_type( + child, "arrow_function" + ) or self._find_child_by_type(child, "function_expression") if name_node and func_node: current_top_level = self._get_node_text(name_node) elif node.type == "variable_declaration": for child in node.children: if child.type == "variable_declarator": name_node = self._find_child_by_type(child, "identifier") - func_node = self._find_child_by_type(child, "arrow_function") or self._find_child_by_type(child, "function_expression") + func_node = self._find_child_by_type( + child, "arrow_function" + ) or self._find_child_by_type(child, "function_expression") if name_node and func_node: current_top_level = self._get_node_text(name_node) @@ -607,14 +632,14 @@ def _traverse_for_calls(self, node, current_top_level) -> None: call_info = self._extract_call_from_node(node, current_top_level) if call_info: self._add_relationship(call_info) - + elif node.type == "await_expression" and current_top_level: call_expr = self._find_child_by_type(node, "call_expression") if call_expr: call_info = self._extract_call_from_node(call_expr, current_top_level) if call_info: self._add_relationship(call_info) - + elif node.type == "new_expression" and current_top_level: constructor_node = self._find_child_by_type(node, "identifier") if constructor_node is not None: @@ -624,7 +649,7 @@ def _traverse_for_calls(self, node, current_top_level) -> None: caller=f"{self._get_relative_path()}::{current_top_level}", callee=self._get_component_id(callee_name) if resolved else callee_name, call_line=node.start_point[0] + 1, - is_resolved=resolved + is_resolved=resolved, ) self._add_relationship(call_info) @@ -684,7 +709,11 @@ def make(callee_name: str, resolved: bool) -> CallRelationship: if obj.type == "identifier": receiver = self._get_node_text(obj) if receiver in self.top_level_nodes or self._receiver_class(node, receiver): - cls = receiver if receiver in self.top_level_nodes else self._receiver_class(node, receiver) + cls = ( + receiver + if receiver in self.top_level_nodes + else self._receiver_class(node, receiver) + ) return make(f"{cls}.{tail}", f"{cls}.{tail}" in self.top_level_nodes) return make(f"{receiver}.{tail}", False) @@ -727,9 +756,12 @@ def _receiver_class(self, call_node, identifier: str) -> Optional[str]: scope = call_node.parent while scope: if scope.type in ( - "method_definition", "function_declaration", - "generator_function_declaration", "arrow_function", - "function_expression", "program", + "method_definition", + "function_declaration", + "generator_function_declaration", + "arrow_function", + "function_expression", + "program", ): found = self._find_new_initializer(scope, identifier) if found: @@ -755,39 +787,40 @@ def _find_new_initializer(self, scope_node, identifier: str) -> Optional[str]: def _extract_jsdoc_type_dependencies(self, node, caller_name: str) -> None: """Extract type dependencies from JSDoc comments.""" try: - if hasattr(node, 'prev_sibling') and node.prev_sibling: + if hasattr(node, "prev_sibling") and node.prev_sibling: prev = node.prev_sibling if prev.type == "comment": comment_text = self._get_node_text(prev) self._parse_jsdoc_types(comment_text, caller_name, node.start_point[0] + 1) - + for child in node.children: if child.type == "comment": comment_text = self._get_node_text(child) self._parse_jsdoc_types(comment_text, caller_name, node.start_point[0] + 1) - + except Exception as e: logger.debug(f"Error extracting JSDoc dependencies: {e}") def _parse_jsdoc_types(self, comment_text: str, caller_name: str, line_number: int) -> None: """Parse JSDoc comment text and extract type references.""" import re + try: type_patterns = [ - r'@param\s*\{([^}]+)\}', # @param {Type} - r'@returns?\s*\{([^}]+)\}', # @return {Type} or @returns {Type} - r'@type\s*\{([^}]+)\}', # @type {Type} - r'@typedef\s*\{[^}]*\}\s*(\w+)', # @typedef {Object} TypeName - r'@interface\s+(\w+)', # @interface InterfaceName + r"@param\s*\{([^}]+)\}", # @param {Type} + r"@returns?\s*\{([^}]+)\}", # @return {Type} or @returns {Type} + r"@type\s*\{([^}]+)\}", # @type {Type} + r"@typedef\s*\{[^}]*\}\s*(\w+)", # @typedef {Object} TypeName + r"@interface\s+(\w+)", # @interface InterfaceName ] - + for pattern in type_patterns: matches = re.findall(pattern, comment_text) for match in matches: type_name = match.strip() - + base_types = self._extract_base_types_from_jsdoc(type_name) - + for base_type in base_types: if base_type and not self._is_builtin_type_js(base_type): caller_id = f"{self._get_relative_path()}::{caller_name}" @@ -798,61 +831,99 @@ def _parse_jsdoc_types(self, comment_text: str, caller_name: str, line_number: i caller=caller_id, callee=callee_id, call_line=line_number, - is_resolved=resolved + is_resolved=resolved, ) - + if self._add_relationship(type_rel): pass - + except Exception as e: logger.debug(f"Error parsing JSDoc types: {e}") def _extract_base_types_from_jsdoc(self, type_str: str) -> list: import re + type_str = type_str.strip() - + base_types = [] - - main_type_match = re.match(r'^(\w+)', type_str) + + main_type_match = re.match(r"^(\w+)", type_str) if main_type_match: base_types.append(main_type_match.group(1)) - - generic_matches = re.findall(r'<([^<>]+)>', type_str) + + generic_matches = re.findall(r"<([^<>]+)>", type_str) for generic in generic_matches: - subtypes = re.findall(r'\b(\w+)\b', generic) + subtypes = re.findall(r"\b(\w+)\b", generic) base_types.extend(subtypes) - - if '|' in type_str: - union_types = type_str.split('|') + + if "|" in type_str: + union_types = type_str.split("|") for union_type in union_types: - clean_type = re.match(r'\b(\w+)\b', union_type.strip()) + clean_type = re.match(r"\b(\w+)\b", union_type.strip()) if clean_type: base_types.append(clean_type.group(1)) - + return base_types def _is_builtin_type_js(self, name: str) -> bool: """Check if type name is a JavaScript/JSDoc built-in type.""" builtin_types = { # JavaScript primitive types - "string", "number", "boolean", "object", "undefined", "null", "void", "any", - + "string", + "number", + "boolean", + "object", + "undefined", + "null", + "void", + "any", # Global JavaScript types - "Array", "Promise", "Date", "RegExp", "Error", "Map", "Set", "WeakMap", "WeakSet", - "Function", "Object", "String", "Number", "Boolean", "Symbol", "BigInt", - - "Element", "HTMLElement", "Document", "Window", "Event", "EventTarget", "Node", - "Response", "Request", "Headers", "URL", "URLSearchParams", "FormData", "Blob", "File", - + "Array", + "Promise", + "Date", + "RegExp", + "Error", + "Map", + "Set", + "WeakMap", + "WeakSet", + "Function", + "Object", + "String", + "Number", + "Boolean", + "Symbol", + "BigInt", + "Element", + "HTMLElement", + "Document", + "Window", + "Event", + "EventTarget", + "Node", + "Response", + "Request", + "Headers", + "URL", + "URLSearchParams", + "FormData", + "Blob", + "File", # Common JSDoc generic parameters - "T", "U", "V", "K", "P", "R", "E" + "T", + "U", + "V", + "K", + "P", + "R", + "E", } return name in builtin_types def _extract_callee_name(self, call_node) -> Optional[str]: if not call_node.children: return None - + callee_node = call_node.children[0] if callee_node.type == "identifier": @@ -861,7 +932,7 @@ def _extract_callee_name(self, call_node) -> Optional[str]: property_node = self._find_child_by_type(callee_node, "property_identifier") if property_node: return self._get_node_text(property_node) - + computed_property = self._find_child_by_type(callee_node, "computed_property_name") if computed_property: for child in computed_property.children: @@ -871,7 +942,7 @@ def _extract_callee_name(self, call_node) -> Optional[str]: return "super" elif callee_node.type == "this": return "this" - + return None def _find_child_by_type(self, node, node_type: str): @@ -905,6 +976,7 @@ def _extract_assignment_name(self, node) -> Optional[str]: return self._get_node_text(property_node) return None + def analyze_javascript_file_treesitter( file_path: str, content: str, repo_path: str = None ) -> Tuple[List[Node], List[CallRelationship]]: @@ -920,7 +992,3 @@ def analyze_javascript_file_treesitter( except Exception as e: logger.error(f"Error in tree-sitter JS analysis for {file_path}: {e}", exc_info=True) return [], [] - - - - diff --git a/codewiki/src/be/dependency_analyzer/analyzers/kotlin.py b/codewiki/src/be/dependency_analyzer/analyzers/kotlin.py index 7029541..41cca12 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/kotlin.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/kotlin.py @@ -1,7 +1,6 @@ import logging from typing import List, Optional, Tuple from pathlib import Path -import sys import os from tree_sitter import Parser, Language @@ -10,6 +9,7 @@ logger = logging.getLogger(__name__) + class TreeSitterKotlinAnalyzer: def __init__(self, file_path: str, content: str, repo_path: Optional[str] = None): self.file_path = Path(file_path) @@ -18,7 +18,7 @@ def __init__(self, file_path: str, content: str, repo_path: Optional[str] = None self.nodes: List[Node] = [] self.call_relationships: List[CallRelationship] = [] self._analyze() - + def _get_module_path(self) -> str: if self.repo_path: try: @@ -27,13 +27,13 @@ def _get_module_path(self) -> str: rel_path = str(self.file_path) else: rel_path = str(self.file_path) - - for ext in ['.kt', '.kts']: + + for ext in [".kt", ".kts"]: if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] + rel_path = rel_path[: -len(ext)] break - return rel_path.replace('/', '.').replace('\\', '.') - + return rel_path.replace("/", ".").replace("\\", ".") + def _get_relative_path(self) -> str: """Get relative path from repo root.""" if self.repo_path: @@ -46,7 +46,7 @@ def _get_relative_path(self) -> str: return str(self.file_path) else: return str(self.file_path) - + def _get_component_id(self, name: str, parent_class: Optional[str] = None) -> str: rel_path = self._get_relative_path() if parent_class: @@ -62,21 +62,21 @@ def _analyze(self): tree = parser.parse(bytes(self.content, "utf8")) root = tree.root_node lines = self.content.splitlines() - + top_level_nodes = {} - + self._extract_nodes(root, top_level_nodes, lines) self._extract_relationships(root, top_level_nodes) except Exception as e: logger.error(f"Error parsing Kotlin file {self.file_path}: {e}") - + def _extract_nodes(self, node, top_level_nodes, lines): node_type = None node_name = None - + if node.type == "class_declaration": is_interface = any(c.type == "interface" for c in node.children) - + if is_interface: node_type = "interface" else: @@ -91,15 +91,15 @@ def _extract_nodes(self, node, top_level_nodes, lines): node_type = "annotation class" else: node_type = "class" - + name_node = next((c for c in node.children if c.type == "identifier"), None) node_name = name_node.text.decode() if name_node else None - + elif node.type == "object_declaration": node_type = "object" name_node = next((c for c in node.children if c.type == "identifier"), None) node_name = name_node.text.decode() if name_node else None - + elif node.type == "function_declaration": name_node = next((c for c in node.children if c.type == "identifier"), None) if name_node: @@ -111,22 +111,24 @@ def _extract_nodes(self, node, top_level_nodes, lines): else: node_type = "function" node_name = method_name - + if node_type and node_name: component_id = self._get_component_id(node_name) relative_path = self._get_relative_path() - + # Extract docstring if present docstring = "" if node.prev_sibling and hasattr(node.prev_sibling, "type"): if node.prev_sibling.type in ("line_comment", "block_comment"): docstring = node.prev_sibling.text.decode().strip() - + # Safely extract code lines start_line_idx = node.start_point[0] end_line_idx = node.end_point[0] + 1 - code_snippet = "\n".join(lines[start_line_idx:end_line_idx]) if start_line_idx < len(lines) else "" - + code_snippet = ( + "\n".join(lines[start_line_idx:end_line_idx]) if start_line_idx < len(lines) else "" + ) + node_obj = Node( id=component_id, name=node_name, @@ -134,8 +136,8 @@ def _extract_nodes(self, node, top_level_nodes, lines): file_path=str(self.file_path), relative_path=relative_path, source_code=code_snippet, - start_line=node.start_point[0]+1, - end_line=node.end_point[0]+1, + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, has_docstring=bool(docstring), docstring=docstring, parameters=None, @@ -143,14 +145,14 @@ def _extract_nodes(self, node, top_level_nodes, lines): base_classes=None, class_name=None, display_name=f"{node_type} {node_name}", - component_id=component_id + component_id=component_id, ) self.nodes.append(node_obj) top_level_nodes[node_name] = node_obj - + for child in node.children: self._extract_nodes(child, top_level_nodes, lines) - + def _get_class_modifiers(self, class_node) -> set: """Extract class modifiers (abstract, data, enum, annotation, etc.).""" modifiers = set() @@ -161,7 +163,7 @@ def _get_class_modifiers(self, class_node) -> set: for inner in mod.children: modifiers.add(inner.type) return modifiers - + def _extract_relationships(self, node, top_level_nodes): # 1. Inheritance and Interface Implementation via delegation_specifiers if node.type == "class_declaration": @@ -182,35 +184,37 @@ def _extract_relationships(self, node, top_level_nodes): type_name = self._get_type_name(user_type) elif child.type == "user_type": type_name = self._get_type_name(child) - + if type_name and not self._is_primitive_type(type_name): caller_id = self._get_component_id(class_name) callee_id = self._get_component_id(type_name) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_id, - call_line=node.start_point[0]+1, - is_resolved=False - )) - + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_id, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + # 2. Property Type Use (field types) if node.type == "property_declaration": containing_class = self._find_containing_class(node, top_level_nodes) var_decl = next((c for c in node.children if c.type == "variable_declaration"), None) if containing_class and var_decl: - type_node = next( - (c for c in var_decl.children if c.type == "user_type"), None - ) + type_node = next((c for c in var_decl.children if c.type == "user_type"), None) if type_node: prop_type_name = self._get_type_name(type_node) if prop_type_name and not self._is_primitive_type(prop_type_name): - self.call_relationships.append(CallRelationship( - caller=containing_class, - callee=prop_type_name, - call_line=node.start_point[0]+1, - is_resolved=False - )) - + self.call_relationships.append( + CallRelationship( + caller=containing_class, + callee=prop_type_name, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + # 3. Constructor parameter type use if node.type == "class_parameter": containing_class_node = node.parent @@ -219,52 +223,61 @@ def _extract_relationships(self, node, top_level_nodes): if containing_class_node: class_name = self._get_identifier_name(containing_class_node) if class_name and class_name in top_level_nodes: - type_node = next( - (c for c in node.children if c.type == "user_type"), None - ) + type_node = next((c for c in node.children if c.type == "user_type"), None) if type_node: param_type = self._get_type_name(type_node) if param_type and not self._is_primitive_type(param_type): caller_id = self._get_component_id(class_name) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=param_type, - call_line=node.start_point[0]+1, - is_resolved=False - )) - + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=param_type, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + # 4. Method Calls / Function invocations if node.type == "call_expression": - caller_id = self._find_containing_method(node) or self._find_containing_class(node, top_level_nodes) - + caller_id = self._find_containing_method(node) or self._find_containing_class( + node, top_level_nodes + ) + target_expr = next( - (c for c in node.children if c.type in ["identifier", "navigation_expression"]), None + (c for c in node.children if c.type in ["identifier", "navigation_expression"]), + None, ) - + if target_expr and caller_id: if target_expr.type == "identifier": callee_name = target_expr.text.decode() - if callee_name and callee_name[0].isupper() and not self._is_primitive_type(callee_name): + if ( + callee_name + and callee_name[0].isupper() + and not self._is_primitive_type(callee_name) + ): callee_id = self._get_component_id(callee_name) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_id, - call_line=node.start_point[0]+1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_id, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) elif callee_name and not self._is_primitive_type(callee_name): - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_name, - call_line=node.start_point[0]+1, - is_resolved=False - )) - + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_name, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + elif target_expr.type == "navigation_expression": children = list(target_expr.children) - object_node = next( - (c for c in children if c.type == "identifier"), None - ) + object_node = next((c for c in children if c.type == "identifier"), None) method_node = None identifiers = [c for c in children if c.type == "identifier"] if len(identifiers) >= 2: @@ -279,47 +292,83 @@ def _extract_relationships(self, node, top_level_nodes): object_node = self._get_root_identifier(nav_child) else: object_node = None - + if object_node and method_node: - object_name = object_node.text.decode() if hasattr(object_node, 'text') else str(object_node) - method_name = method_node.text.decode() - + object_name = ( + object_node.text.decode() + if hasattr(object_node, "text") + else str(object_node) + ) + method_node.text.decode() + target_type = None if object_name in top_level_nodes: target_type = object_name else: - target_type = self._find_variable_type(node, object_name, top_level_nodes) - + target_type = self._find_variable_type( + node, object_name, top_level_nodes + ) + if target_type and not self._is_primitive_type(target_type): callee_id = self._get_component_id(target_type) - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_id, - call_line=node.start_point[0]+1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_id, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) elif method_node and not object_node: callee_name = method_node.text.decode() - self.call_relationships.append(CallRelationship( - caller=caller_id, - callee=callee_name, - call_line=node.start_point[0]+1, - is_resolved=False - )) - + self.call_relationships.append( + CallRelationship( + caller=caller_id, + callee=callee_name, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) + for child in node.children: self._extract_relationships(child, top_level_nodes) def _is_primitive_type(self, type_name: str) -> bool: """Check if type is a Kotlin primitive or common built-in type.""" primitives = { - "Boolean", "Byte", "Char", "Double", "Float", "Int", "Long", "Short", - "String", "Unit", "Nothing", "Any", - "List", "Set", "Map", "Collection", "Iterable", "Sequence", - "MutableList", "MutableSet", "MutableMap", "MutableCollection", - "Array", "IntArray", "LongArray", "FloatArray", "DoubleArray", - "BooleanArray", "ByteArray", "CharArray", "ShortArray", - "Pair", "Triple", + "Boolean", + "Byte", + "Char", + "Double", + "Float", + "Int", + "Long", + "Short", + "String", + "Unit", + "Nothing", + "Any", + "List", + "Set", + "Map", + "Collection", + "Iterable", + "Sequence", + "MutableList", + "MutableSet", + "MutableMap", + "MutableCollection", + "Array", + "IntArray", + "LongArray", + "FloatArray", + "DoubleArray", + "BooleanArray", + "ByteArray", + "CharArray", + "ShortArray", + "Pair", + "Triple", } return type_name in primitives @@ -327,7 +376,7 @@ def _get_identifier_name(self, node): """Get identifier name from a node.""" name_node = next((c for c in node.children if c.type == "identifier"), None) return name_node.text.decode() if name_node else None - + def _get_type_name(self, node) -> Optional[str]: """Get the primary type name from a type node, stripping generics.""" if node.type == "user_type": @@ -340,7 +389,7 @@ def _get_type_name(self, node) -> Optional[str]: elif node.type == "identifier": return node.text.decode() return None - + def _get_root_identifier(self, nav_node): """Get the root identifier from a chain of navigation_expressions.""" first_child = nav_node.children[0] if nav_node.children else None @@ -361,12 +410,12 @@ def _find_containing_class_name(self, node): return name_node.text.decode() current = current.parent return None - + def _find_containing_class(self, node, top_level_nodes): """Find the component ID of the containing class.""" class_name = self._find_containing_class_name(node) if class_name and class_name in top_level_nodes: - return self._get_component_id(class_name) + return self._get_component_id(class_name) return None def _find_containing_method(self, node): @@ -391,7 +440,7 @@ def _find_variable_type(self, node, variable_name: str, top_level_nodes) -> Opti func_node = node.parent while func_node and func_node.type != "function_declaration": func_node = func_node.parent - + if func_node: params_node = next( (c for c in func_node.children if c.type == "function_value_parameters"), None @@ -404,25 +453,28 @@ def _find_variable_type(self, node, variable_name: str, top_level_nodes) -> Opti ) if param_name_node and param_name_node.text.decode() == variable_name: type_node = next( - (c for c in param.children if c.type in ("user_type", "nullable_type")), None + ( + c + for c in param.children + if c.type in ("user_type", "nullable_type") + ), + None, ) if type_node: return self._get_type_name(type_node) - - body_node = next( - (c for c in func_node.children if c.type == "function_body"), None - ) + + body_node = next((c for c in func_node.children if c.type == "function_body"), None) if body_node: block = next((c for c in body_node.children if c.type == "block"), None) if block: result = self._search_variable_declaration(block, variable_name) if result: return result - + class_node = node.parent while class_node and class_node.type not in ("class_declaration", "object_declaration"): class_node = class_node.parent - + if class_node: primary_ctor = next( (c for c in class_node.children if c.type == "primary_constructor"), None @@ -439,19 +491,26 @@ def _find_variable_type(self, node, variable_name: str, top_level_nodes) -> Opti ) if param_name and param_name.text.decode() == variable_name: type_node = next( - (c for c in param.children if c.type in ("user_type", "nullable_type")), None + ( + c + for c in param.children + if c.type in ("user_type", "nullable_type") + ), + None, ) if type_node: return self._get_type_name(type_node) - + class_body = next( - (c for c in class_node.children if c.type in ("class_body", "enum_class_body")), None + (c for c in class_node.children if c.type in ("class_body", "enum_class_body")), + None, ) if class_body: for body_child in class_body.children: if body_child.type == "property_declaration": var_decl = next( - (c for c in body_child.children if c.type == "variable_declaration"), None + (c for c in body_child.children if c.type == "variable_declaration"), + None, ) if var_decl: prop_name = next( @@ -459,13 +518,18 @@ def _find_variable_type(self, node, variable_name: str, top_level_nodes) -> Opti ) if prop_name and prop_name.text.decode() == variable_name: type_node = next( - (c for c in var_decl.children if c.type in ("user_type", "nullable_type")), None + ( + c + for c in var_decl.children + if c.type in ("user_type", "nullable_type") + ), + None, ) if type_node: return self._get_type_name(type_node) - + return None - + def _search_variable_declaration(self, block_node, variable_name: str) -> Optional[str]: """Search for a local variable declaration with explicit type annotation in a block.""" for child in block_node.children: @@ -474,16 +538,19 @@ def _search_variable_declaration(self, block_node, variable_name: str) -> Option (c for c in child.children if c.type == "variable_declaration"), None ) if var_decl: - name_node = next( - (c for c in var_decl.children if c.type == "identifier"), None - ) + name_node = next((c for c in var_decl.children if c.type == "identifier"), None) if name_node and name_node.text.decode() == variable_name: type_node = next( - (c for c in var_decl.children if c.type in ("user_type", "nullable_type")), None + ( + c + for c in var_decl.children + if c.type in ("user_type", "nullable_type") + ), + None, ) if type_node: return self._get_type_name(type_node) - + init_expr = next( (c for c in child.children if c.type == "call_expression"), None ) @@ -495,14 +562,17 @@ def _search_variable_declaration(self, block_node, variable_name: str) -> Option inferred = call_id.text.decode() if inferred and inferred[0].isupper(): return inferred - + elif child.type == "block": result = self._search_variable_declaration(child, variable_name) if result: return result - + return None -def analyze_kotlin_file(file_path: str, content: str, repo_path: Optional[str] = None) -> Tuple[List[Node], List[CallRelationship]]: + +def analyze_kotlin_file( + file_path: str, content: str, repo_path: Optional[str] = None +) -> Tuple[List[Node], List[CallRelationship]]: analyzer = TreeSitterKotlinAnalyzer(file_path, content, repo_path) return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/php.py b/codewiki/src/be/dependency_analyzer/analyzers/php.py index 870222e..81d6df0 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/php.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/php.py @@ -18,15 +18,48 @@ # PHP primitive and built-in types to exclude from dependencies PHP_PRIMITIVES: Set[str] = { - "string", "int", "float", "bool", "array", "object", "callable", - "iterable", "mixed", "void", "null", "false", "true", "never", - "self", "static", "parent", "integer", "boolean", "double", + "string", + "int", + "float", + "bool", + "array", + "object", + "callable", + "iterable", + "mixed", + "void", + "null", + "false", + "true", + "never", + "self", + "static", + "parent", + "integer", + "boolean", + "double", # Common PHP classes that are built-in - "Exception", "Error", "Throwable", "Closure", "Generator", - "Iterator", "IteratorAggregate", "Traversable", "ArrayAccess", - "Serializable", "Countable", "JsonSerializable", "Stringable", - "DateTime", "DateTimeInterface", "DateTimeImmutable", "DateInterval", - "stdClass", "ArrayObject", "SplObjectStorage", "WeakReference", + "Exception", + "Error", + "Throwable", + "Closure", + "Generator", + "Iterator", + "IteratorAggregate", + "Traversable", + "ArrayAccess", + "Serializable", + "Countable", + "JsonSerializable", + "Stringable", + "DateTime", + "DateTimeInterface", + "DateTimeImmutable", + "DateInterval", + "stdClass", + "ArrayObject", + "SplObjectStorage", + "WeakReference", } # Template file patterns to skip @@ -130,12 +163,12 @@ def _get_module_path(self) -> str: rel_path = str(self.file_path) # Remove .php extension - for ext in ['.php', '.phtml', '.inc']: + for ext in [".php", ".phtml", ".inc"]: if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] + rel_path = rel_path[: -len(ext)] break - return rel_path.replace('/', '.').replace('\\', '.') + return rel_path.replace("/", ".").replace("\\", ".") def _get_relative_path(self) -> str: """Get relative path from repo root.""" @@ -213,7 +246,11 @@ def _extract_use_statement(self, node): alias_node = self._find_child_by_type(child, "namespace_aliasing_clause") if name_node: - fqn = f"{prefix}\\{name_node.text.decode()}" if prefix else name_node.text.decode() + fqn = ( + f"{prefix}\\{name_node.text.decode()}" + if prefix + else name_node.text.decode() + ) alias = None if alias_node: alias_name = self._find_child_by_type(alias_node, "name") @@ -224,8 +261,9 @@ def _extract_use_statement(self, node): # Handle simple use: use App\User; or use App\User as U; for child in node.children: if child.type == "namespace_use_clause": - name_node = self._find_child_by_type(child, "qualified_name") or \ - self._find_child_by_type(child, "namespace_name") + name_node = self._find_child_by_type( + child, "qualified_name" + ) or self._find_child_by_type(child, "namespace_name") alias_node = self._find_child_by_type(child, "namespace_aliasing_clause") if name_node: @@ -253,8 +291,8 @@ def _extract_nodes(self, node, lines: List[str], depth: int = 0, parent_class: s if node.type == "class_declaration": # Check for abstract class is_abstract = any( - c.type == "abstract_modifier" or - (c.type == "modifier" and c.text.decode() == "abstract") + c.type == "abstract_modifier" + or (c.type == "modifier" and c.text.decode() == "abstract") for c in node.children ) node_type = "abstract class" if is_abstract else "class" @@ -312,7 +350,7 @@ def _extract_nodes(self, node, lines: List[str], depth: int = 0, parent_class: s component_type=node_type, file_path=str(self.file_path), relative_path=relative_path, - source_code="\n".join(lines[node.start_point[0]:node.end_point[0]+1]), + source_code="\n".join(lines[node.start_point[0] : node.end_point[0] + 1]), start_line=node.start_point[0] + 1, end_line=node.end_point[0] + 1, has_docstring=bool(docstring), @@ -322,7 +360,7 @@ def _extract_nodes(self, node, lines: List[str], depth: int = 0, parent_class: s base_classes=base_classes, class_name=parent_class, display_name=f"{node_type} {node_name}", - component_id=component_id + component_id=component_id, ) self.nodes.append(node_obj) self._top_level_nodes[node_name] = node_obj @@ -352,12 +390,14 @@ def _extract_relationships(self, node, depth: int = 0): base_name = self._get_type_from_clause(base_clause) if base_name and not self._is_primitive(base_name): resolved_base = self.namespace_resolver.resolve(base_name) - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(class_name), - callee=resolved_base.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(class_name), + callee=resolved_base.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # 3. Interface implementation (implements) if node.type in ("class_declaration", "enum_declaration"): @@ -369,60 +409,71 @@ def _extract_relationships(self, node, depth: int = 0): interface_name = child.text.decode() if not self._is_primitive(interface_name): resolved_interface = self.namespace_resolver.resolve(interface_name) - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(implementer_name), - callee=resolved_interface.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(implementer_name), + callee=resolved_interface.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # 4. Object creation (new) if node.type == "object_creation_expression": containing_class = self._find_containing_class_name(node) - type_node = self._find_child_by_type(node, "name") or \ - self._find_child_by_type(node, "qualified_name") + type_node = self._find_child_by_type(node, "name") or self._find_child_by_type( + node, "qualified_name" + ) if type_node: created_type = type_node.text.decode() if not self._is_primitive(created_type) and containing_class: resolved_type = self.namespace_resolver.resolve(created_type) - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(containing_class), - callee=resolved_type.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(containing_class), + callee=resolved_type.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # 5. Static method calls (::) if node.type == "scoped_call_expression": containing_class = self._find_containing_class_name(node) - scope_node = self._find_child_by_type(node, "name") or \ - self._find_child_by_type(node, "qualified_name") + scope_node = self._find_child_by_type(node, "name") or self._find_child_by_type( + node, "qualified_name" + ) if scope_node and containing_class: target_class = scope_node.text.decode() if not self._is_primitive(target_class): resolved_target = self.namespace_resolver.resolve(target_class) - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(containing_class), - callee=resolved_target.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(containing_class), + callee=resolved_target.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # 6. Property promotion in constructor (PHP 8+) if node.type == "property_promotion_parameter": containing_class = self._find_containing_class_name(node) - type_node = self._find_child_by_type(node, "type_list") or \ - self._find_child_by_type(node, "named_type") + type_node = self._find_child_by_type(node, "type_list") or self._find_child_by_type( + node, "named_type" + ) if type_node and containing_class: type_name = self._extract_type_name(type_node) if type_name and not self._is_primitive(type_name): resolved_type = self.namespace_resolver.resolve(type_name) - self.call_relationships.append(CallRelationship( - caller=self._get_component_id(containing_class), - callee=resolved_type.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=self._get_component_id(containing_class), + callee=resolved_type.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) # Recursively process children for child in node.children: @@ -433,18 +484,21 @@ def _add_use_relationships(self, node): # Get all use clauses from the declaration for child in node.children: if child.type == "namespace_use_clause": - name_node = self._find_child_by_type(child, "qualified_name") or \ - self._find_child_by_type(child, "namespace_name") + name_node = self._find_child_by_type( + child, "qualified_name" + ) or self._find_child_by_type(child, "namespace_name") if name_node: fqn = name_node.text.decode().replace("\\", ".") # Add relationship from file to imported class file_id = self._get_relative_path() - self.call_relationships.append(CallRelationship( - caller=file_id, - callee=fqn, - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=file_id, + callee=fqn, + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) elif child.type == "namespace_use_group": prefix_node = self._find_child_by_type(node, "namespace_name") prefix = prefix_node.text.decode() if prefix_node else "" @@ -453,14 +507,20 @@ def _add_use_relationships(self, node): if group_child.type == "namespace_use_group_clause": name_node = self._find_child_by_type(group_child, "namespace_name") if name_node: - fqn = f"{prefix}\\{name_node.text.decode()}" if prefix else name_node.text.decode() + fqn = ( + f"{prefix}\\{name_node.text.decode()}" + if prefix + else name_node.text.decode() + ) file_id = self._get_relative_path() - self.call_relationships.append(CallRelationship( - caller=file_id, - callee=fqn.replace("\\", "."), - call_line=node.start_point[0] + 1, - is_resolved=False - )) + self.call_relationships.append( + CallRelationship( + caller=file_id, + callee=fqn.replace("\\", "."), + call_line=node.start_point[0] + 1, + is_resolved=False, + ) + ) def _find_child_by_type(self, node, child_type: str): """Find first child of a specific type.""" @@ -484,8 +544,9 @@ def _get_type_from_clause(self, clause_node) -> Optional[str]: def _extract_type_name(self, type_node) -> Optional[str]: """Extract type name from a type node.""" if type_node.type == "named_type": - name_node = self._find_child_by_type(type_node, "name") or \ - self._find_child_by_type(type_node, "qualified_name") + name_node = self._find_child_by_type(type_node, "name") or self._find_child_by_type( + type_node, "qualified_name" + ) if name_node: return name_node.text.decode() elif type_node.type in ("name", "qualified_name"): @@ -495,14 +556,18 @@ def _extract_type_name(self, type_node) -> Optional[str]: for child in type_node.children: if child.type == "named_type": return self._extract_type_name(child) - return type_node.text.decode() if hasattr(type_node, 'text') else None + return type_node.text.decode() if hasattr(type_node, "text") else None def _find_containing_class_name(self, node) -> Optional[str]: """Find the name of the containing class/interface/trait/enum.""" current = node.parent while current: - if current.type in ("class_declaration", "interface_declaration", - "trait_declaration", "enum_declaration"): + if current.type in ( + "class_declaration", + "interface_declaration", + "trait_declaration", + "enum_declaration", + ): name_node = self._find_child_by_type(current, "name") if name_node: return name_node.text.decode() @@ -544,14 +609,19 @@ def _extract_parameters(self, node) -> Optional[List[str]]: if params_node: params = [] for child in params_node.children: - if child.type in ("simple_parameter", "property_promotion_parameter", "variadic_parameter"): + if child.type in ( + "simple_parameter", + "property_promotion_parameter", + "variadic_parameter", + ): # Get the variable name var_node = self._find_child_by_type(child, "variable_name") if var_node: param_text = var_node.text.decode() # Get type if present - type_node = self._find_child_by_type(child, "named_type") or \ - self._find_child_by_type(child, "primitive_type") + type_node = self._find_child_by_type( + child, "named_type" + ) or self._find_child_by_type(child, "primitive_type") if type_node: param_text = f"{type_node.text.decode()} {param_text}" params.append(param_text) @@ -585,7 +655,9 @@ def _is_primitive(self, type_name: str) -> bool: return clean_name.lower() in {p.lower() for p in PHP_PRIMITIVES} -def analyze_php_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: +def analyze_php_file( + file_path: str, content: str, repo_path: str = None +) -> Tuple[List[Node], List[CallRelationship]]: """ Analyze a PHP file and extract nodes and call relationships. diff --git a/codewiki/src/be/dependency_analyzer/analyzers/python.py b/codewiki/src/be/dependency_analyzer/analyzers/python.py index 087037b..c58eb4a 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/python.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/python.py @@ -25,7 +25,6 @@ def is_project_import(target: str, project_modules: Set[str]) -> bool: class PythonASTAnalyzer(ast.NodeVisitor): - def __init__( self, file_path: str, @@ -90,15 +89,15 @@ def _get_relative_path(self) -> str: def _get_module_path(self) -> str: try: path = self._get_relative_path() - for ext in ['.py', '.pyx']: + for ext in [".py", ".pyx"]: if path.endswith(ext): - path = path[:-len(ext)] + path = path[: -len(ext)] break - module = path.replace('/', '.').replace('\\', '.') + module = path.replace("/", ".").replace("\\", ".") except Exception: - module = str(self.file_path).replace('/', '.').replace('\\', '.') - if module.endswith('.__init__'): - module = module[: -len('.__init__')] + module = str(self.file_path).replace("/", ".").replace("\\", ".") + if module.endswith(".__init__"): + module = module[: -len(".__init__")] return module def _current_class_dotted(self) -> Optional[str]: @@ -115,12 +114,14 @@ def _add_relationship(self, callee: str, is_resolved: bool, line: int) -> None: caller = self._caller_id() if not caller: return - self.call_relationships.append(CallRelationship( - caller=caller, - callee=callee, - call_line=line, - is_resolved=is_resolved, - )) + self.call_relationships.append( + CallRelationship( + caller=caller, + callee=callee, + call_line=line, + is_resolved=is_resolved, + ) + ) # ------------------------------------------------------------------ # Imports @@ -158,9 +159,7 @@ def _note_import_root(self, target: str) -> None: root = target.split(".")[0] if not root or root in PYTHON_STDLIB_MODULES: return - if self.project_modules is not None and not is_project_import( - target, self.project_modules - ): + if self.project_modules is not None and not is_project_import(target, self.project_modules): self.external_import_roots.add(root) # ------------------------------------------------------------------ @@ -219,12 +218,14 @@ def visit_ClassDef(self, node: ast.ClassDef): for base_name in base_classes: resolved = self._resolve_name_reference(base_name) if resolved: - self.call_relationships.append(CallRelationship( - caller=component_id, - callee=resolved[0], - call_line=node.lineno, - is_resolved=resolved[1], - )) + self.call_relationships.append( + CallRelationship( + caller=component_id, + callee=resolved[0], + call_line=node.lineno, + is_resolved=resolved[1], + ) + ) self.scope_stack.append(("class", node.name)) self.component_stack.append(component_id) @@ -428,8 +429,11 @@ class (with same-file inheritance), known classes and typed variables if rest: method = rest[-1] if root in self.var_types: - return self._resolve_method_on_class(self.var_types[root], method) if len(rest) == 1 else ( - ".".join([self.var_types[root], *rest]), False) + return ( + self._resolve_method_on_class(self.var_types[root], method) + if len(rest) == 1 + else (".".join([self.var_types[root], *rest]), False) + ) if root in self.class_methods and len(rest) == 1: return self._resolve_method_on_class(root, method) if root in self.from_imports: @@ -463,7 +467,9 @@ def _resolve_name_reference(self, name: str) -> Optional[Tuple[str, bool]]: return (self.module_imports[name], False) return (name, False) - def _resolve_method_on_class(self, class_dotted: Optional[str], method: str) -> Tuple[str, bool]: + def _resolve_method_on_class( + self, class_dotted: Optional[str], method: str + ) -> Tuple[str, bool]: if class_dotted: if method in self.class_methods.get(class_dotted, ()): return (f"{self._get_relative_path()}::{class_dotted}.{method}", True) @@ -473,7 +479,9 @@ def _resolve_method_on_class(self, class_dotted: Optional[str], method: str) -> return (f"{self._get_module_path()}.{class_dotted}.{method}", False) return (method, False) - def _resolve_method_via_bases(self, class_dotted: str, method: str) -> Optional[Tuple[str, bool]]: + def _resolve_method_via_bases( + self, class_dotted: str, method: str + ) -> Optional[Tuple[str, bool]]: seen = set() queue = list(self.class_bases.get(class_dotted, ())) while queue: diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/__init__.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/__init__.py index e6925d6..3882921 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/__init__.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/__init__.py @@ -3,6 +3,7 @@ Each extractor is a callable ``(file_path, content, repo_name) -> List[RouteNode]``. Register extractors in the ``EXTRACTORS`` dict keyed by file extension. """ + from __future__ import annotations from typing import Callable, Dict, List, Optional diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/go_routes.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/go_routes.py index 94d6bf3..b2ec493 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/go_routes.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/go_routes.py @@ -2,6 +2,7 @@ Uses regex-based heuristics on raw source text. """ + from __future__ import annotations import logging @@ -10,10 +11,13 @@ from typing import List, Optional from codewiki.src.be.dependency_analyzer.models.cross_service import ( - RouteNode, RouteProtocol, RouteRole, + RouteNode, + RouteProtocol, + RouteRole, ) from codewiki.src.be.dependency_analyzer.utils.path_canonicalizer import ( - canonicalize_path, make_route_key, + canonicalize_path, + make_route_key, ) logger = logging.getLogger(__name__) @@ -58,7 +62,7 @@ def _strip_url_to_path(url: str) -> str: return url for scheme in ("https://", "http://"): if url.startswith(scheme): - rest = url[len(scheme):] + rest = url[len(scheme) :] slash = rest.find("/") return rest[slash:] if slash != -1 else "/" return "/" + url if not url.startswith("/") else url @@ -92,22 +96,24 @@ def _extract_gin_routes(self): ) for m in pattern.finditer(self.content): path = m.group(2) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) http_method = method if method != "ANY" else "GET" - self.routes.append(RouteNode( - route_key=make_route_key(http_method, path), - protocol=RouteProtocol.HTTP, - method=http_method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name or path), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="gin", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(http_method, path), + protocol=RouteProtocol.HTTP, + method=http_method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name or path), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="gin", + ) + ) # ---- Chi / Echo / mux ---- @@ -126,41 +132,45 @@ def _extract_mux_routes(self): else: method = method_raw.upper() - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name or path), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="mux", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name or path), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="mux", + ) + ) # ---- net/http server ---- def _extract_http_server_routes(self): for m in _HTTP_SERVER_PATTERN.finditer(self.content): path = m.group(1) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key("GET", path), - protocol=RouteProtocol.HTTP, - method="GET", - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name or path), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="net/http", - )) + self.routes.append( + RouteNode( + route_key=make_route_key("GET", path), + protocol=RouteProtocol.HTTP, + method="GET", + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name or path), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="net/http", + ) + ) # ---- Client-side HTTP calls ---- @@ -171,25 +181,32 @@ def _extract_client_calls(self): url = m.group(2) path = _strip_url_to_path(url) - method_map = {"Get": "GET", "Post": "POST", "PostForm": "POST", - "Head": "HEAD", "Do": "GET"} + method_map = { + "Get": "GET", + "Post": "POST", + "PostForm": "POST", + "Head": "HEAD", + "Do": "GET", + } method = method_map.get(go_method, "GET") - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=self._make_component_id(func_name or "unknown"), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="net/http", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=self._make_component_id(func_name or "unknown"), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="net/http", + ) + ) # http.NewRequest("METHOD", "url", ...) for m in _NEW_REQUEST_PATTERN.finditer(self.content): @@ -197,27 +214,29 @@ def _extract_client_calls(self): url = m.group(2) path = _strip_url_to_path(url) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=self._make_component_id(func_name or "unknown"), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="net/http", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=self._make_component_id(func_name or "unknown"), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="net/http", + ) + ) # ---- helpers ---- def _find_enclosing_function(self, pos: int) -> Optional[str]: before = self.content[:pos] - matches = list(re.finditer(r'func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\(', before)) + matches = list(re.finditer(r"func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\(", before)) return matches[-1].group(1) if matches else None diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/java_routes.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/java_routes.py index 99952a6..ef53e25 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/java_routes.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/java_routes.py @@ -3,50 +3,54 @@ Uses Tree-sitter Java AST (same parser as the existing Java analyzer) to detect server-side annotations and client-side HTTP calls. """ + from __future__ import annotations import logging import os import re -from typing import List, Optional, Tuple +from typing import List, Optional from codewiki.src.be.dependency_analyzer.models.cross_service import ( - RouteNode, RouteProtocol, RouteRole, + RouteNode, + RouteProtocol, + RouteRole, ) from codewiki.src.be.dependency_analyzer.utils.path_canonicalizer import ( - canonicalize_path, make_route_key, + canonicalize_path, + make_route_key, ) logger = logging.getLogger(__name__) # Spring MVC annotations _SPRING_MAPPING_ANNOTATIONS = { - "GetMapping": "GET", - "PostMapping": "POST", - "PutMapping": "PUT", + "GetMapping": "GET", + "PostMapping": "POST", + "PutMapping": "PUT", "DeleteMapping": "DELETE", - "PatchMapping": "PATCH", + "PatchMapping": "PATCH", } _JAXRS_METHOD_ANNOTATIONS = { - "GET": "GET", - "POST": "POST", - "PUT": "PUT", + "GET": "GET", + "POST": "POST", + "PUT": "PUT", "DELETE": "DELETE", - "PATCH": "PATCH", - "HEAD": "HEAD", - "OPTIONS":"OPTIONS", + "PATCH": "PATCH", + "HEAD": "HEAD", + "OPTIONS": "OPTIONS", } # Client-side method patterns _REST_TEMPLATE_METHODS = { - "getForObject": "GET", - "getForEntity": "GET", - "postForObject": "POST", - "postForEntity": "POST", - "put": "PUT", - "delete": "DELETE", - "exchange": None, # method from HttpMethod arg + "getForObject": "GET", + "getForEntity": "GET", + "postForObject": "POST", + "postForEntity": "POST", + "put": "PUT", + "delete": "DELETE", + "exchange": None, # method from HttpMethod arg } _WEBCLIENT_METHODS = {"get", "post", "put", "delete", "patch", "head", "options"} @@ -72,7 +76,7 @@ def _strip_url_to_path(url: str) -> str: return url for scheme in ("https://", "http://"): if url.startswith(scheme): - rest = url[len(scheme):] + rest = url[len(scheme) :] slash = rest.find("/") return rest[slash:] if slash != -1 else "/" return "/" + url if not url.startswith("/") else url @@ -125,27 +129,29 @@ def _extract_spring_annotations(self): # Prepend class-level @RequestMapping prefix if class_prefix: path = class_prefix.rstrip("/") + "/" + path.lstrip("/") - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_next_method_name(m.end()) class_name = self._find_enclosing_class(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name, class_name), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="spring", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name, class_name), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="spring", + ) + ) # @RequestMapping(value="/path", method=RequestMethod.GET) # Only emit method-level @RequestMapping as routes; skip class-level ones # (class-level is used as prefix above). rm_pattern = re.compile( - r'@RequestMapping\s*\(([^)]+)\)', + r"@RequestMapping\s*\(([^)]+)\)", re.MULTILINE, ) for m in rm_pattern.finditer(self.content): @@ -154,7 +160,9 @@ def _extract_spring_annotations(self): continue params = m.group(1) - path = self._extract_param_value(params, "value") or self._extract_param_value(params, "path") + path = self._extract_param_value(params, "value") or self._extract_param_value( + params, "path" + ) if not path: # Try positional: @RequestMapping("/path") path = _extract_string_literal(params.strip()) @@ -174,30 +182,33 @@ def _extract_spring_annotations(self): method = m_upper break - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_next_method_name(m.end()) class_name = self._find_enclosing_class(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name, class_name), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="spring", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name, class_name), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="spring", + ) + ) # ---- JAX-RS ---- def _extract_jaxrs_annotations(self): import re + # @Path("/base") on class, @GET/@POST on method # Find methods with both @Path and a method annotation for ann_name, method in _JAXRS_METHOD_ANNOTATIONS.items(): - pattern = re.compile(rf'@{ann_name}\b', re.MULTILINE) + pattern = re.compile(rf"@{ann_name}\b", re.MULTILINE) for m in pattern.finditer(self.content): # Look for @Path near this annotation context_start = max(0, m.start() - 200) @@ -208,7 +219,7 @@ def _extract_jaxrs_annotations(self): if not path_match: continue path = path_match.group(1) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_next_method_name(m.end()) class_name = self._find_enclosing_class(m.start()) @@ -217,23 +228,25 @@ def _extract_jaxrs_annotations(self): if class_path: path = class_path.rstrip("/") + "/" + path.lstrip("/") - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name, class_name), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="jaxrs", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name, class_name), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="jaxrs", + ) + ) # ---- Feign clients ---- def _extract_feign_clients(self): - import re + # @FeignClient(name = "service-name") on interface # Then @GetMapping/@PostMapping on methods if "@FeignClient" not in self.content: @@ -244,20 +257,21 @@ def _extract_feign_clients(self): # Variable name patterns that indicate Map/collection receivers (not HTTP clients) _MAP_RECEIVER_PATTERN = re.compile( - r'(?:map|Map|hashMap|HashMap|concurrentMap|ConcurrentHashMap|linkedHashMap|' - r'LinkedHashMap|hashtable|Hashtable|properties|Properties|headers|headersMap|' - r'config|params|attributes|attrs|cache|registry|store|map\w*|\w+Map)\s*$', + r"(?:map|Map|hashMap|HashMap|concurrentMap|ConcurrentHashMap|linkedHashMap|" + r"LinkedHashMap|hashtable|Hashtable|properties|Properties|headers|headersMap|" + r"config|params|attributes|attrs|cache|registry|store|map\w*|\w+Map)\s*$", ) # Variable name patterns that indicate a RestTemplate / HTTP client receiver _HTTP_CLIENT_RECEIVER_PATTERN = re.compile( - r'(?:restTemplate|RestTemplate|template|httpClient|HttpClient|client|' - r'restClient|RestClient|http|webClient|WebClient)\s*$', + r"(?:restTemplate|RestTemplate|template|httpClient|HttpClient|client|" + r"restClient|RestClient|http|webClient|WebClient)\s*$", re.IGNORECASE, ) def _extract_client_calls(self): import re + # RestTemplate: restTemplate.getForObject("/path", ...) for method_name, http_method in _REST_TEMPLATE_METHODS.items(): pattern = re.compile( @@ -277,24 +291,26 @@ def _extract_client_calls(self): continue path = _strip_url_to_path(url) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_method(m.start()) class_name = self._find_enclosing_class(m.start()) method = http_method or "GET" - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=self._make_component_id( - func_name or "unknown", class_name or "" - ), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="resttemplate", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=self._make_component_id( + func_name or "unknown", class_name or "" + ), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="resttemplate", + ) + ) # WebClient: webClient.get().uri("/path") wc_pattern = re.compile( @@ -305,81 +321,87 @@ def _extract_client_calls(self): method = m.group(1).upper() url = m.group(2) path = _strip_url_to_path(url) - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_method(m.start()) class_name = self._find_enclosing_class(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=self._make_component_id( - func_name or "unknown", class_name or "" - ), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="webclient", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=self._make_component_id(func_name or "unknown", class_name or ""), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="webclient", + ) + ) # ---- helpers ---- def _extract_param_value(self, params: str, key: str) -> Optional[str]: import re + m = re.search(rf'{key}\s*=\s*"([^"]*)"', params) return m.group(1) if m else None def _find_next_method_name(self, pos: int) -> str: """Find the next Java method declaration after *pos*.""" - rest = self.content[pos:pos + 1000] + rest = self.content[pos : pos + 1000] m = re.search( - r'(?:public|private|protected|static|final|abstract|synchronized|\s)+\s+' - r'[\w<>\[\],\s]+\s+(\w+)\s*\(', + r"(?:public|private|protected|static|final|abstract|synchronized|\s)+\s+" + r"[\w<>\[\],\s]+\s+(\w+)\s*\(", rest, ) return m.group(1) if m else "unknown" def _find_enclosing_class(self, pos: int) -> str: import re + before = self.content[:pos] - matches = list(re.finditer(r'(?:class|interface)\s+(\w+)', before)) + matches = list(re.finditer(r"(?:class|interface)\s+(\w+)", before)) return matches[-1].group(1) if matches else "" def _find_enclosing_method(self, pos: int) -> Optional[str]: before = self.content[:pos] - matches = list(re.finditer( - r'(?:public|private|protected|static|final|abstract|synchronized|\s)+\s+' - r'[\w<>\[\],\s]+\s+(\w+)\s*\(', - before, - )) + matches = list( + re.finditer( + r"(?:public|private|protected|static|final|abstract|synchronized|\s)+\s+" + r"[\w<>\[\],\s]+\s+(\w+)\s*\(", + before, + ) + ) return matches[-1].group(1) if matches else None def _find_class_path(self, pos: int) -> str: """Find the @Path annotation on the enclosing class.""" import re + before = self.content[:pos] # Find last class declaration - class_matches = list(re.finditer(r'class\s+\w+', before)) + class_matches = list(re.finditer(r"class\s+\w+", before)) if not class_matches: return "" class_pos = class_matches[-1].start() # Look for @Path before the class - pre_class = before[max(0, class_pos - 300):class_pos] + pre_class = before[max(0, class_pos - 300) : class_pos] path_match = re.search(r'@Path\s*\(\s*"([^"]+)"', pre_class) return path_match.group(1) if path_match else "" def _find_class_request_mapping(self) -> str: """Find the class-level @RequestMapping value (Spring MVC prefix).""" import re + # Find the first class/interface declaration - class_match = re.search(r'(?:class|interface)\s+\w+', self.content) + class_match = re.search(r"(?:class|interface)\s+\w+", self.content) if not class_match: return "" class_pos = class_match.start() # Look for @RequestMapping in the 500 chars before the class declaration - pre_class = self.content[max(0, class_pos - 500):class_pos] - rm_match = re.search(r'@RequestMapping\s*\(([^)]+)\)', pre_class) + pre_class = self.content[max(0, class_pos - 500) : class_pos] + rm_match = re.search(r"@RequestMapping\s*\(([^)]+)\)", pre_class) if not rm_match: # Also try simple form: @RequestMapping("/path") rm_simple = re.search(r'@RequestMapping\s*\(\s*"([^"]+)"', pre_class) @@ -396,16 +418,17 @@ def _find_class_request_mapping(self) -> str: def _is_class_level_annotation(self, pos: int) -> bool: """Check if the annotation at *pos* is class-level (before a class/interface decl).""" import re + # Look at the text between this annotation and the next declaration - after = self.content[pos:pos + 500] + after = self.content[pos : pos + 500] # If the next significant declaration after the annotation is a class/interface, # then this is a class-level annotation next_decl = re.search( - r'(?:public|private|protected|static|final|abstract|\s)*\s*(class|interface)\s+\w+', + r"(?:public|private|protected|static|final|abstract|\s)*\s*(class|interface)\s+\w+", after, ) next_method = re.search( - r'(?:public|private|protected|static|final|abstract|synchronized|\s)+\s*[\w<>\[\],\s]+\s+\w+\s*\(', + r"(?:public|private|protected|static|final|abstract|synchronized|\s)+\s*[\w<>\[\],\s]+\s+\w+\s*\(", after, ) if next_decl and (not next_method or next_decl.start() <= next_method.start()): diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/js_routes.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/js_routes.py index 6e82146..9d1ac12 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/js_routes.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/js_routes.py @@ -3,6 +3,7 @@ Uses regex-based heuristics on raw source text (the main analyzers already use tree-sitter; route extraction is a lightweight post-pass). """ + from __future__ import annotations import logging @@ -11,10 +12,13 @@ from typing import List, Optional from codewiki.src.be.dependency_analyzer.models.cross_service import ( - RouteNode, RouteProtocol, RouteRole, + RouteNode, + RouteProtocol, + RouteRole, ) from codewiki.src.be.dependency_analyzer.utils.path_canonicalizer import ( - canonicalize_path, make_route_key, + canonicalize_path, + make_route_key, ) logger = logging.getLogger(__name__) @@ -55,7 +59,7 @@ def _strip_url_to_path(url: str) -> str: return url for scheme in ("https://", "http://"): if url.startswith(scheme): - rest = url[len(scheme):] + rest = url[len(scheme) :] slash = rest.find("/") return rest[slash:] if slash != -1 else "/" return "/" + url if not url.startswith("/") else url @@ -89,7 +93,7 @@ def _extract_express_routes(self): # Detect axios instance variables: const X = axios.create(...), const X = axios, # import X from 'axios' for m in re.finditer( - r'(?:const|let|var)\s+(\w+)\s*=\s*axios(?:\s*\.\s*create\s*\(|\s*[;,\n])', + r"(?:const|let|var)\s+(\w+)\s*=\s*axios(?:\s*\.\s*create\s*\(|\s*[;,\n])", self.content, ): _excluded_names.add(m.group(1)) @@ -126,21 +130,23 @@ def _extract_express_routes(self): else: continue - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name or path), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="express", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name or path), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="express", + ) + ) # ---- NestJS ---- @@ -160,21 +166,23 @@ def _extract_nestjs_routes(self): if controller_prefix: path = controller_prefix.rstrip("/") + "/" + path.lstrip("/") - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) - self.routes.append(RouteNode( - route_key=make_route_key(method, path or "/"), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path or "/"), - role=RouteRole.SERVER, - component_id=self._make_component_id(func_name or path), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="nestjs", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path or "/"), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path or "/"), + role=RouteRole.SERVER, + component_id=self._make_component_id(func_name or path), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="nestjs", + ) + ) # ---- Client-side HTTP calls ---- @@ -195,10 +203,8 @@ def _extract_client_calls(self): method = "GET" # default, might be overridden in options # Check for method in nearby options object context_end = min(len(self.content), m.end() + 200) - context = self.content[m.start():context_end] - method_match = re.search( - r'method\s*:\s*["\'`](\w+)["\'`]', context - ) + context = self.content[m.start() : context_end] + method_match = re.search(r'method\s*:\s*["\'`](\w+)["\'`]', context) if method_match: method = method_match.group(1).upper() else: @@ -208,7 +214,7 @@ def _extract_client_calls(self): if not path or len(path) < 2: continue - lineno = self.content[:m.start()].count("\n") + 1 + lineno = self.content[: m.start()].count("\n") + 1 func_name = self._find_enclosing_function(m.start()) framework = "axios" @@ -217,18 +223,20 @@ def _extract_client_calls(self): elif "got" in m.group(0) or "ky" in m.group(0): framework = "got" - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=self._make_component_id(func_name or "unknown"), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework=framework, - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=self._make_component_id(func_name or "unknown"), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework=framework, + ) + ) # ---- helpers ---- @@ -241,13 +249,15 @@ def _find_enclosing_function(self, pos: int) -> Optional[str]: # group 4: async name( # Group 2 must NOT match plain variable assignments like `const res = await ...`; # only match when RHS is a function definition (has `=>` or `function` keyword). - matches = list(re.finditer( - r'(?:function\s+(\w+)' - r'|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function\b|(?:\([^)]*\)|\w+)\s*=>)' - r'|(\w+)\s*\([^)]*\)\s*\{' - r'|async\s+(\w+)\s*\()', - before, - )) + matches = list( + re.finditer( + r"(?:function\s+(\w+)" + r"|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function\b|(?:\([^)]*\)|\w+)\s*=>)" + r"|(\w+)\s*\([^)]*\)\s*\{" + r"|async\s+(\w+)\s*\()", + before, + ) + ) if matches: last = matches[-1] return last.group(1) or last.group(2) or last.group(3) or last.group(4) diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/mq_patterns.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/mq_patterns.py index d9ce571..caa6f39 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/mq_patterns.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/mq_patterns.py @@ -3,6 +3,7 @@ Detects message-queue producers and consumers in source code, creating Route nodes with ``RouteProtocol.MQ``. """ + from __future__ import annotations import logging @@ -11,7 +12,9 @@ from typing import List, Optional from codewiki.src.be.dependency_analyzer.models.cross_service import ( - RouteNode, RouteProtocol, RouteRole, + RouteNode, + RouteProtocol, + RouteRole, ) from codewiki.src.be.dependency_analyzer.utils.path_canonicalizer import make_mq_route_key @@ -36,9 +39,9 @@ def _component_id(file_path: str, func_name: str, class_name: str = "") -> str: # Each pattern is (regex, broker, role, group_indices_for_topic_and_func) # group_indices: (topic_group, ) — func_name is found from enclosing function + class _Pattern: - def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, - topic_group: int = 1): + def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, topic_group: int = 1): self.pattern = pattern self.broker = broker self.role = role @@ -51,7 +54,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:kafkaTemplate|producer|kafkaProducer)\s*\.\s*(?:send|sendDefault)\s*\(\s*"([^"]+)"', re.MULTILINE, ), - broker="kafka", role=RouteRole.CLIENT, topic_group=1, + broker="kafka", + role=RouteRole.CLIENT, + topic_group=1, ) _KAFKA_CONSUMER = _Pattern( @@ -59,7 +64,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'@KafkaListener\s*\(\s*(?:topics\s*=\s*)?(?:\{)?\s*"([^"]+)"', re.MULTILINE, ), - broker="kafka", role=RouteRole.SERVER, topic_group=1, + broker="kafka", + role=RouteRole.SERVER, + topic_group=1, ) # RabbitMQ @@ -68,7 +75,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:rabbitTemplate|amqpTemplate)\s*\.\s*(?:convertAndSend|send)\s*\(\s*"([^"]+)"', re.MULTILINE, ), - broker="rabbitmq", role=RouteRole.CLIENT, topic_group=1, + broker="rabbitmq", + role=RouteRole.CLIENT, + topic_group=1, ) _RABBIT_CONSUMER = _Pattern( @@ -76,7 +85,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'@RabbitListener\s*\(\s*(?:queues\s*=\s*)?\s*"([^"]+)"', re.MULTILINE, ), - broker="rabbitmq", role=RouteRole.SERVER, topic_group=1, + broker="rabbitmq", + role=RouteRole.SERVER, + topic_group=1, ) # RocketMQ @@ -85,7 +96,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:rocketMQTemplate|defaultMQProducer)\s*\.\s*(?:send|convertAndSend|syncSend)\s*\(\s*"([^"]+)"', re.MULTILINE, ), - broker="rocketmq", role=RouteRole.CLIENT, topic_group=1, + broker="rocketmq", + role=RouteRole.CLIENT, + topic_group=1, ) _ROCKET_CONSUMER = _Pattern( @@ -93,7 +106,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'@RocketMQMessageListener\s*\([^)]*topic\s*=\s*"([^"]+)"', re.MULTILINE, ), - broker="rocketmq", role=RouteRole.SERVER, topic_group=1, + broker="rocketmq", + role=RouteRole.SERVER, + topic_group=1, ) # Celery @@ -102,7 +117,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:celery|app)\s*\.\s*send_task\s*\(\s*"([^"]+)"', re.MULTILINE, ), - broker="celery", role=RouteRole.CLIENT, topic_group=1, + broker="celery", + role=RouteRole.CLIENT, + topic_group=1, ) _CELERY_CONSUMER = _Pattern( @@ -110,7 +127,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'@(?:celery|app)\s*\.\s*task\s*(?:\(\s*name\s*=\s*"([^"]+)")?', re.MULTILINE, ), - broker="celery", role=RouteRole.SERVER, topic_group=1, + broker="celery", + role=RouteRole.SERVER, + topic_group=1, ) # Python Kafka (kafka-python / confluent-kafka) @@ -119,7 +138,9 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:producer|kafka_producer)\s*\.\s*(?:send|produce)\s*\(\s*["\']([^"\']+)["\']', re.MULTILINE, ), - broker="kafka", role=RouteRole.CLIENT, topic_group=1, + broker="kafka", + role=RouteRole.CLIENT, + topic_group=1, ) # Go Kafka (segmentio/kafka-go, confluent-kafka-go) @@ -128,14 +149,20 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, r'(?:writer|producer)\s*\.\s*(?:WriteMessages|Produce)\s*\([^,]*,\s*kafka\.Message\{[^}]*Topic:\s*"([^"]+)"', re.MULTILINE | re.DOTALL, ), - broker="kafka", role=RouteRole.CLIENT, topic_group=1, + broker="kafka", + role=RouteRole.CLIENT, + topic_group=1, ) ALL_PATTERNS = [ - _KAFKA_PRODUCER, _KAFKA_CONSUMER, - _RABBIT_PRODUCER, _RABBIT_CONSUMER, - _ROCKET_PRODUCER, _ROCKET_CONSUMER, - _CELERY_PRODUCER, _CELERY_CONSUMER, + _KAFKA_PRODUCER, + _KAFKA_CONSUMER, + _RABBIT_PRODUCER, + _RABBIT_CONSUMER, + _ROCKET_PRODUCER, + _ROCKET_CONSUMER, + _CELERY_PRODUCER, + _CELERY_CONSUMER, _PY_KAFKA_PRODUCER, _GO_KAFKA_PRODUCER, ] @@ -144,11 +171,15 @@ def __init__(self, pattern: re.Pattern, broker: str, role: RouteRole, def extract_mq_routes(file_path: str, content: str, repo_name: str) -> List[RouteNode]: """Extract MQ producer/consumer routes from any language source file.""" routes: List[RouteNode] = [] - rel_path = _get_relative_path(file_path) + _get_relative_path(file_path) for pat in ALL_PATTERNS: for m in pat.pattern.finditer(content): - topic = m.group(pat.topic_group) if pat.topic_group <= len(m.groups()) and m.group(pat.topic_group) else "" + topic = ( + m.group(pat.topic_group) + if pat.topic_group <= len(m.groups()) and m.group(pat.topic_group) + else "" + ) if not topic: # For Celery @app.task without name, use function name func_name = _find_enclosing_function(content, m.start()) @@ -157,25 +188,27 @@ def extract_mq_routes(file_path: str, content: str, repo_name: str) -> List[Rout else: continue - lineno = content[:m.start()].count("\n") + 1 + lineno = content[: m.start()].count("\n") + 1 func_name = _find_enclosing_function(content, m.start()) or "unknown" # Determine class name for Java/Kotlin class_name = _find_enclosing_class(content, m.start()) - routes.append(RouteNode( - route_key=make_mq_route_key(pat.broker, topic), - protocol=RouteProtocol.MQ, - method=None, - path=topic, - role=pat.role, - component_id=_component_id(file_path, func_name, class_name), - repo_name=repo_name, - file_path=file_path, - line_number=lineno, - framework=pat.broker, - extra={"broker": pat.broker, "topic": topic}, - )) + routes.append( + RouteNode( + route_key=make_mq_route_key(pat.broker, topic), + protocol=RouteProtocol.MQ, + method=None, + path=topic, + role=pat.role, + component_id=_component_id(file_path, func_name, class_name), + repo_name=repo_name, + file_path=file_path, + line_number=lineno, + framework=pat.broker, + extra={"broker": pat.broker, "topic": topic}, + ) + ) return routes @@ -186,13 +219,15 @@ def _find_enclosing_function(content: str, pos: int) -> Optional[str]: # Java/Kotlin/C#/Go: func/method declarations patterns = [ # Java/Kotlin: (public|private|...) Type methodName( - re.compile(r'(?:public|private|protected|static|final|synchronized|abstract|\s)+\s+\S+\s+(\w+)\s*\('), + re.compile( + r"(?:public|private|protected|static|final|synchronized|abstract|\s)+\s+\S+\s+(\w+)\s*\(" + ), # Python: def func_name( - re.compile(r'def\s+(\w+)\s*\('), + re.compile(r"def\s+(\w+)\s*\("), # Go: func (receiver)? name( - re.compile(r'func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\('), + re.compile(r"func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\("), # JS/TS: function name( or const name = ( or name( - re.compile(r'(?:function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=|async\s+(\w+)\s*\()'), + re.compile(r"(?:function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=|async\s+(\w+)\s*\()"), ] for pat in patterns: matches = list(pat.finditer(before)) @@ -207,5 +242,5 @@ def _find_enclosing_function(content: str, pos: int) -> Optional[str]: def _find_enclosing_class(content: str, pos: int) -> str: before = content[:pos] - matches = list(re.finditer(r'(?:class|interface)\s+(\w+)', before)) + matches = list(re.finditer(r"(?:class|interface)\s+(\w+)", before)) return matches[-1].group(1) if matches else "" diff --git a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/python_routes.py b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/python_routes.py index cb31f81..7f6a6df 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/python_routes.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/route_extractors/python_routes.py @@ -4,6 +4,7 @@ PythonASTAnalyzer) to detect server-side route decorators and client-side HTTP calls. """ + from __future__ import annotations import ast @@ -12,10 +13,13 @@ from typing import List, Optional from codewiki.src.be.dependency_analyzer.models.cross_service import ( - RouteNode, RouteProtocol, RouteRole, + RouteNode, + RouteProtocol, + RouteRole, ) from codewiki.src.be.dependency_analyzer.utils.path_canonicalizer import ( - canonicalize_path, make_route_key, + canonicalize_path, + make_route_key, ) logger = logging.getLogger(__name__) @@ -24,13 +28,13 @@ # decorator.func.attr → (method, framework) _DECORATOR_METHOD_MAP = { - "get": ("GET", "fastapi"), - "post": ("POST", "fastapi"), - "put": ("PUT", "fastapi"), + "get": ("GET", "fastapi"), + "post": ("POST", "fastapi"), + "put": ("PUT", "fastapi"), "delete": ("DELETE", "fastapi"), - "patch": ("PATCH", "fastapi"), - "head": ("HEAD", "fastapi"), - "options":("OPTIONS","fastapi"), + "patch": ("PATCH", "fastapi"), + "head": ("HEAD", "fastapi"), + "options": ("OPTIONS", "fastapi"), } _FLASK_ROUTE_ATTRS = {"route"} @@ -42,24 +46,24 @@ _CLIENT_LIBRARIES = { # module.method → (method, framework) - "requests.get": ("GET", "requests"), - "requests.post": ("POST", "requests"), - "requests.put": ("PUT", "requests"), + "requests.get": ("GET", "requests"), + "requests.post": ("POST", "requests"), + "requests.put": ("PUT", "requests"), "requests.delete": ("DELETE", "requests"), - "requests.patch": ("PATCH", "requests"), - "requests.head": ("HEAD", "requests"), - "requests.request":(None, "requests"), # method from 1st arg - "httpx.get": ("GET", "httpx"), - "httpx.post": ("POST", "httpx"), - "httpx.put": ("PUT", "httpx"), - "httpx.delete": ("DELETE", "httpx"), - "httpx.patch": ("PATCH", "httpx"), - "httpx.request": (None, "httpx"), - "aiohttp.get": ("GET", "aiohttp"), - "aiohttp.post": ("POST", "aiohttp"), - "aiohttp.put": ("PUT", "aiohttp"), - "aiohttp.delete": ("DELETE", "aiohttp"), - "aiohttp.patch": ("PATCH", "aiohttp"), + "requests.patch": ("PATCH", "requests"), + "requests.head": ("HEAD", "requests"), + "requests.request": (None, "requests"), # method from 1st arg + "httpx.get": ("GET", "httpx"), + "httpx.post": ("POST", "httpx"), + "httpx.put": ("PUT", "httpx"), + "httpx.delete": ("DELETE", "httpx"), + "httpx.patch": ("PATCH", "httpx"), + "httpx.request": (None, "httpx"), + "aiohttp.get": ("GET", "aiohttp"), + "aiohttp.post": ("POST", "aiohttp"), + "aiohttp.put": ("PUT", "aiohttp"), + "aiohttp.delete": ("DELETE", "aiohttp"), + "aiohttp.patch": ("PATCH", "aiohttp"), } _HTTP_METHODS_UPPER = {"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"} @@ -105,9 +109,7 @@ def __init__(self, file_path: str, repo_name: str): # ---- helpers ---- def _make_component_id(self, name: str) -> str: - return _component_id_from_context( - self.file_path, name, self._current_class or "" - ) + return _component_id_from_context(self.file_path, name, self._current_class or "") def _extract_string_arg(self, node: ast.expr) -> Optional[str]: """Extract a string literal from an AST expression.""" @@ -144,20 +146,22 @@ def _check_decorator(self, decorator: ast.expr, lineno: int): path = self._extract_string_arg(args[0]) if path: method, fw = _DECORATOR_METHOD_MAP[attr] - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id( - self._current_func or "unknown" - ), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework=fw, - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id( + self._current_func or "unknown" + ), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework=fw, + ) + ) return # Flask: @app.route("/path", methods=["GET"]) @@ -175,20 +179,22 @@ def _check_decorator(self, decorator: ast.expr, lineno: int): if m and m.upper() in _HTTP_METHODS_UPPER: method = m.upper() break - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.SERVER, - component_id=self._make_component_id( - self._current_func or "unknown" - ), - repo_name=self.repo_name, - file_path=self.file_path, - line_number=lineno, - framework="flask", - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.SERVER, + component_id=self._make_component_id( + self._current_func or "unknown" + ), + repo_name=self.repo_name, + file_path=self.file_path, + line_number=lineno, + framework="flask", + ) + ) return # Django: path("route/", view_func) @@ -243,18 +249,20 @@ def _check_client_call(self, node: ast.Call): method = method_hint or "GET" comp_id = self._make_component_id(self._current_func or "unknown") - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=comp_id, - repo_name=self.repo_name, - file_path=self.file_path, - line_number=node.lineno, - framework=framework, - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=comp_id, + repo_name=self.repo_name, + file_path=self.file_path, + line_number=node.lineno, + framework=framework, + ) + ) return # Pattern: instance.method(...) e.g. c.get("/path") where c is a @@ -276,18 +284,20 @@ def _check_client_call(self, node: ast.Call): if not path: return comp_id = self._make_component_id(self._current_func or "unknown") - self.routes.append(RouteNode( - route_key=make_route_key(method, path), - protocol=RouteProtocol.HTTP, - method=method, - path=canonicalize_path(path), - role=RouteRole.CLIENT, - component_id=comp_id, - repo_name=self.repo_name, - file_path=self.file_path, - line_number=node.lineno, - framework=framework, - )) + self.routes.append( + RouteNode( + route_key=make_route_key(method, path), + protocol=RouteProtocol.HTTP, + method=method, + path=canonicalize_path(path), + role=RouteRole.CLIENT, + component_id=comp_id, + repo_name=self.repo_name, + file_path=self.file_path, + line_number=node.lineno, + framework=framework, + ) + ) # ---- AST visitor methods ---- @@ -369,7 +379,7 @@ def _strip_url_to_path(url: str) -> str: # Full URL — strip scheme + host for scheme in ("https://", "http://"): if url.startswith(scheme): - rest = url[len(scheme):] + rest = url[len(scheme) :] slash = rest.find("/") if slash != -1: return rest[slash:] diff --git a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py index 43d4c39..929c278 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py @@ -3,9 +3,6 @@ import traceback from typing import List, Set, Optional, Tuple from pathlib import Path -import sys -import os -from traceback import print_exc from tree_sitter import Parser, Language import tree_sitter_typescript @@ -17,8 +14,8 @@ logger = logging.getLogger(__name__) -class TreeSitterTSAnalyzer: +class TreeSitterTSAnalyzer: def __init__(self, file_path: str, content: str, repo_path: str = None): self.file_path = Path(file_path) self.content = content @@ -51,11 +48,11 @@ def analyze(self) -> None: logger.debug(f"Parsed AST with root node type: {root_node.type}") - all_entities = {} + all_entities = {} self._extract_all_entities(root_node, all_entities) - + self._filter_top_level_declarations(all_entities) - + self._extract_all_relationships(root_node, all_entities) except Exception as e: @@ -64,7 +61,7 @@ def analyze(self) -> None: def _extract_all_entities(self, node, all_entities: dict, depth=0) -> None: entity = None entity_name = None - + if node.type == "function_declaration": entity = self._extract_function_entity(node, "function", depth) elif node.type == "generator_function_declaration": @@ -93,17 +90,17 @@ def _extract_all_entities(self, node, all_entities: dict, depth=0) -> None: entity = self._extract_variable_declaration_entity(node, depth) elif node.type == "ambient_declaration": entity = self._extract_ambient_declaration_entity(node, depth) - - if entity and entity.get('name'): - entity_name = entity['name'] - entity['depth'] = depth - entity['node'] = node - entity['parent_context'] = self._get_parent_context(node) + + if entity and entity.get("name"): + entity_name = entity["name"] + entity["depth"] = depth + entity["node"] = node + entity["parent_context"] = self._get_parent_context(node) all_entities[entity_name] = entity - + for child in node.children: self._extract_all_entities(child, all_entities, depth + 1) - + def _filter_top_level_declarations(self, all_entities: dict) -> None: for entity_name, entity_data in all_entities.items(): if self._is_actually_top_level(entity_data): @@ -111,51 +108,54 @@ def _filter_top_level_declarations(self, all_entities: dict) -> None: if node_obj and self._should_include_node(node_obj): self.nodes.append(node_obj) self.top_level_nodes[entity_name] = node_obj - + if entity_data["type"] in ["class_declaration", "abstract_class_declaration"]: self._extract_constructor_dependencies(entity_data["node"], entity_name) - + def _is_actually_top_level(self, entity_data: dict) -> bool: - node = entity_data.get('node') + node = entity_data.get("node") if not node or not node.parent: return True - - entity_type = entity_data.get('type') + + entity_data.get("type") if self._is_inside_function_body(node): return False - + current = node.parent while current: parent_type = current.type - + if parent_type == "program": return True - + if parent_type == "export_statement": return True - + if parent_type == "ambient_declaration": return True - + if parent_type == "module": return True - + if parent_type == "statement_block": grandparent = current.parent if grandparent and grandparent.type in ["module", "ambient_declaration"]: return True - + current = current.parent - + return False - + def _is_inside_function_body(self, node) -> bool: current = node.parent while current: if current.type == "statement_block": if current.parent and current.parent.type in [ - "function_declaration", "generator_function_declaration", - "arrow_function", "function_expression", "method_definition" + "function_declaration", + "generator_function_declaration", + "arrow_function", + "function_expression", + "method_definition", ]: return True current = current.parent @@ -171,25 +171,29 @@ def _extract_ambient_declaration_entity(self, node, depth: int) -> dict: break break elif child.type == "namespace": - name = self._get_node_text(child.children[1]) if len(child.children) > 1 else "unknown_namespace" + name = ( + self._get_node_text(child.children[1]) + if len(child.children) > 1 + else "unknown_namespace" + ) break - + return { - 'name': f"{name}", - 'type': 'ambient_declaration', - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'parameters': [], - 'return_type': None, - 'modifiers': ['ambient'], - 'complexity': 1 + "name": f"{name}", + "type": "ambient_declaration", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "parameters": [], + "return_type": None, + "modifiers": ["ambient"], + "complexity": 1, } - + def _get_parent_context(self, node) -> str: """Get the parent context of a node for better top-level detection""" if not node.parent: return "root" - + parent_type = node.parent.type if parent_type in ["program", "source_file"]: return "program" @@ -203,30 +207,33 @@ def _get_parent_context(self, node) -> str: if node.parent.parent and node.parent.parent.type in ["module", "ambient_declaration"]: return "module_block" return "statement_block" + def _extract_function_entity(self, node, func_type: str, depth: int) -> dict: name_node = self._find_child_by_type(node, "identifier") if not name_node: return None - + func_name = self._get_node_text(name_node) parameters = self._extract_parameters(node) code_snippet = self._get_node_text(node) - - is_async = "async" in code_snippet.split("function")[0] if "function" in code_snippet else False + + is_async = ( + "async" in code_snippet.split("function")[0] if "function" in code_snippet else False + ) display_name = f"{'async ' if is_async else ''}{func_type} {func_name}" - + return { - 'name': func_name, - 'type': 'function', - 'subtype': func_type, - 'parameters': parameters, - 'code_snippet': code_snippet, - 'display_name': display_name, - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'is_async': is_async + "name": func_name, + "type": "function", + "subtype": func_type, + "parameters": parameters, + "code_snippet": code_snippet, + "display_name": display_name, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "is_async": is_async, } - + def _extract_arrow_function_entity(self, node, depth: int) -> dict: """Extract arrow function""" parent = node.parent @@ -236,23 +243,23 @@ def _extract_arrow_function_entity(self, node, depth: int) -> dict: func_name = self._get_node_text(name_node) parameters = self._extract_parameters(node) code_snippet = self._get_node_text(parent) - + is_async = "async" in code_snippet.split("=")[0] if "=" in code_snippet else False display_name = f"{'async ' if is_async else ''}arrow function {func_name}" - + return { - 'name': func_name, - 'type': 'function', - 'subtype': 'arrow_function', - 'parameters': parameters, - 'code_snippet': code_snippet, - 'display_name': display_name, - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'is_async': is_async + "name": func_name, + "type": "function", + "subtype": "arrow_function", + "parameters": parameters, + "code_snippet": code_snippet, + "display_name": display_name, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "is_async": is_async, } return None - + def _extract_method_entity(self, node, depth: int) -> dict: """Extract method entity (at any depth), qualified by its class.""" name_node = self._find_child_by_type(node, "property_identifier") @@ -268,208 +275,218 @@ def _extract_method_entity(self, node, depth: int) -> dict: is_async = "async" in code_snippet is_static = "static" in code_snippet - display_name = f"{'static ' if is_static else ''}{'async ' if is_async else ''}method {qualified}" + display_name = ( + f"{'static ' if is_static else ''}{'async ' if is_async else ''}method {qualified}" + ) return { - 'name': qualified, - 'type': 'function', - 'subtype': 'method', - 'class_name': class_name, - 'parameters': parameters, - 'code_snippet': code_snippet, - 'display_name': display_name, - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'is_async': is_async, - 'is_static': is_static + "name": qualified, + "type": "function", + "subtype": "method", + "class_name": class_name, + "parameters": parameters, + "code_snippet": code_snippet, + "display_name": display_name, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "is_async": is_async, + "is_static": is_static, } def _enclosing_class_name(self, node) -> Optional[str]: current = node.parent while current: if current.type in ("class_declaration", "abstract_class_declaration"): - name_node = self._find_child_by_type(current, "type_identifier") or self._find_child_by_type(current, "identifier") + name_node = self._find_child_by_type( + current, "type_identifier" + ) or self._find_child_by_type(current, "identifier") return self._get_node_text(name_node) if name_node else None current = current.parent return None - + def _extract_class_entity(self, node, class_type: str, depth: int) -> dict: - name_node = self._find_child_by_type(node, "type_identifier") or self._find_child_by_type(node, "identifier") + name_node = self._find_child_by_type(node, "type_identifier") or self._find_child_by_type( + node, "identifier" + ) if not name_node: return None - + class_name = self._get_node_text(name_node) base_classes = self._extract_inheritance(node) code_snippet = self._get_node_text(node) - + display_name = f"{class_type} {class_name}" if base_classes: display_name += f" extends {', '.join(base_classes)}" - + return { - 'name': class_name, - 'type': 'class', - 'subtype': class_type, - 'base_classes': base_classes, - 'code_snippet': code_snippet, - 'display_name': display_name, - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1 + "name": class_name, + "type": "class", + "subtype": class_type, + "base_classes": base_classes, + "code_snippet": code_snippet, + "display_name": display_name, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, } - + def _extract_interface_entity(self, node, depth: int) -> dict: name_node = self._find_child_by_type(node, "type_identifier") if not name_node: return None - + interface_name = self._get_node_text(name_node) base_classes = self._extract_inheritance(node) code_snippet = self._get_node_text(node) - + display_name = f"interface {interface_name}" if base_classes: display_name += f" extends {', '.join(base_classes)}" - + return { - 'name': interface_name, - 'type': 'interface', - 'subtype': 'interface', - 'base_classes': base_classes, - 'code_snippet': code_snippet, - 'display_name': display_name, - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1 + "name": interface_name, + "type": "interface", + "subtype": "interface", + "base_classes": base_classes, + "code_snippet": code_snippet, + "display_name": display_name, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, } - + def _extract_type_alias_entity(self, node, depth: int) -> dict: name_node = self._find_child_by_type(node, "type_identifier") if not name_node: return None - + type_name = self._get_node_text(name_node) code_snippet = self._get_node_text(node) - + return { - 'name': type_name, - 'type': 'type', - 'subtype': 'type_alias', - 'code_snippet': code_snippet, - 'display_name': f"type {type_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1 + "name": type_name, + "type": "type", + "subtype": "type_alias", + "code_snippet": code_snippet, + "display_name": f"type {type_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, } - + def _extract_enum_entity(self, node, depth: int) -> dict: name_node = self._find_child_by_type(node, "identifier") if not name_node: return None - + enum_name = self._get_node_text(name_node) code_snippet = self._get_node_text(node) - + return { - 'name': enum_name, - 'type': 'enum', - 'subtype': 'enum', - 'code_snippet': code_snippet, - 'display_name': f"enum {enum_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1 + "name": enum_name, + "type": "enum", + "subtype": "enum", + "code_snippet": code_snippet, + "display_name": f"enum {enum_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, } - + def _extract_variable_entity(self, node, depth: int) -> dict: name_node = self._find_child_by_type(node, "identifier") if not name_node: return None - + var_name = self._get_node_text(name_node) code_snippet = self._get_node_text(node) - - has_function = self._find_child_by_type(node, "arrow_function") or self._find_child_by_type(node, "function_expression") - + + has_function = self._find_child_by_type(node, "arrow_function") or self._find_child_by_type( + node, "function_expression" + ) + return { - 'name': var_name, - 'type': 'variable', - 'subtype': 'variable', - 'code_snippet': code_snippet, - 'display_name': f"variable {var_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'has_function': bool(has_function) + "name": var_name, + "type": "variable", + "subtype": "variable", + "code_snippet": code_snippet, + "display_name": f"variable {var_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "has_function": bool(has_function), } - + def _extract_export_statement_entity(self, node, depth: int) -> dict: code_snippet = self._get_node_text(node) - + func_decl = self._find_child_by_type(node, "function_declaration") class_decl = self._find_child_by_type(node, "class_declaration") interface_decl = self._find_child_by_type(node, "interface_declaration") lexical_decl = self._find_child_by_type(node, "lexical_declaration") - + if func_decl: name_node = self._find_child_by_type(func_decl, "identifier") if name_node: func_name = self._get_node_text(name_node) return { - 'name': func_name, - 'type': 'function', - 'subtype': 'export_function', - 'code_snippet': code_snippet, - 'display_name': f"export function {func_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'parameters': self._extract_parameters(func_decl), - 'is_export': True + "name": func_name, + "type": "function", + "subtype": "export_function", + "code_snippet": code_snippet, + "display_name": f"export function {func_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "parameters": self._extract_parameters(func_decl), + "is_export": True, } elif class_decl: name_node = self._find_child_by_type(class_decl, "type_identifier") if name_node: class_name = self._get_node_text(name_node) return { - 'name': class_name, - 'type': 'class', - 'subtype': 'export_class', - 'code_snippet': code_snippet, - 'display_name': f"export class {class_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'base_classes': self._extract_inheritance(class_decl), - 'is_export': True + "name": class_name, + "type": "class", + "subtype": "export_class", + "code_snippet": code_snippet, + "display_name": f"export class {class_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "base_classes": self._extract_inheritance(class_decl), + "is_export": True, } elif interface_decl: name_node = self._find_child_by_type(interface_decl, "type_identifier") if name_node: interface_name = self._get_node_text(name_node) return { - 'name': interface_name, - 'type': 'interface', - 'subtype': 'export_interface', - 'code_snippet': code_snippet, - 'display_name': f"export interface {interface_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'base_classes': self._extract_inheritance(interface_decl), - 'is_export': True + "name": interface_name, + "type": "interface", + "subtype": "export_interface", + "code_snippet": code_snippet, + "display_name": f"export interface {interface_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "base_classes": self._extract_inheritance(interface_decl), + "is_export": True, } elif lexical_decl: var_declarator = self._find_child_by_type(lexical_decl, "variable_declarator") if var_declarator: name_node = self._find_child_by_type(var_declarator, "identifier") - func_expr = self._find_child_by_type(var_declarator, "arrow_function") or self._find_child_by_type(var_declarator, "function_expression") + func_expr = self._find_child_by_type( + var_declarator, "arrow_function" + ) or self._find_child_by_type(var_declarator, "function_expression") if name_node and func_expr: var_name = self._get_node_text(name_node) return { - 'name': var_name, - 'type': 'function', - 'subtype': 'export_arrow_function', - 'code_snippet': code_snippet, - 'display_name': f"export const {var_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'parameters': self._extract_parameters(func_expr), - 'is_export': True + "name": var_name, + "type": "function", + "subtype": "export_arrow_function", + "code_snippet": code_snippet, + "display_name": f"export const {var_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "parameters": self._extract_parameters(func_expr), + "is_export": True, } - + default_keyword = None call_expr = None for child in node.children: @@ -477,110 +494,112 @@ def _extract_export_statement_entity(self, node, depth: int) -> dict: default_keyword = child elif child.type == "call_expression": call_expr = child - + if default_keyword and call_expr: callee = call_expr.children[0] if call_expr.children else None if callee: callee_name = self._get_node_text(callee) return { - 'name': callee_name, - 'type': 'function', - 'subtype': 'export_default_call', - 'code_snippet': code_snippet, - 'display_name': f"export default {callee_name}(...)", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'parameters': [], - 'is_export': True + "name": callee_name, + "type": "function", + "subtype": "export_default_call", + "code_snippet": code_snippet, + "display_name": f"export default {callee_name}(...)", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "parameters": [], + "is_export": True, } - - return None - + + return None + def _extract_lexical_declaration_entity(self, node, depth: int) -> dict: """Extract lexical declaration entity (const/let).""" # Find the variable declarator var_declarator = self._find_child_by_type(node, "variable_declarator") if not var_declarator: return None - + name_node = self._find_child_by_type(var_declarator, "identifier") if not name_node: return None - + var_name = self._get_node_text(name_node) code_snippet = self._get_node_text(node) - + # Check declaration type (const/let) decl_type = "const" if "const" in code_snippet else "let" - - has_function = (self._find_child_by_type(var_declarator, "arrow_function") or - self._find_child_by_type(var_declarator, "function_expression")) - + + has_function = self._find_child_by_type( + var_declarator, "arrow_function" + ) or self._find_child_by_type(var_declarator, "function_expression") + return { - 'name': var_name, - 'type': 'variable', - 'subtype': f'{decl_type}_declaration', - 'code_snippet': code_snippet, - 'display_name': f"{decl_type} {var_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'has_function': bool(has_function), - 'declaration_type': decl_type + "name": var_name, + "type": "variable", + "subtype": f"{decl_type}_declaration", + "code_snippet": code_snippet, + "display_name": f"{decl_type} {var_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "has_function": bool(has_function), + "declaration_type": decl_type, } - + def _extract_variable_declaration_entity(self, node, depth: int) -> dict: var_declarator = self._find_child_by_type(node, "variable_declarator") if not var_declarator: return None - + name_node = self._find_child_by_type(var_declarator, "identifier") if not name_node: return None - + var_name = self._get_node_text(name_node) code_snippet = self._get_node_text(node) - - has_function = (self._find_child_by_type(var_declarator, "arrow_function") or - self._find_child_by_type(var_declarator, "function_expression")) - + + has_function = self._find_child_by_type( + var_declarator, "arrow_function" + ) or self._find_child_by_type(var_declarator, "function_expression") + return { - 'name': var_name, - 'type': 'variable', - 'subtype': 'var_declaration', - 'code_snippet': code_snippet, - 'display_name': f"var {var_name}", - 'start_line': node.start_point[0] + 1, - 'end_line': node.end_point[0] + 1, - 'has_function': bool(has_function), - 'declaration_type': 'var' + "name": var_name, + "type": "variable", + "subtype": "var_declaration", + "code_snippet": code_snippet, + "display_name": f"var {var_name}", + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "has_function": bool(has_function), + "declaration_type": "var", } - + def _create_node_from_entity(self, entity_data: dict) -> Optional[Node]: """Create Node object from entity data.""" try: - component_type = entity_data['type'] - name = entity_data['name'] - node_type = entity_data.get('subtype', entity_data['type']) - + component_type = entity_data["type"] + name = entity_data["name"] + node_type = entity_data.get("subtype", entity_data["type"]) + component_id = self._get_component_id(name) relative_path = self._get_relative_path() - + return Node( id=component_id, name=name, component_type=component_type, file_path=str(self.file_path), relative_path=relative_path, - source_code=entity_data['code_snippet'], - start_line=entity_data['start_line'], - end_line=entity_data['end_line'], + source_code=entity_data["code_snippet"], + start_line=entity_data["start_line"], + end_line=entity_data["end_line"], has_docstring=False, docstring="", - parameters=entity_data.get('parameters', []), + parameters=entity_data.get("parameters", []), node_type=node_type, - base_classes=entity_data.get('base_classes'), - class_name=entity_data.get('class_name'), - display_name=entity_data['display_name'], + base_classes=entity_data.get("base_classes"), + class_name=entity_data.get("class_name"), + display_name=entity_data["display_name"], component_id=component_id, language="typescript", qualified_name=name, @@ -603,7 +622,7 @@ def _extract_constructor_dependencies(self, class_node, class_name: str) -> None class_body = self._find_child_by_type(class_node, "class_body") if not class_body: return - + for child in class_body.children: if child.type == "method_definition": property_name = self._find_child_by_type(child, "property_identifier") @@ -635,7 +654,6 @@ def _extract_parameter_dependencies(self, formal_params, caller_name: str) -> No except Exception as e: logger.debug(f"Error extracting parameter dependencies: {e}") - def _get_module_path(self) -> str: if self.repo_path: try: @@ -644,13 +662,13 @@ def _get_module_path(self) -> str: rel_path = str(self.file_path) else: rel_path = str(self.file_path) - - for ext in ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']: + + for ext in [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]: if rel_path.endswith(ext): - rel_path = rel_path[:-len(ext)] + rel_path = rel_path[: -len(ext)] break - return rel_path.replace('/', '.').replace('\\', '.') - + return rel_path.replace("/", ".").replace("\\", ".") + def _get_relative_path(self) -> str: if self.repo_path: try: @@ -669,19 +687,19 @@ def _get_component_id(self, name: str) -> str: def _extract_inheritance(self, node) -> List[str]: """Extract inheritance/implementation relationships.""" base_classes = [] - + extends_clause = self._find_child_by_type(node, "extends_clause") if extends_clause: for child in extends_clause.children: if child.type in ["identifier", "type_identifier"]: base_classes.append(self._get_node_text(child)) - - implements_clause = self._find_child_by_type(node, "implements_clause") + + implements_clause = self._find_child_by_type(node, "implements_clause") if implements_clause: for child in implements_clause.children: if child.type in ["identifier", "type_identifier"]: base_classes.append(self._get_node_text(child)) - + return base_classes def _extract_parameters(self, node) -> List[str]: @@ -701,24 +719,25 @@ def _extract_parameters(self, node) -> List[str]: def _extract_all_relationships(self, node, all_entities: dict) -> None: self._traverse_for_relationships(node, all_entities, current_top_level=None) - def _traverse_for_relationships(self, node, all_entities: dict, current_top_level: str = None) -> None: + def _traverse_for_relationships( + self, node, all_entities: dict, current_top_level: str = None + ) -> None: if current_top_level is None or self._is_new_top_level(node): new_top_level = self._get_top_level_name(node) if new_top_level and new_top_level in self.top_level_nodes: current_top_level = new_top_level - if current_top_level: if node.type == "call_expression": self._extract_call_relationship(node, current_top_level, all_entities) elif node.type == "new_expression": self._extract_new_relationship(node, current_top_level, all_entities) - + elif node.type == "type_annotation": self._extract_type_relationship(node, current_top_level, all_entities) elif node.type == "type_arguments": self._extract_type_arguments_relationship(node, current_top_level, all_entities) - + elif node.type == "extends_clause": self._extract_inheritance_relationship(node, current_top_level, all_entities) elif node.type == "implements_clause": @@ -726,13 +745,18 @@ def _traverse_for_relationships(self, node, all_entities: dict, current_top_leve for child in node.children: self._traverse_for_relationships(child, all_entities, current_top_level) - + def _is_new_top_level(self, node) -> bool: return node.type in [ - "function_declaration", "generator_function_declaration", - "class_declaration", "abstract_class_declaration", - "interface_declaration", "type_alias_declaration", "enum_declaration", - "export_statement", "method_definition" + "function_declaration", + "generator_function_declaration", + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + "type_alias_declaration", + "enum_declaration", + "export_statement", + "method_definition", ] def _get_top_level_name(self, node) -> Optional[str]: @@ -746,8 +770,15 @@ def _get_top_level_name(self, node) -> Optional[str]: elif node.type in ["function_declaration", "generator_function_declaration"]: name_node = self._find_child_by_type(node, "identifier") result = self._get_node_text(name_node) if name_node else None - elif node.type in ["class_declaration", "abstract_class_declaration", "interface_declaration", "type_alias_declaration"]: - name_node = self._find_child_by_type(node, "type_identifier") or self._find_child_by_type(node, "identifier") + elif node.type in [ + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + "type_alias_declaration", + ]: + name_node = self._find_child_by_type( + node, "type_identifier" + ) or self._find_child_by_type(node, "identifier") result = self._get_node_text(name_node) if name_node else None elif node.type == "enum_declaration": name_node = self._find_child_by_type(node, "identifier") @@ -764,21 +795,21 @@ def _get_top_level_name(self, node) -> Optional[str]: func_decl = self._find_child_by_type(node, "function_declaration") class_decl = self._find_child_by_type(node, "class_declaration") lexical_decl = self._find_child_by_type(node, "lexical_declaration") - + if func_decl: name_node = self._find_child_by_type(func_decl, "identifier") if name_node: - result = self._get_node_text(name_node) + result = self._get_node_text(name_node) elif class_decl: name_node = self._find_child_by_type(class_decl, "type_identifier") if name_node: - result = self._get_node_text(name_node) + result = self._get_node_text(name_node) elif lexical_decl: var_declarator = self._find_child_by_type(lexical_decl, "variable_declarator") if var_declarator: name_node = self._find_child_by_type(var_declarator, "identifier") if name_node: - result = self._get_node_text(name_node) + result = self._get_node_text(name_node) else: result = "unnamed_export" elif node.type in ["lexical_declaration", "variable_declaration"]: @@ -791,7 +822,7 @@ def _get_top_level_name(self, node) -> Optional[str]: result = None else: result = None - + return result def _extract_call_relationship(self, node, caller_name: str, all_entities: dict) -> None: @@ -832,11 +863,15 @@ def _extract_call_relationship(self, node, caller_name: str, all_entities: dict) return if receiver_kind in ("this", "super"): - self._emit_instance_method_call(caller_name, tail, call_line, prefer_bases=(receiver_kind == "super")) + self._emit_instance_method_call( + caller_name, tail, call_line, prefer_bases=(receiver_kind == "super") + ) elif receiver_kind == "identifier": self._emit_receiver_method_call(node, caller_name, receiver_text, tail, call_line) elif receiver_kind == "chain": - self._add_relationship(caller_name, f"{receiver_text}.{tail}", call_line, resolved=False) + self._add_relationship( + caller_name, f"{receiver_text}.{tail}", call_line, resolved=False + ) else: # Composite or literal receiver: a builtin-prototype method on # an unknowable receiver can never resolve to a project @@ -854,7 +889,9 @@ def _member_call_parts(self, member_node) -> Optional[Tuple[str, Optional[str], pure dotted identifier chain), "composite" (call result, subscript, parenthesized, new expression), or "literal". """ - property_node = member_node.child_by_field_name("property") or self._find_child_by_type(member_node, "property_identifier") + property_node = member_node.child_by_field_name("property") or self._find_child_by_type( + member_node, "property_identifier" + ) tail = self._get_node_text(property_node) if property_node else "" obj = member_node.child_by_field_name("object") if obj is None and member_node.children: @@ -873,7 +910,16 @@ def _member_call_parts(self, member_node) -> Optional[Tuple[str, Optional[str], if chain: return ("chain", chain, tail) return ("composite", None, tail) - if obj.type in ("array", "string", "template_string", "number", "object", "regex", "true", "false"): + if obj.type in ( + "array", + "string", + "template_string", + "number", + "object", + "regex", + "true", + "false", + ): return ("literal", None, tail) return ("composite", None, tail) @@ -895,7 +941,9 @@ def _identifier_chain(self, node) -> Optional[str]: parts.append(self._get_node_text(current)) return ".".join(reversed(parts)) - def _emit_instance_method_call(self, caller_name: str, tail: str, call_line: int, prefer_bases: bool = False) -> None: + def _emit_instance_method_call( + self, caller_name: str, tail: str, call_line: int, prefer_bases: bool = False + ) -> None: """Resolve this.m()/super.m() against the enclosing class and its bases.""" class_name = caller_name.split(".")[0] if caller_name else None if not class_name: @@ -917,7 +965,9 @@ def _emit_instance_method_call(self, caller_name: str, tail: str, call_line: int target = candidates[0] if candidates else class_name self._add_relationship(caller_name, f"{target}.{tail}", call_line, resolved=False) - def _emit_receiver_method_call(self, call_node, caller_name: str, receiver: str, tail: str, call_line: int) -> None: + def _emit_receiver_method_call( + self, call_node, caller_name: str, receiver: str, tail: str, call_line: int + ) -> None: """Resolve recv.m() where recv is a simple identifier.""" if not receiver: return @@ -946,10 +996,14 @@ def _infer_identifier_type(self, call_node, identifier: str) -> Optional[str]: scopes = [] while scope: if scope.type in ( - "method_definition", "function_declaration", - "generator_function_declaration", "arrow_function", - "function_expression", "class_declaration", - "abstract_class_declaration", "program", + "method_definition", + "function_declaration", + "generator_function_declaration", + "arrow_function", + "function_expression", + "class_declaration", + "abstract_class_declaration", + "program", ): scopes.append(scope) scope = scope.parent @@ -994,7 +1048,7 @@ def _extract_new_relationship(self, node, caller_name: str, all_entities: dict) if node.children: constructor_node = None for child in node.children: - if child.type not in ['new', 'type_arguments', 'arguments']: + if child.type not in ["new", "type_arguments", "arguments"]: constructor_node = child break @@ -1003,7 +1057,9 @@ def _extract_new_relationship(self, node, caller_name: str, all_entities: dict) if constructor_name: resolved = constructor_name in self.top_level_nodes - self._add_relationship(caller_name, constructor_name, call_line, resolved=resolved) + self._add_relationship( + caller_name, constructor_name, call_line, resolved=resolved + ) except Exception as e: logger.debug(f"Error extracting new relationship: {e}") @@ -1012,33 +1068,35 @@ def _extract_type_relationship(self, node, caller_name: str, all_entities: dict) try: type_identifiers = [] self._find_all_type_identifiers(node, type_identifiers) - + call_line = node.start_point[0] + 1 - + for type_node in type_identifiers: type_name = self._get_node_text(type_node) - + if self._is_builtin_type(type_name): continue - + if type_name in all_entities: target_name = self._resolve_to_top_level(type_name, all_entities) if target_name and target_name in self.top_level_nodes: self._add_relationship(caller_name, target_name, call_line, resolved=True) else: self._add_relationship(caller_name, type_name, call_line, resolved=False) - + except Exception as e: logger.debug(f"Error extracting type relationship: {e}") - + def _find_all_type_identifiers(self, node, type_identifiers: list) -> None: if node.type == "type_identifier": type_identifiers.append(node) - + for child in node.children: self._find_all_type_identifiers(child, type_identifiers) - - def _extract_type_arguments_relationship(self, node, caller_name: str, all_entities: dict) -> None: + + def _extract_type_arguments_relationship( + self, node, caller_name: str, all_entities: dict + ) -> None: try: for child in node.children: if child.type == "type_identifier": @@ -1047,10 +1105,12 @@ def _extract_type_arguments_relationship(self, node, caller_name: str, all_entit target_name = self._resolve_to_top_level(type_name, all_entities) if target_name and target_name in self.top_level_nodes: call_line = node.start_point[0] + 1 - self._add_relationship(caller_name, target_name, call_line, resolved=True) + self._add_relationship( + caller_name, target_name, call_line, resolved=True + ) except Exception as e: logger.debug(f"Error extracting type arguments relationship: {e}") - + def _extract_inheritance_relationship(self, node, caller_name: str, all_entities: dict) -> None: """Extract inheritance/implementation relationships""" try: @@ -1061,23 +1121,32 @@ def _extract_inheritance_relationship(self, node, caller_name: str, all_entities target_name = self._resolve_to_top_level(base_name, all_entities) if target_name and target_name in self.top_level_nodes: call_line = node.start_point[0] + 1 - self._add_relationship(caller_name, target_name, call_line, resolved=True) + self._add_relationship( + caller_name, target_name, call_line, resolved=True + ) else: - self._add_relationship(caller_name, base_name, call_line=node.start_point[0] + 1, resolved=False) + self._add_relationship( + caller_name, + base_name, + call_line=node.start_point[0] + 1, + resolved=False, + ) except Exception as e: logger.debug(f"Error extracting inheritance relationship: {e}") def _resolve_to_top_level(self, entity_name: str, all_entities: dict) -> Optional[str]: if entity_name in self.top_level_nodes: return entity_name - + entity_data = all_entities.get(entity_name) - if entity_data and entity_data.get('depth', 0) > 2: + if entity_data and entity_data.get("depth", 0) > 2: return None - + return entity_name if entity_name in self.top_level_nodes else None - def _add_relationship(self, caller_name: str, callee_name: str, call_line: int, resolved: bool = False) -> None: + def _add_relationship( + self, caller_name: str, callee_name: str, call_line: int, resolved: bool = False + ) -> None: """Record one relationship. Resolved callees are component ids in this file; unresolved callees stay bare logical names so they never look like project components they are not.""" @@ -1101,7 +1170,16 @@ def _is_builtin_type(self, name: str) -> bool: """Check if type name is a TypeScript/JavaScript built-in type.""" builtin_types = { # Primitive types - "string", "number", "boolean", "object", "undefined", "null", "void", "never", "any", "unknown" + "string", + "number", + "boolean", + "object", + "undefined", + "null", + "void", + "never", + "any", + "unknown", } return name in builtin_types @@ -1121,7 +1199,6 @@ def _get_node_text(self, node) -> str: return self.content.encode("utf8")[start_byte:end_byte].decode("utf8") - def analyze_typescript_file_treesitter( file_path: str, content: str, repo_path: str = None ) -> Tuple[List[Node], List[CallRelationship]]: @@ -1135,4 +1212,4 @@ def analyze_typescript_file_treesitter( return analyzer.nodes, analyzer.call_relationships except Exception as e: logger.error(f"Error in tree-sitter TS analysis for {file_path}: {e}", exc_info=True) - return [], [] \ No newline at end of file + return [], [] diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index 916fff5..d634910 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -1,11 +1,8 @@ import os import json import logging -import argparse -from dataclasses import dataclass, field -from typing import Dict, List, Set, Tuple, Optional, Any, Union +from typing import Dict, List, Set, Optional from pathlib import Path -import re from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService from codewiki.src.be.dependency_analyzer.models.core import Node @@ -18,11 +15,13 @@ class DependencyParser: """Parser for extracting code components from multi-language repositories.""" - - def __init__(self, repo_path: str, include_patterns: List[str] = None, exclude_patterns: List[str] = None): + + def __init__( + self, repo_path: str, include_patterns: List[str] = None, exclude_patterns: List[str] = None + ): """ Initialize the dependency parser. - + Args: repo_path: Path to the repository include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) @@ -37,8 +36,9 @@ def __init__(self, repo_path: str, include_patterns: List[str] = None, exclude_p self.analysis_service = AnalysisService() - def parse_repository(self, filtered_folders: List[str] = None, - skip_file_paths: Optional[set] = None) -> Dict[str, Node]: + def parse_repository( + self, filtered_folders: List[str] = None, skip_file_paths: Optional[set] = None + ) -> Dict[str, Node]: logger.debug(f"Parsing repository at {self.repo_path}") # Log custom patterns if set @@ -50,7 +50,7 @@ def parse_repository(self, filtered_folders: List[str] = None, structure_result = self.analysis_service._analyze_structure( self.repo_path, include_patterns=self.include_patterns, - exclude_patterns=self.exclude_patterns + exclude_patterns=self.exclude_patterns, ) call_graph_result = self.analysis_service._analyze_call_graph( @@ -63,25 +63,27 @@ def parse_repository(self, filtered_folders: List[str] = None, self.routes = call_graph_result.get("routes", []) self._build_components_from_analysis(call_graph_result) - + logger.debug(f"Found {len(self.components)} components across {len(self.modules)} modules") return self.components - + def _build_components_from_analysis(self, call_graph_result: Dict): functions = call_graph_result.get("functions", []) relationships = call_graph_result.get("relationships", []) - + component_id_mapping = {} - + for func_dict in functions: component_id = func_dict.get("id", "") if not component_id: continue - + node = Node( id=component_id, name=func_dict.get("name", ""), - component_type=func_dict.get("component_type", func_dict.get("node_type", "function")), + component_type=func_dict.get( + "component_type", func_dict.get("node_type", "function") + ), file_path=func_dict.get("file_path", ""), relative_path=func_dict.get("relative_path", ""), source_code=func_dict.get("source_code", func_dict.get("code_snippet", "")), @@ -96,16 +98,16 @@ def _build_components_from_analysis(self, call_graph_result: Dict): display_name=func_dict.get("display_name", ""), component_id=component_id, language=func_dict.get("language") - or CODE_EXTENSIONS.get(Path(func_dict.get("file_path", "")).suffix.lower()), + or CODE_EXTENSIONS.get(Path(func_dict.get("file_path", "")).suffix.lower()), ) - + self.components[component_id] = node - + component_id_mapping[component_id] = component_id legacy_id = f"{func_dict.get('file_path', '')}:{func_dict.get('name', '')}" if legacy_id and legacy_id != component_id: component_id_mapping[legacy_id] = component_id - + if "::" in component_id: file_path_part = component_id.split("::")[0] if file_path_part: @@ -115,7 +117,7 @@ def _build_components_from_analysis(self, call_graph_result: Dict): module_path = ".".join(module_parts) if module_path: self.modules.add(module_path) - + processed_relationships = 0 # Build name→id index for O(1) fallback lookup instead of O(C) linear scan name_to_id: Dict[str, str] = {} @@ -126,52 +128,80 @@ def _build_components_from_analysis(self, call_graph_result: Dict): for rel_dict in relationships: caller_id = rel_dict.get("caller", "") callee_id = rel_dict.get("callee", "") - is_resolved = rel_dict.get("is_resolved", False) - + rel_dict.get("is_resolved", False) + caller_component_id = component_id_mapping.get(caller_id) - + callee_component_id = component_id_mapping.get(callee_id) if not callee_component_id: callee_component_id = name_to_id.get(callee_id) - + if caller_component_id and caller_component_id in self.components: if callee_component_id: self.components[caller_component_id].depends_on.add(callee_component_id) processed_relationships += 1 - + def _determine_component_type(self, func_dict: Dict) -> str: if func_dict.get("is_method", False): return "method" - + node_type = func_dict.get("node_type", "") - if node_type in ["class", "interface", "struct", "enum", "record", "abstract class", "annotation", "delegate"]: + if node_type in [ + "class", + "interface", + "struct", + "enum", + "record", + "abstract class", + "annotation", + "delegate", + ]: return node_type - + return "function" - + def _file_to_module_path(self, file_path: str) -> str: path = file_path - extensions = ['.py', '.js', '.ts', '.java', '.cs', '.cpp', '.hpp', '.h', '.c', '.tsx', '.jsx', '.cc', '.mjs', '.cxx', '.cc', '.cjs', '.kt', '.kts'] + extensions = [ + ".py", + ".js", + ".ts", + ".java", + ".cs", + ".cpp", + ".hpp", + ".h", + ".c", + ".tsx", + ".jsx", + ".cc", + ".mjs", + ".cxx", + ".cc", + ".cjs", + ".kt", + ".kts", + ] for ext in extensions: if path.endswith(ext): - path = path[:-len(ext)] + path = path[: -len(ext)] break return path.replace(os.path.sep, ".") - + def save_dependency_graph(self, output_path: str): result = {} for component_id, component in self.components.items(): component_dict = component.model_dump() - if 'depends_on' in component_dict and isinstance(component_dict['depends_on'], set): - component_dict['depends_on'] = list(component_dict['depends_on']) + if "depends_on" in component_dict and isinstance(component_dict["depends_on"], set): + component_dict["depends_on"] = list(component_dict["depends_on"]) result[component_id] = component_dict - + dir_name = os.path.dirname(output_path) if dir_name: os.makedirs(dir_name, exist_ok=True) - - with open(output_path, 'w', encoding='utf-8') as f: + + with open(output_path, "w", encoding="utf-8") as f: json.dump(result, f, indent=2, ensure_ascii=False) - + logger.debug(f"Saved {len(self.components)} components to {output_path}") return result diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index e86f490..7820b01 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -2,20 +2,26 @@ import os from codewiki.src.config import Config from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser -from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes +from codewiki.src.be.dependency_analyzer.topo_sort import ( + build_graph_from_components, + get_leaf_nodes, +) from codewiki.src.utils import file_manager import logging + logger = logging.getLogger(__name__) class DependencyGraphBuilder: """Handles dependency analysis and graph building.""" - + def __init__(self, config: Config): self.config = config - - def build_dependency_graph(self, skip_file_paths: set = None) -> tuple[Dict[str, Any], List[str], List[Dict]]: + + def build_dependency_graph( + self, skip_file_paths: set = None + ) -> tuple[Dict[str, Any], List[str], List[Dict]]: """ Build and save dependency graph, returning components, leaf nodes, and routes. @@ -30,24 +36,22 @@ def build_dependency_graph(self, skip_file_paths: set = None) -> tuple[Dict[str, # Prepare dependency graph path repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) + sanitized_repo_name = "".join(c if c.isalnum() else "_" for c in repo_name) dependency_graph_path = os.path.join( - self.config.dependency_graph_dir, - f"{sanitized_repo_name}_dependency_graph.json" + self.config.dependency_graph_dir, f"{sanitized_repo_name}_dependency_graph.json" ) - filtered_folders_path = os.path.join( - self.config.dependency_graph_dir, - f"{sanitized_repo_name}_filtered_folders.json" + os.path.join( + self.config.dependency_graph_dir, f"{sanitized_repo_name}_filtered_folders.json" ) # Get custom include/exclude patterns from config include_patterns = self.config.include_patterns if self.config.include_patterns else None exclude_patterns = self.config.exclude_patterns if self.config.exclude_patterns else None - + parser = DependencyParser( self.config.repo_path, include_patterns=include_patterns, - exclude_patterns=exclude_patterns + exclude_patterns=exclude_patterns, ) filtered_folders = None @@ -62,37 +66,44 @@ def build_dependency_graph(self, skip_file_paths: set = None) -> tuple[Dict[str, # Parse repository components = parser.parse_repository(filtered_folders, skip_file_paths=skip_file_paths) - + # Save dependency graph parser.save_dependency_graph(dependency_graph_path) - + # Build graph for traversal graph = build_graph_from_components(components) - + # Get leaf nodes leaf_nodes = get_leaf_nodes(graph, components) # check if leaf_nodes are in components, only keep the ones that are in components # and type is one of the following: class, interface, struct (or function for C-based projects) - + # Determine if we should include functions based on available component types available_types = set() for comp in components.values(): available_types.add(comp.component_type) - + # Valid types for leaf nodes - include functions for C-based codebases valid_types = {"class", "interface", "struct"} # If no classes/interfaces/structs are found, include functions if not available_types.intersection(valid_types): valid_types.add("function") - + keep_leaf_nodes = [] for leaf_node in leaf_nodes: # Skip any leaf nodes that are clearly error strings or invalid identifiers - if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']): + if ( + not isinstance(leaf_node, str) + or leaf_node.strip() == "" + or any( + err_keyword in leaf_node.lower() + for err_keyword in ["error", "exception", "failed", "invalid"] + ) + ): logger.warning(f"Skipping invalid leaf node identifier: '{leaf_node}'") continue - + if leaf_node in components: if components[leaf_node].component_type in valid_types: keep_leaf_nodes.append(leaf_node) @@ -103,6 +114,6 @@ def build_dependency_graph(self, skip_file_paths: set = None) -> tuple[Dict[str, logger.warning(f"Leaf node {leaf_node} not found in components, removing it") # Collect cross-service routes from the parser - routes = getattr(parser, 'routes', []) + routes = getattr(parser, "routes", []) - return components, keep_leaf_nodes, routes \ No newline at end of file + return components, keep_leaf_nodes, routes diff --git a/codewiki/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py index 6174e05..6f75ecd 100644 --- a/codewiki/src/be/dependency_analyzer/models/core.py +++ b/codewiki/src/be/dependency_analyzer/models/core.py @@ -1,35 +1,33 @@ from pydantic import BaseModel -from typing import List, Optional, Dict, Any, Set -from datetime import datetime - +from typing import List, Optional, Set class Node(BaseModel): id: str name: str - + component_type: str - + file_path: str - + relative_path: str - + depends_on: Set[str] = set() - + source_code: Optional[str] = None - + start_line: int = 0 end_line: int = 0 - + has_docstring: bool = False - + docstring: str = "" - + parameters: Optional[List[str]] = None - node_type: Optional[str] = None + node_type: Optional[str] = None base_classes: Optional[List[str]] = None @@ -63,5 +61,5 @@ class Repository(BaseModel): name: str clone_path: str - + analysis_id: str diff --git a/codewiki/src/be/dependency_analyzer/models/cross_service.py b/codewiki/src/be/dependency_analyzer/models/cross_service.py index 0b350c9..c30a9d8 100644 --- a/codewiki/src/be/dependency_analyzer/models/cross_service.py +++ b/codewiki/src/be/dependency_analyzer/models/cross_service.py @@ -3,6 +3,7 @@ Route nodes, cross-service links, and workspace topology for inter-repository API call detection and matching. """ + from __future__ import annotations from enum import Enum @@ -19,23 +20,23 @@ class RouteProtocol(str, Enum): class RouteRole(str, Enum): - SERVER = "server" # 服务端路由处理器 - CLIENT = "client" # 客户端 HTTP / MQ 调用 + SERVER = "server" # 服务端路由处理器 + CLIENT = "client" # 客户端 HTTP / MQ 调用 class RouteNode(BaseModel): """A protocol-agnostic rendezvous point (borrowed from CBM).""" - route_key: str # "__route__POST__/api/orders/{}" + route_key: str # "__route__POST__/api/orders/{}" protocol: RouteProtocol = RouteProtocol.HTTP - method: Optional[str] = None # GET, POST, PUT, DELETE, PATCH … - path: str = "" # 规范化后的路径 + method: Optional[str] = None # GET, POST, PUT, DELETE, PATCH … + path: str = "" # 规范化后的路径 role: RouteRole = RouteRole.SERVER - component_id: str = "" # 关联的 Node ID + component_id: str = "" # 关联的 Node ID repo_name: str = "" file_path: str = "" line_number: int = 0 - framework: Optional[str] = None # fastapi, spring, express … + framework: Optional[str] = None # fastapi, spring, express … extra: Dict = Field(default_factory=dict) # 协议专属扩展字段 @@ -52,7 +53,7 @@ class CrossServiceLink(BaseModel): server_repo: str = "" server_component_id: str = "" server_function: str = "" - confidence: float = 1.0 # 1.0 = 精确匹配, <1 = 模糊匹配 + confidence: float = 1.0 # 1.0 = 精确匹配, <1 = 模糊匹配 class WorkspaceTopology(BaseModel): diff --git a/codewiki/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py index ccc4adf..ecc88ca 100644 --- a/codewiki/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -19,11 +19,11 @@ def detect_cycles(graph: Dict[str, Set[str]]) -> List[List[str]]: """ Detect cycles in a dependency graph using Tarjan's algorithm to find strongly connected components. - + Args: graph: A dependency graph represented as adjacency lists (node -> set of dependencies) - + Returns: A list of lists, where each inner list contains the nodes in a cycle """ @@ -34,7 +34,7 @@ def detect_cycles(graph: Dict[str, Set[str]]) -> List[List[str]]: onstack = set() # nodes currently on the stack stack = [] # stack of nodes result = [] # list of cycles (strongly connected components) - + def strongconnect(node): # Set the depth index for node index[node] = index_counter[0] @@ -42,7 +42,7 @@ def strongconnect(node): index_counter[0] += 1 stack.append(node) onstack.add(node) - + # Consider successors for successor in graph.get(node, set()): if successor not in index: @@ -52,7 +52,7 @@ def strongconnect(node): elif successor in onstack: # Successor is on the stack and hence in the current SCC lowlink[node] = min(lowlink[node], index[successor]) - + # If node is a root node, pop the stack and generate an SCC if lowlink[node] == index[node]: # Start a new strongly connected component @@ -63,46 +63,47 @@ def strongconnect(node): scc.append(successor) if successor == node: break - + # Only include SCCs with more than one node (actual cycles) if len(scc) > 1: result.append(scc) - + # Visit each node for node in graph: if node not in index: strongconnect(node) - + return result + def resolve_cycles(graph: Dict[str, Set[str]]) -> Dict[str, Set[str]]: """ Resolve cycles in a dependency graph by identifying strongly connected components and breaking cycles. - + Args: graph: A dependency graph represented as adjacency lists (node -> set of dependencies) - + Returns: A new acyclic graph with the same nodes but with cycles broken """ # Detect cycles (SCCs) cycles = detect_cycles(graph) - + if not cycles: logger.debug("No cycles detected in the dependency graph") return graph - + logger.debug(f"Detected {len(cycles)} cycles in the dependency graph") - + # Create a copy of the graph to modify new_graph = {node: deps.copy() for node, deps in graph.items()} - + # Process each cycle for i, cycle in enumerate(cycles): - logger.debug(f"Cycle {i+1}: {' -> '.join(cycle)}") - + logger.debug(f"Cycle {i + 1}: {' -> '.join(cycle)}") + # Strategy: Break the cycle by removing the "weakest" dependency # Here, we just arbitrarily remove the last edge to make the graph acyclic # In a real-world scenario, you might use heuristics to determine which edge to break @@ -110,31 +111,32 @@ def resolve_cycles(graph: Dict[str, Set[str]]) -> Dict[str, Set[str]]: for j in range(len(cycle) - 1): current = cycle[j] next_node = cycle[j + 1] - + if next_node in new_graph[current]: logger.debug(f"Breaking cycle by removing dependency: {current} -> {next_node}") new_graph[current].remove(next_node) break - + return new_graph + def topological_sort(graph: Dict[str, Set[str]]) -> List[str]: """ Perform a topological sort on a dependency graph. - + Args: graph: A dependency graph represented as adjacency lists (node -> set of dependencies) - + Returns: A list of nodes in topological order (dependencies first) """ # First, check for and resolve cycles acyclic_graph = resolve_cycles(graph) - + # Initialize in-degree counter for all nodes in_degree = {node: 0 for node in acyclic_graph} - + # Build reverse adjacency list for O(V+E) Kahn's algorithm: # reverse_adj[node] = node's dependencies, so processing node only # touches its direct deps instead of scanning all V nodes (O(V×E) → O(V+E)). @@ -144,157 +146,157 @@ def topological_sort(graph: Dict[str, Set[str]]) -> List[str]: if dep in in_degree: in_degree[dep] += 1 reverse_adj[node].append(dep) - + # Queue of nodes with no dependencies (in-degree of 0) queue = deque([node for node, degree in in_degree.items() if degree == 0]) - + # Result list to store the topological order result = [] - + # Process nodes in topological order while queue: node = queue.popleft() result.append(node) - + # Reduce in-degree only for nodes that actually depend on current node for dependent in reverse_adj.get(node, []): in_degree[dependent] -= 1 if in_degree[dependent] == 0: queue.append(dependent) - + # Check if the sort was successful (all nodes included) if len(result) != len(acyclic_graph): logger.warning("Topological sort failed: graph has cycles that weren't resolved") # Return all nodes in some order to avoid breaking the process return list(acyclic_graph.keys()) - + # Reverse the result to get dependencies first return result[::-1] + def dependency_first_dfs(graph: Dict[str, Set[str]]) -> List[str]: """ Perform a depth-first traversal of the dependency graph, starting from root nodes that have no dependencies. - + The graph uses natural dependency direction: - If A depends on B, the graph has an edge A → B - This means an edge from X to Y represents "X depends on Y" - Root nodes (nodes with no incoming edges/dependencies) are processed first, followed by nodes that depend on them - + Args: graph: A dependency graph with natural direction (A→B if A depends on B) - + Returns: A list of nodes in an order where dependencies come before their dependents """ # First, resolve cycles to ensure we have a DAG acyclic_graph = resolve_cycles(graph) - + # Find root nodes (nodes with no dependencies) root_nodes = [] # Create a reverse graph to easily check if a node has incoming edges has_incoming_edge = {node: False for node in acyclic_graph} - + for node, deps in acyclic_graph.items(): for dep in deps: has_incoming_edge[dep] = True - + # Nodes with no incoming edges are root nodes for node in acyclic_graph: if not has_incoming_edge.get(node, False) and node in acyclic_graph: root_nodes.append(node) - + if not root_nodes: logger.warning("No root nodes found in the graph, using arbitrary starting point") root_nodes = list(acyclic_graph.keys())[:1] # Use the first node as starting point - + # Track visited nodes visited = set() result = [] - + # DFS function that processes dependencies first def dfs(node): if node in visited: return visited.add(node) - + # Visit all dependencies first for dep in sorted(acyclic_graph.get(node, set())): dfs(dep) - + # Add this node to the result after all its dependencies result.append(node) - + # Start DFS from each root node for root in sorted(root_nodes): dfs(root) - + # Check if all nodes were visited if len(result) != len(acyclic_graph): # Some nodes weren't visited - try to visit remaining nodes for node in sorted(acyclic_graph.keys()): if node not in visited: dfs(node) - + return result + def build_graph_from_components(components: Dict[str, Any]) -> Dict[str, Set[str]]: """ Build a dependency graph from a collection of code components. - + The graph uses the natural dependency direction: - If A depends on B, we create an edge A → B - This means an edge from node X to node Y represents "X depends on Y" - Root nodes (nodes with no dependencies) are components that don't depend on anything - + Args: components: A dictionary of code components, where each component has a 'depends_on' attribute - + Returns: A dependency graph with natural dependency direction """ graph = {} - + for comp_id, component in components.items(): # Initialize the node's adjacency list if comp_id not in graph: graph[comp_id] = set() - + # Add dependencies for dep_id in component.depends_on: # Only include dependencies that are actual components in our repository if dep_id in components: graph[comp_id].add(dep_id) - - return graph + + return graph def get_leaf_nodes(graph: Dict[str, Set[str]], components: Dict[str, Node]) -> List[str]: """ Find leaf nodes (nodes that no other nodes depend on) and build dependency trees showing the full dependency chain from each leaf back to the ultimate dependencies. - + The graph uses natural dependency direction: - If A depends on B, the graph has an edge A → B - Leaf nodes are nodes that appear in no other node's dependency set - Each tree shows the dependency chain: leaf → its dependencies → their dependencies, etc. - + Args: graph: A dependency graph with natural direction (A→B if A depends on B) - + Returns: A list of leaf nodes """ # First, resolve cycles to ensure we have a DAG acyclic_graph = resolve_cycles(graph) - + # Find leaf nodes (nodes that no other nodes depend on) leaf_nodes = set(acyclic_graph.keys()) - - def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes = set() for node in leaf_nodes: @@ -303,15 +305,15 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes.add(node.replace(".__init__", "")) else: concise_leaf_nodes.add(node) - + keep_leaf_nodes = [] - + # Determine if we should include functions based on available component types # For C-based projects, we need to include functions since they don't have classes available_types = set() for comp in components.values(): available_types.add(comp.component_type) - + # Valid types for leaf nodes - include functions for C-based codebases valid_types = {"class", "interface", "struct"} # If no classes/interfaces/structs are found, include functions @@ -320,10 +322,17 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: for leaf_node in leaf_nodes: # Skip any leaf nodes that are clearly error strings or invalid identifiers - if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']): + if ( + not isinstance(leaf_node, str) + or leaf_node.strip() == "" + or any( + err_keyword in leaf_node.lower() + for err_keyword in ["error", "exception", "failed", "invalid"] + ) + ): logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'") continue - + if leaf_node in components: if components[leaf_node].component_type in valid_types: keep_leaf_nodes.append(leaf_node) @@ -344,21 +353,23 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: logger.warning( "Leaf nodes (%d) exceed threshold (%d, %.0f%% of %d components); " "pruning nodes that are dependencies of other nodes", - len(concise_leaf_nodes), leaf_threshold, - leaf_threshold / max(len(components), 1) * 100, len(components), + len(concise_leaf_nodes), + leaf_threshold, + leaf_threshold / max(len(components), 1) * 100, + len(components), ) # Remove nodes that are dependencies of other nodes for node, deps in acyclic_graph.items(): for dep in deps: leaf_nodes.discard(dep) - + concise_leaf_nodes = concise_node(leaf_nodes) logger.info("After pruning: %d leaf nodes remain", len(concise_leaf_nodes)) - + if not leaf_nodes: logger.warning("No leaf nodes found in the graph") return [] - + return concise_leaf_nodes @@ -366,6 +377,7 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: # Transitive impact analysis # --------------------------------------------------------------------------- + def build_reverse_graph(graph: Dict[str, Set[str]]) -> Dict[str, Set[str]]: """Build the reverse adjacency list. @@ -499,13 +511,17 @@ def transitive_impact( """ if direction == "both": fwd = transitive_impact( - graph, start_nodes, - max_depth=max_depth, direction="depends_on", + graph, + start_nodes, + max_depth=max_depth, + direction="depends_on", track_paths=track_paths, ) rev = transitive_impact( - graph, start_nodes, - max_depth=max_depth, direction="depended_by", + graph, + start_nodes, + max_depth=max_depth, + direction="depended_by", track_paths=track_paths, ) # Merge: keep the smaller depth when a node appears in both @@ -594,4 +610,4 @@ def resolve_files_to_components( ): matched.append(comp_id) break - return matched \ No newline at end of file + return matched diff --git a/codewiki/src/be/dependency_analyzer/utils/external_symbols.py b/codewiki/src/be/dependency_analyzer/utils/external_symbols.py index 1ab2158..ca6e443 100644 --- a/codewiki/src/be/dependency_analyzer/utils/external_symbols.py +++ b/codewiki/src/be/dependency_analyzer/utils/external_symbols.py @@ -318,40 +318,127 @@ # audit lesson, prefer resolving over enumerating, so this set stays small. CSHARP_EXTERNAL_SYMBOLS = { # System core - "Object", "String", "Console", "Convert", "Math", "Array", "Enum", - "Attribute", "Type", "Activator", "Environment", "GC", "Random", "Uri", - "Guid", "DateTime", "DateTimeOffset", "TimeSpan", "Nullable", "Lazy", - "Tuple", "ValueTuple", "Span", "Memory", "ReadOnlySpan", "BitConverter", - "Buffer", "Version", "Index", "Range", + "Object", + "String", + "Console", + "Convert", + "Math", + "Array", + "Enum", + "Attribute", + "Type", + "Activator", + "Environment", + "GC", + "Random", + "Uri", + "Guid", + "DateTime", + "DateTimeOffset", + "TimeSpan", + "Nullable", + "Lazy", + "Tuple", + "ValueTuple", + "Span", + "Memory", + "ReadOnlySpan", + "BitConverter", + "Buffer", + "Version", + "Index", + "Range", # Delegates / common interfaces - "Action", "Func", "Predicate", "Comparison", "EventHandler", "EventArgs", - "IDisposable", "IAsyncDisposable", "IComparable", "IEquatable", - "IFormattable", "IFormatProvider", "ICloneable", + "Action", + "Func", + "Predicate", + "Comparison", + "EventHandler", + "EventArgs", + "IDisposable", + "IAsyncDisposable", + "IComparable", + "IEquatable", + "IFormattable", + "IFormatProvider", + "ICloneable", # Exceptions - "Exception", "ArgumentException", "ArgumentNullException", - "ArgumentOutOfRangeException", "InvalidOperationException", - "NotImplementedException", "NotSupportedException", "NullReferenceException", - "IndexOutOfRangeException", "FormatException", "OverflowException", - "ObjectDisposedException", "OperationCanceledException", - "AggregateException", "TimeoutException", "InvalidCastException", - "KeyNotFoundException", "ApplicationException", "SystemException", + "Exception", + "ArgumentException", + "ArgumentNullException", + "ArgumentOutOfRangeException", + "InvalidOperationException", + "NotImplementedException", + "NotSupportedException", + "NullReferenceException", + "IndexOutOfRangeException", + "FormatException", + "OverflowException", + "ObjectDisposedException", + "OperationCanceledException", + "AggregateException", + "TimeoutException", + "InvalidCastException", + "KeyNotFoundException", + "ApplicationException", + "SystemException", # System.Collections.Generic - "List", "Dictionary", "HashSet", "SortedSet", "SortedDictionary", - "SortedList", "Queue", "Stack", "LinkedList", "KeyValuePair", - "IEnumerable", "IEnumerator", "ICollection", "IList", "IDictionary", - "ISet", "IReadOnlyList", "IReadOnlyCollection", "IReadOnlyDictionary", - "Comparer", "EqualityComparer", + "List", + "Dictionary", + "HashSet", + "SortedSet", + "SortedDictionary", + "SortedList", + "Queue", + "Stack", + "LinkedList", + "KeyValuePair", + "IEnumerable", + "IEnumerator", + "ICollection", + "IList", + "IDictionary", + "ISet", + "IReadOnlyList", + "IReadOnlyCollection", + "IReadOnlyDictionary", + "Comparer", + "EqualityComparer", # System.IO - "File", "Directory", "Path", "Stream", "StreamReader", "StreamWriter", - "MemoryStream", "FileStream", "TextReader", "TextWriter", "BinaryReader", - "BinaryWriter", "FileInfo", "DirectoryInfo", + "File", + "Directory", + "Path", + "Stream", + "StreamReader", + "StreamWriter", + "MemoryStream", + "FileStream", + "TextReader", + "TextWriter", + "BinaryReader", + "BinaryWriter", + "FileInfo", + "DirectoryInfo", # System.Linq - "Enumerable", "IQueryable", "IGrouping", "IOrderedEnumerable", + "Enumerable", + "IQueryable", + "IGrouping", + "IOrderedEnumerable", # System.Net.Http - "HttpClient", "HttpRequestMessage", "HttpResponseMessage", "HttpContent", + "HttpClient", + "HttpRequestMessage", + "HttpResponseMessage", + "HttpContent", # System.Threading / Tasks - "Thread", "Interlocked", "Monitor", "Mutex", "SemaphoreSlim", - "CancellationToken", "CancellationTokenSource", "Task", "ValueTask", + "Thread", + "Interlocked", + "Monitor", + "Mutex", + "SemaphoreSlim", + "CancellationToken", + "CancellationTokenSource", + "Task", + "ValueTask", "TaskCompletionSource", # System.Text (not implicit but ubiquitous) "StringBuilder", diff --git a/codewiki/src/be/dependency_analyzer/utils/logging_config.py b/codewiki/src/be/dependency_analyzer/utils/logging_config.py index 4ead7f9..7740aaf 100644 --- a/codewiki/src/be/dependency_analyzer/utils/logging_config.py +++ b/codewiki/src/be/dependency_analyzer/utils/logging_config.py @@ -9,17 +9,17 @@ - WARNING: Yellow - Warning messages that need attention - ERROR: Red - Error messages - CRITICAL: Bright Red - Critical issues requiring immediate attention - + Additional Colors: - Timestamp: Blue - Module Name: Magenta - + Usage: from codewiki.src.be.dependency_analyzer.utils.logging_config import setup_logging - + # Setup colored logging for the entire application setup_logging(level=logging.INFO) - + # Or setup for a specific module logger = setup_module_logging('my_module', level=logging.DEBUG) """ @@ -34,78 +34,80 @@ class ColoredFormatter(logging.Formatter): """Custom formatter with colored output for better readability. - + This formatter adds colors to different log levels and components: - Log levels are colored based on severity - Timestamps are shown in blue - Module names are shown in magenta - Messages are shown in the default terminal color """ - + # Define colors for different log levels COLORS = { - 'DEBUG': Fore.BLUE, - 'INFO': Fore.CYAN, - 'WARNING': Fore.YELLOW, - 'ERROR': Fore.RED, - 'CRITICAL': Fore.RED + Style.BRIGHT, + "DEBUG": Fore.BLUE, + "INFO": Fore.CYAN, + "WARNING": Fore.YELLOW, + "ERROR": Fore.RED, + "CRITICAL": Fore.RED + Style.BRIGHT, } - + # Define colors for different components COMPONENT_COLORS = { - 'timestamp': Fore.BLUE, - 'module': Fore.MAGENTA, - 'reset': Style.RESET_ALL, + "timestamp": Fore.BLUE, + "module": Fore.MAGENTA, + "reset": Style.RESET_ALL, } - + def format(self, record): """Format log record with colors.""" # Get the color for this log level - level_color = self.COLORS.get(record.levelname, '') - + level_color = self.COLORS.get(record.levelname, "") + # Format timestamp - timestamp = self.formatTime(record, '%H:%M:%S') - colored_timestamp = f"{self.COMPONENT_COLORS['timestamp']}[{timestamp}]{self.COMPONENT_COLORS['reset']}" - + timestamp = self.formatTime(record, "%H:%M:%S") + colored_timestamp = ( + f"{self.COMPONENT_COLORS['timestamp']}[{timestamp}]{self.COMPONENT_COLORS['reset']}" + ) + # Format log level with color colored_level = f"{level_color}{record.levelname:8}{self.COMPONENT_COLORS['reset']}" - + # Format the message with the same color as the log level message = record.getMessage() colored_message = f"{level_color}{message}{self.COMPONENT_COLORS['reset']}" - + # Combine all parts (without module name column) log_line = f"{colored_timestamp} {colored_level} {colored_message}" - + # Handle exceptions if record.exc_info: log_line += "\n" + self.formatException(record.exc_info) - + return log_line def setup_logging(level=logging.INFO): """ Set up logging configuration with colored output. - + Args: level: Logging level (default: logging.INFO) """ # Create console handler console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(level) - + # Set colored formatter colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + # Configure root logger root_logger = logging.getLogger() root_logger.setLevel(level) - + # Remove existing handlers to avoid duplicates root_logger.handlers.clear() - + # Add our console handler root_logger.addHandler(console_handler) @@ -113,31 +115,29 @@ def setup_logging(level=logging.INFO): def setup_module_logging(module_name: str, level=logging.INFO): """ Set up logging for a specific module with colored output. - + Args: module_name: Name of the module to configure logging for level: Logging level (default: logging.INFO) """ logger = logging.getLogger(module_name) logger.setLevel(level) - + # Create console handler console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(level) - + # Set colored formatter colored_formatter = ColoredFormatter() console_handler.setFormatter(colored_formatter) - + # Remove existing handlers logger.handlers.clear() - + # Add console handler logger.addHandler(console_handler) - + # Prevent propagation to avoid duplicate logs logger.propagate = False - - return logger - + return logger diff --git a/codewiki/src/be/dependency_analyzer/utils/path_canonicalizer.py b/codewiki/src/be/dependency_analyzer/utils/path_canonicalizer.py index b58c29e..682c2c3 100644 --- a/codewiki/src/be/dependency_analyzer/utils/path_canonicalizer.py +++ b/codewiki/src/be/dependency_analyzer/utils/path_canonicalizer.py @@ -4,6 +4,7 @@ All parameter placeholders are unified to ``{}`` so that different frameworks can match against the same Route key. """ + from __future__ import annotations import re diff --git a/codewiki/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py index 1ce855a..5aa511d 100644 --- a/codewiki/src/be/dependency_analyzer/utils/patterns.py +++ b/codewiki/src/be/dependency_analyzer/utils/patterns.py @@ -453,8 +453,19 @@ "rust": ["fn {name}", "pub fn {name}"], "c": ["void {name}", "int {name}", "{name}("], "cpp": ["void {name}", "int {name}", "{name}("], - "php": ["function {name}", "public function {name}", "private function {name}", "protected function {name}"], - "kotlin": ["fun {name}", "private fun {name}", "public fun {name}", "internal fun {name}", "protected fun {name}"], + "php": [ + "function {name}", + "public function {name}", + "private function {name}", + "protected function {name}", + ], + "kotlin": [ + "fun {name}", + "private fun {name}", + "public fun {name}", + "internal fun {name}", + "protected fun {name}", + ], "general": ["{name}("], # Fallback pattern } diff --git a/codewiki/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py index 61b455c..4d5bed7 100644 --- a/codewiki/src/be/dependency_analyzer/utils/security.py +++ b/codewiki/src/be/dependency_analyzer/utils/security.py @@ -6,6 +6,7 @@ # are skipped to prevent OOM on large legacy projects. MAX_FILE_SIZE = 2 * 1024 * 1024 # 2MB + def _inside(base: Path, target: Path) -> bool: base_r = base.resolve() try: @@ -14,6 +15,7 @@ def _inside(base: Path, target: Path) -> bool: except AttributeError: return str(target.resolve()).startswith(str(base_r)) + def assert_safe_path(base_dir: Path, target: Path): # Block symlinks (file or dir) if target.is_symlink(): @@ -22,6 +24,7 @@ def assert_safe_path(base_dir: Path, target: Path): if not _inside(base_dir, target): raise PermissionError(f"Path escapes repo: {target} -> {target.resolve()}") + def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): assert_safe_path(base_dir, target) flags = os.O_RDONLY diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index d261737..7cbbbc6 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -25,7 +25,7 @@ MODULE_TREE_FILENAME, OVERVIEW_FILENAME, meta_join, - meta_resolve + meta_resolve, ) from codewiki.src.utils import file_manager @@ -38,60 +38,67 @@ def __init__(self, config: Config, commit_id: str = None, backend: LLMBackend = self.commit_id = commit_id self.graph_builder = DependencyGraphBuilder(config) self.backend: LLMBackend = backend or get_backend(config) - - def create_documentation_metadata(self, working_dir: str, components: Dict[str, Any], num_leaf_nodes: int): + + def create_documentation_metadata( + self, working_dir: str, components: Dict[str, Any], num_leaf_nodes: int + ): """Create a metadata file with documentation generation information.""" from datetime import datetime - + metadata = { "generation_info": { "timestamp": datetime.now().isoformat(), "main_model": self.config.main_model, "generator_version": "5.2.1", "repo_path": self.config.repo_path, - "commit_id": self.commit_id + "commit_id": self.commit_id, }, "statistics": { "total_components": len(components), "leaf_nodes": num_leaf_nodes, - "max_depth": self.config.max_depth + "max_depth": self.config.max_depth, }, "files_generated": [ "overview.md", ".meta/module_tree.json", - ".meta/first_module_tree.json" - ] + ".meta/first_module_tree.json", + ], } - + # Add generated markdown files to the metadata try: for file_path in os.listdir(working_dir): - if file_path.endswith('.md') and file_path not in metadata["files_generated"]: + if file_path.endswith(".md") and file_path not in metadata["files_generated"]: metadata["files_generated"].append(file_path) except Exception as e: logger.warning(f"Could not list generated files: {e}") - + metadata_path = meta_join(working_dir, "metadata.json") file_manager.save_json(metadata, metadata_path) - - def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]: + def get_processing_order( + self, module_tree: Dict[str, Any], parent_path: List[str] = [] + ) -> List[tuple[List[str], str]]: """Get the processing order using topological sort (leaf modules first).""" processing_order = [] - + def collect_modules(tree: Dict[str, Any], path: List[str]): for module_name, module_info in tree.items(): current_path = path + [module_name] - + # If this module has children, process them first - if module_info.get("children") and isinstance(module_info["children"], dict) and module_info["children"]: + if ( + module_info.get("children") + and isinstance(module_info["children"], dict) + and module_info["children"] + ): collect_modules(module_info["children"], current_path) # Add this parent module after its children processing_order.append((current_path, module_name)) else: # This is a leaf module, add it immediately processing_order.append((current_path, module_name)) - + collect_modules(module_tree, parent_path) return processing_order @@ -100,10 +107,11 @@ def is_leaf_module(self, module_info: Dict[str, Any]) -> bool: children = module_info.get("children", {}) return not children or (isinstance(children, dict) and len(children) == 0) - def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str], - working_dir: str) -> Dict[str, Any]: + def build_overview_structure( + self, module_tree: Dict[str, Any], module_path: List[str], working_dir: str + ) -> Dict[str, Any]: """Build structure for overview generation with 1-depth children docs and target indicator.""" - + processed_module_tree = deepcopy(module_tree) module_info = processed_module_tree for path_part in module_path: @@ -121,7 +129,9 @@ def build_overview_structure(self, module_tree: Dict[str, Any], module_path: Lis if child_docs_path is not None: child_info["docs"] = file_manager.load_text(child_docs_path) else: - logger.warning(f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}") + logger.warning( + f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}" + ) child_info["docs"] = "" return processed_module_tree @@ -156,7 +166,9 @@ def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None: return candidate_path return None - async def generate_module_documentation(self, components: Dict[str, Any], leaf_nodes: List[str]) -> str: + async def generate_module_documentation( + self, components: Dict[str, Any], leaf_nodes: List[str] + ) -> str: """Generate documentation for all modules using dynamic programming approach.""" # Prepare output directory working_dir = os.path.abspath(self.config.docs_dir) @@ -166,11 +178,10 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n first_module_tree_path = meta_resolve(working_dir, FIRST_MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) first_module_tree = file_manager.load_json(first_module_tree_path) - + # Get processing order (leaf modules first) processing_order = self.get_processing_order(first_module_tree) - # Process modules in dependency order final_module_tree = module_tree processed_modules = set() @@ -180,19 +191,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n try: # Reload module tree to get latest hierarchical structure from sub-agent modifications module_tree = file_manager.load_json(module_tree_path) - + # Get the module info from the tree module_info = module_tree for path_part in module_path: module_info = module_info[path_part] if path_part != module_path[-1]: # Not the last part module_info = module_info.get("children", {}) - + # Skip if already processed module_key = "/".join(module_path) if module_key in processed_modules: continue - + # Process the module if self.is_leaf_module(module_info): logger.info(f"📄 Processing leaf module: {module_key}") @@ -208,21 +219,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n final_module_tree = await self.generate_parent_module_docs( module_path, working_dir ) - + processed_modules.add(module_key) - + except Exception as e: logger.error(f"Failed to process module {module_key}: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") continue # Generate repo overview - logger.info(f"📚 Generating repository overview") - final_module_tree = await self.generate_parent_module_docs( - [], working_dir - ) + logger.info("📚 Generating repository overview") + final_module_tree = await self.generate_parent_module_docs([], working_dir) else: - logger.info(f"Processing whole repo because repo can fit in the context window") + logger.info("Processing whole repo because repo can fit in the context window") repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) final_module_tree = await self.backend.run_module_agent( module_name=repo_name, @@ -239,16 +248,21 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") if os.path.exists(repo_overview_path): os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) - + return working_dir - async def generate_parent_module_docs(self, module_path: List[str], - working_dir: str) -> Dict[str, Any]: + async def generate_parent_module_docs( + self, module_path: List[str], working_dir: str + ) -> Dict[str, Any]: """Generate documentation for a parent module based on its children's documentation.""" - module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path)) + module_name = ( + module_path[-1] + if len(module_path) >= 1 + else os.path.basename(os.path.normpath(self.config.repo_path)) + ) logger.info(f"Generating parent documentation for: {module_name}") - + # Load module tree module_tree_path = meta_resolve(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) @@ -260,7 +274,10 @@ async def generate_parent_module_docs(self, module_path: List[str], return module_tree # check if parent docs already exists - parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") + parent_docs_path = os.path.join( + working_dir, + f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md", + ) if os.path.exists(parent_docs_path): logger.info(f"✓ Parent docs already exists at {parent_docs_path}") return module_tree @@ -274,16 +291,20 @@ async def generate_parent_module_docs(self, module_path: List[str], if addition: custom_section = f"\n\n{addition}\n" - prompt = MODULE_OVERVIEW_PROMPT.format( - module_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4), - custom_instructions=custom_section, - ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format( - repo_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4), - custom_instructions=custom_section, + prompt = ( + MODULE_OVERVIEW_PROMPT.format( + module_name=module_name, + repo_structure=json.dumps(repo_structure, indent=4), + custom_instructions=custom_section, + ) + if len(module_path) >= 1 + else REPO_OVERVIEW_PROMPT.format( + repo_name=module_name, + repo_structure=json.dumps(repo_structure, indent=4), + custom_instructions=custom_section, + ) ) - + try: parent_docs = self.backend.complete(prompt) @@ -300,15 +321,15 @@ async def generate_parent_module_docs(self, module_path: List[str], ) parent_content = parent_docs.strip() file_manager.save_text(parent_content, parent_docs_path) - + logger.debug(f"Successfully generated parent documentation for: {module_name}") return module_tree - + except Exception as e: logger.error(f"Error generating parent documentation for {module_name}: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") raise - + async def run(self) -> None: """Run the complete documentation generation process using dynamic programming.""" try: @@ -318,22 +339,20 @@ async def run(self) -> None: logger.debug(f"Found {len(leaf_nodes)} leaf nodes") # logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") # exit() - + # Cluster modules working_dir = os.path.abspath(self.config.docs_dir) file_manager.ensure_directory(working_dir) first_module_tree_path = meta_resolve(working_dir, FIRST_MODULE_TREE_FILENAME) module_tree_path = meta_resolve(working_dir, MODULE_TREE_FILENAME) - + # Check if module tree exists if os.path.exists(first_module_tree_path): logger.debug(f"Module tree found at {first_module_tree_path}") module_tree = file_manager.load_json(first_module_tree_path) else: logger.debug(f"Module tree not found at {module_tree_path}, clustering modules") - clustering_tokens = get_clustering_input_token_count( - leaf_nodes, components - ) + clustering_tokens = get_clustering_input_token_count(leaf_nodes, components) logger.info( "Preparing %d leaf nodes for module clustering (%d tokens, threshold %d)", len(leaf_nodes), @@ -352,9 +371,9 @@ async def run(self) -> None: completer=lambda p: self.backend.complete(p, model=cluster_model), ) file_manager.save_json(module_tree, first_module_tree_path) - + file_manager.save_json(module_tree, module_tree_path) - + if len(module_tree) == 0: logger.info( "Module clustering produced no top-level modules; continuing in " @@ -365,18 +384,20 @@ async def run(self) -> None: "Grouped components into %d top-level modules", len(module_tree), ) - + # Generate module documentation using dynamic programming approach # This processes leaf modules first, then parent modules working_dir = await self.generate_module_documentation(components, leaf_nodes) - + # Create documentation metadata self.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - - logger.debug(f"Documentation generation completed successfully using dynamic programming!") - logger.debug(f"Processing order: leaf modules → parent modules → repository overview") + + logger.debug( + "Documentation generation completed successfully using dynamic programming!" + ) + logger.debug("Processing order: leaf modules → parent modules → repository overview") logger.debug(f"Documentation saved to: {working_dir}") - + except Exception as e: logger.error(f"Documentation generation failed: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") diff --git a/codewiki/src/be/main.py b/codewiki/src/be/main.py index 917c5de..686268a 100644 --- a/codewiki/src/be/main.py +++ b/codewiki/src/be/main.py @@ -31,15 +31,10 @@ def parse_arguments() -> argparse.Namespace: """Parse command line arguments.""" parser = argparse.ArgumentParser( - description='Generate comprehensive documentation for Python components in dependency order.' + description="Generate comprehensive documentation for Python components in dependency order." ) - parser.add_argument( - '--repo-path', - type=str, - required=True, - help='Path to the repository' - ) - + parser.add_argument("--repo-path", type=str, required=True, help="Path to the repository") + return parser.parse_args() @@ -49,11 +44,11 @@ async def main() -> None: # Parse arguments and create configuration args = parse_arguments() config = Config.from_args(args) - + # Create and run documentation generator doc_generator = DocumentationGenerator(config) await doc_generator.run() - + except KeyboardInterrupt: logger.debug("Documentation generation interrupted by user") except Exception as e: @@ -63,4 +58,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 3136012..3f92044 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -370,27 +370,63 @@ _DEFAULT_CODE_ROUTING = { "boilerplate": { "suffixes": [ - "DTO", "VO", "Request", "Response", "Entity", "PO", "DO", - "Model", "Schema", "Form", "Serializer", "Mapper", "Repository", - "Dao", "DAO", "DataClass", + "DTO", + "VO", + "Request", + "Response", + "Entity", + "PO", + "DO", + "Model", + "Schema", + "Form", + "Serializer", + "Mapper", + "Repository", + "Dao", + "DAO", + "DataClass", ], "annotations": ["@Data", "@Getter", "@Setter", "@Entity", "@Table", "@Document"], "path_keywords": ["model", "models", "dto", "vo", "entity", "entities", "schema", "pojo"], }, "business": { "suffixes": [ - "Service", "Controller", "Job", "Consumer", "Handler", - "Manager", "Processor", "Executor", "UseCase", "Interactor", - "Provider", "Resolver", "Facade", "Orchestrator", + "Service", + "Controller", + "Job", + "Consumer", + "Handler", + "Manager", + "Processor", + "Executor", + "UseCase", + "Interactor", + "Provider", + "Resolver", + "Facade", + "Orchestrator", ], "annotations": ["@Service", "@RestController", "@Controller", "@Component", "@Scheduled"], "path_keywords": ["service", "services", "controller", "handler", "job", "consumer"], }, "infra": { "suffixes": [ - "Util", "Utils", "Helper", "Factory", "Builder", "Interceptor", - "Filter", "Middleware", "Adapter", "Wrapper", "Proxy", "Client", - "Config", "Configuration", "Properties", + "Util", + "Utils", + "Helper", + "Factory", + "Builder", + "Interceptor", + "Filter", + "Middleware", + "Adapter", + "Wrapper", + "Proxy", + "Client", + "Config", + "Configuration", + "Properties", ], "annotations": ["@Configuration", "@ConfigurationProperties", "@Bean"], "path_keywords": ["util", "utils", "config", "infrastructure", "common", "shared"], @@ -405,18 +441,23 @@ def _normalize_routing_config(config: dict) -> dict: and the schema.yaml format written by schema_generator ({"boilerplate_patterns": {"suffix": [...], "annotation": [...]}}). """ - key_aliases = {"suffix": "suffixes", "annotation": "annotations", "path_keyword": "path_keywords"} + key_aliases = { + "suffix": "suffixes", + "annotation": "annotations", + "path_keyword": "path_keywords", + } normalized: dict = {} for category, rules in config.items(): if not isinstance(rules, dict): continue - cat = category[:-len("_patterns")] if category.endswith("_patterns") else category + cat = category[: -len("_patterns")] if category.endswith("_patterns") else category normalized[cat] = {key_aliases.get(k, k): v for k, v in rules.items()} return normalized -def classify_component(name: str, relative_path: str = "", source_code: str = "", - routing_config: dict | None = None) -> str: +def classify_component( + name: str, relative_path: str = "", source_code: str = "", routing_config: dict | None = None +) -> str: """Classify a component as 'boilerplate', 'business', or 'infra'. Uses suffix matching on the component/class name, path keyword matching, @@ -462,37 +503,40 @@ def classify_component(name: str, relative_path: str = "", source_code: str = "" ".hpp": "cpp", ".tsx": "typescript", ".cc": "cpp", - ".hpp": "cpp", ".cxx": "cpp", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript", - ".jsx": "javascript", ".cs": "csharp", ".kt": "kotlin", ".kts": "kotlin", ".php": "php", ".phtml": "php", - ".inc": "php" + ".inc": "php", } -def format_user_prompt(module_name: str, core_component_ids: list[str], components: Dict[str, Any], module_tree: dict[str, any]) -> str: +def format_user_prompt( + module_name: str, + core_component_ids: list[str], + components: Dict[str, Any], + module_tree: dict[str, any], +) -> str: """ Format the user prompt with module name and organized core component codes. - + Args: module_name: Name of the module to document core_component_ids: List of component IDs to include components: Dictionary mapping component IDs to CodeComponent objects - + Returns: Formatted user prompt string """ # format module tree lines = [] - + def _format_module_tree(module_tree: dict[str, any], indent: int = 0): for key, value in module_tree.items(): if key == module_name: @@ -502,8 +546,9 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): # Group components by file from collections import defaultdict + by_file = defaultdict(list) - for c in value['components']: + for c in value["components"]: if "::" in c: fpath, name = c.split("::", 1) by_file[fpath].append(name) @@ -551,14 +596,14 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): is_boilerplate = file_categories == {"boilerplate"} core_component_codes += f"# File: {path}\n\n" - core_component_codes += f"## Core Components in this file:\n" + core_component_codes += "## Core Components in this file:\n" for component_id in component_ids_in_file: core_component_codes += f"- {component_id}\n" if is_boilerplate: # Abbreviated: signature-only for boilerplate (DTO/VO/Entity/Mapper) - core_component_codes += f"\n## File Content (data class — signature only):\n" + core_component_codes += "\n## File Content (data class — signature only):\n" for cid in component_ids_in_file: comp = components[cid] params = getattr(comp, "parameters", None) or [] @@ -567,10 +612,12 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): core_component_codes += "\n" else: # Full source for business/infra components - lang = EXTENSION_TO_LANGUAGE.get('.' + path.split('.')[-1], "") + lang = EXTENSION_TO_LANGUAGE.get("." + path.split(".")[-1], "") core_component_codes += f"\n## File Content:\n```{lang}\n" try: - core_component_codes += file_manager.load_text(components[component_ids_in_file[0]].file_path) + core_component_codes += file_manager.load_text( + components[component_ids_in_file[0]].file_path + ) except (FileNotFoundError, IOError) as e: core_component_codes += f"# Error reading file: {e}\n" core_component_codes += "```\n\n" @@ -619,8 +666,9 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): return prompt + call_context - -def format_cluster_prompt(potential_core_components: str, module_tree: dict[str, any] = {}, module_name: str = None) -> str: +def format_cluster_prompt( + potential_core_components: str, module_tree: dict[str, any] = {}, module_name: str = None +) -> str: """ Format the cluster prompt with potential core components and module tree. """ @@ -629,18 +677,19 @@ def format_cluster_prompt(potential_core_components: str, module_tree: dict[str, lines = [] # print(f"Module tree:\n{json.dumps(module_tree, indent=2)}") - + def _format_module_tree(module_tree: dict[str, any], indent: int = 0): for key, value in module_tree.items(): if key == module_name: lines.append(f"{' ' * indent}{key} (current module)") else: lines.append(f"{' ' * indent}{key}") - + # Group components by file from collections import defaultdict + by_file = defaultdict(list) - for c in value['components']: + for c in value["components"]: if "::" in c: fpath, name = c.split("::", 1) by_file[fpath].append(name) @@ -652,51 +701,60 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): else: lines.append(f"{' ' * (indent + 1)} {', '.join(names)}") - if ("children" in value) and isinstance(value["children"], dict) and len(value["children"]) > 0: + if ( + ("children" in value) + and isinstance(value["children"], dict) + and len(value["children"]) > 0 + ): lines.append(f"{' ' * (indent + 1)} Children:") _format_module_tree(value["children"], indent + 2) - + _format_module_tree(module_tree, 0) formatted_module_tree = "\n".join(lines) - if module_tree == {}: return CLUSTER_REPO_PROMPT.format(potential_core_components=potential_core_components) else: - return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name) + return CLUSTER_MODULE_PROMPT.format( + potential_core_components=potential_core_components, + module_tree=formatted_module_tree, + module_name=module_name, + ) def format_system_prompt(module_name: str, custom_instructions: str = None) -> str: """ Format the system prompt with module name and optional custom instructions. - + Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append - + Returns: Formatted system prompt string """ custom_section = "" if custom_instructions: custom_section = f"\n\n\n{custom_instructions}\n" - + return SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() def format_leaf_system_prompt(module_name: str, custom_instructions: str = None) -> str: """ Format the leaf system prompt with module name and optional custom instructions. - + Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append - + Returns: Formatted leaf system prompt string """ custom_section = "" if custom_instructions: custom_section = f"\n\n\n{custom_instructions}\n" - - return LEAF_SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() \ No newline at end of file + + return LEAF_SYSTEM_PROMPT.format( + module_name=module_name, custom_instructions=custom_section + ).strip() diff --git a/codewiki/src/be/pydantic_ai_backend.py b/codewiki/src/be/pydantic_ai_backend.py index ca41925..f9ec918 100644 --- a/codewiki/src/be/pydantic_ai_backend.py +++ b/codewiki/src/be/pydantic_ai_backend.py @@ -63,6 +63,7 @@ async def run_module_agent( ) -> Dict[str, Any]: config = self._config from codewiki.src.config import meta_resolve + module_tree_path = meta_resolve(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py index d497708..dcd8e89 100644 --- a/codewiki/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -7,7 +7,6 @@ from typing import List, Tuple import logging import tiktoken -import traceback logger = logging.getLogger(__name__) @@ -29,10 +28,12 @@ def set_main_loop(loop: asyncio.AbstractEventLoop) -> None: _main_loop = loop _main_loop_thread_ident = threading.get_ident() + # ------------------------------------------------------------ # ---------------------- Complexity Check -------------------- # ------------------------------------------------------------ + def is_complex_module(components: dict[str, any], core_component_ids: list[str]) -> bool: files = set() for component_id in core_component_ids: @@ -50,6 +51,7 @@ def is_complex_module(components: dict[str, any], core_component_ids: list[str]) enc = tiktoken.encoding_for_model("gpt-4") + def count_tokens(text: str) -> int: """ Count the number of tokens in a text. @@ -63,10 +65,11 @@ def count_tokens(text: str) -> int: # ---------------------- Mermaid Validation ----------------- # ------------------------------------------------------------ + async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> str: """ Validate all Mermaid diagrams in a markdown file. - + Args: md_file_path: Path to the markdown file to check relative_path: Relative path to the markdown file @@ -80,15 +83,15 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st file_path = Path(md_file_path) if not file_path.exists(): return f"Error: File '{md_file_path}' does not exist" - - content = file_path.read_text(encoding='utf-8') - + + content = file_path.read_text(encoding="utf-8") + # Extract all mermaid code blocks mermaid_blocks = extract_mermaid_blocks(content) - + if not mermaid_blocks: return "No mermaid diagrams found in the file" - + # Validate each mermaid diagram sequentially to avoid segfaults errors = [] for i, (line_start, diagram_content) in enumerate(mermaid_blocks, 1): @@ -96,15 +99,17 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st if error_msg: errors.append("\n") errors.append(error_msg) - + # if errors: # logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") - + if errors: - return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) + return ( + "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) + ) else: return "All mermaid diagrams in file: " + relative_path + " are syntax correct" - + except Exception as e: return f"Error processing file: {str(e)}" @@ -151,7 +156,7 @@ def auto_fix_mermaid_blocks(content: str) -> Tuple[str, List[str]]: continue # Rule 3: Quote CJK subgraph titles - m = re.match(r'^(\s*subgraph\s+)([\u4e00-\u9fff][^\"]*?)$', line) + m = re.match(r"^(\s*subgraph\s+)([\u4e00-\u9fff][^\"]*?)$", line) if m and not m.group(2).strip().startswith('"'): title = m.group(2).strip() line = f'{m.group(1)}"{title}"' @@ -163,11 +168,11 @@ def _quote_brace_label(mo: re.Match) -> str: prefix, label = mo.group(1), mo.group(2) if label.startswith('"'): return mo.group(0) # already quoted - fixes.append(f'Quoted label with curly braces: {label[:30]}') + fixes.append(f"Quoted label with curly braces: {label[:30]}") return f'{prefix}"{label}"]' line = re.sub( - r'(\w\[)([^\]]*[{}][^\]]*)\]', + r"(\w\[)([^\]]*[{}][^\]]*)\]", _quote_brace_label, line, ) @@ -175,25 +180,25 @@ def _quote_brace_label(mo: re.Match) -> str: # Rule 1: Split multi-node one-liners. # Detect ] followed by whitespace and another node-start (word + [ or ( ), # but NOT when --, -, ., ==, --> follows (that's an edge, which is valid). - if ']' in line: + if "]" in line: # Preserve original indentation for split parts - indent = re.match(r'^(\s*)', line).group(1) + indent = re.match(r"^(\s*)", line).group(1) # Find positions where a ] is followed by space + identifier + [ or ( # but not preceded by an edge arrow parts = [line] while True: - m = re.search(r'\](\s+)(\w[\w]*)\s*[\[\(]', parts[-1]) + m = re.search(r"\](\s+)(\w[\w]*)\s*[\[\(]", parts[-1]) if not m: break # Check the text between ] and the next node — reject if it's an edge between = m.group(1) - if re.search(r'(-->|--|-|->|==|\.->|\.\.)', between.strip()): + if re.search(r"(-->|--|-|->|==|\.->|\.\.)", between.strip()): break # Split: everything up to and including ] stays, rest goes to new line split_pos = m.start() + 1 # right after the ] parts.append(indent + parts[-1][split_pos:].strip()) parts[-2] = parts[-2][:split_pos].rstrip() - fixes.append('Split multi-node one-liner into separate lines') + fixes.append("Split multi-node one-liner into separate lines") if len(parts) > 1: result_lines.extend(parts) continue @@ -207,36 +212,36 @@ def _quote_brace_label(mo: re.Match) -> str: def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]: """ Extract all mermaid code blocks from markdown content. - + Returns: List of tuples containing (line_number, diagram_content) """ mermaid_blocks = [] - lines = content.split('\n') + lines = content.split("\n") i = 0 - + while i < len(lines): line = lines[i].strip() - + # Look for mermaid code block start - if line == '```mermaid' or line.startswith('```mermaid'): + if line == "```mermaid" or line.startswith("```mermaid"): start_line = i + 1 diagram_lines = [] i += 1 - + # Collect lines until we find the closing ``` while i < len(lines): - if lines[i].strip() == '```': + if lines[i].strip() == "```": break diagram_lines.append(lines[i]) i += 1 - + if diagram_lines: # Only add non-empty diagrams - diagram_content = '\n'.join(diagram_lines) + diagram_content = "\n".join(diagram_lines) mermaid_blocks.append((start_line, diagram_content)) - + i += 1 - + return mermaid_blocks @@ -273,16 +278,14 @@ async def _try_pythonmonkey_parse(diagram_content: str) -> str | None: return None old_stderr = sys.stderr - sys.stderr = open(os.devnull, 'w') + sys.stderr = open(os.devnull, "w") try: if ( _main_loop is not None and _main_loop.is_running() and threading.get_ident() != _main_loop_thread_ident ): - fut = asyncio.run_coroutine_threadsafe( - parse_mermaid_py(diagram_content), _main_loop - ) + fut = asyncio.run_coroutine_threadsafe(parse_mermaid_py(diagram_content), _main_loop) await asyncio.wrap_future(fut) else: await parse_mermaid_py(diagram_content) @@ -313,6 +316,7 @@ def _parse_via_mermaid_py(diagram_content: str) -> str: text, otherwise a successful SVG gets reported as a parse error. """ import mermaid as md + try: md.Mermaid(diagram_content) return "" @@ -349,7 +353,9 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s # Validation disabled/unavailable is not a syntax error — a # non-empty return here would make callers report valid diagrams # as broken and send agents into fix loops. - logger.debug("Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num) + logger.debug( + "Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num + ) return "" try: core_error = await asyncio.wait_for( @@ -361,7 +367,10 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s # diagrams (and later calls) don't each block 15s on the same # broken Node.js setup. _MERMAID_PY_BROKEN = True - logger.warning("Diagram %d: mermaid validation timed out (15s); skipping further validation", diagram_num) + logger.warning( + "Diagram %d: mermaid validation timed out (15s); skipping further validation", + diagram_num, + ) return "" except Exception as e: return f" Diagram {diagram_num}: Exception during validation - {str(e)}" @@ -369,11 +378,11 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s if not core_error: return "" - line_match = re.search(r'line (\d+)', core_error) + line_match = re.search(r"line (\d+)", core_error) if line_match: error_line_in_diagram = int(line_match.group(1)) actual_line_in_file = line_start + error_line_in_diagram - newline = '\n' + newline = "\n" return f"Diagram {diagram_num}: Parse error on line {actual_line_in_file}:{newline}{newline.join(core_error.split(newline)[1:])}" return f"Diagram {diagram_num}: {core_error}" @@ -381,6 +390,7 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s if __name__ == "__main__": # Test with the provided file import asyncio + test_file = "output/docs/SWE_agent-docs/agent_hooks.md" result = asyncio.run(validate_mermaid_diagrams(test_file, "agent_hooks.md")) - print(result) \ No newline at end of file + print(result) diff --git a/codewiki/src/config.py b/codewiki/src/config.py index b706154..d3f9cb3 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -1,69 +1,69 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional, List, Dict, Any import argparse import os import re -import sys # Constants -OUTPUT_BASE_DIR = 'output' -DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' -DOCS_DIR = 'docs' -META_DIR = '.meta' -FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' -MODULE_TREE_FILENAME = 'module_tree.json' -OVERVIEW_FILENAME = 'overview.md' +OUTPUT_BASE_DIR = "output" +DEPENDENCY_GRAPHS_DIR = "dependency_graphs" +DOCS_DIR = "docs" +META_DIR = ".meta" +FIRST_MODULE_TREE_FILENAME = "first_module_tree.json" +MODULE_TREE_FILENAME = "module_tree.json" +OVERVIEW_FILENAME = "overview.md" # LLM Wiki constants -SCHEMA_FILENAME = 'schema.yaml' -NOTES_DIR = 'notes' -INDEX_FILENAME = 'index.md' -LOG_FILENAME = 'log.md' -SEARCH_INDEX_FILENAME = 'search_index.db' -SYMBOL_MAP_FILENAME = 'symbol_map.json' +SCHEMA_FILENAME = "schema.yaml" +NOTES_DIR = "notes" +INDEX_FILENAME = "index.md" +LOG_FILENAME = "log.md" +SEARCH_INDEX_FILENAME = "search_index.db" +SYMBOL_MAP_FILENAME = "symbol_map.json" # LLM Wiki knowledge layer — structured layout constants -WIKI_DIR = 'wiki' -RAW_DIR = 'raw' -RAW_SOURCES_DIR = 'raw/sources' +WIKI_DIR = "wiki" +RAW_DIR = "raw" +RAW_SOURCES_DIR = "raw/sources" # L0 archive (team-memory fusion): distilled conversations are moved here for # permanent provenance. Link-only layer — NOT indexed for BM25 search; reached # by following note source_ref links (view_repo_file). raw/ stays the pending # staging queue. -CONVERSATIONS_DIR = 'conversations' +CONVERSATIONS_DIR = "conversations" # Task memory layer — per-task knowledge store (task.md + memories.md + index) -TASKS_DIR = 'tasks' -TASKS_INDEX_FILENAME = '.index.json' -TASKS_MEMORIES_FILENAME = 'memories.md' -TASK_BINDINGS_DIR = 'task_bindings' -SOURCE_REGISTRY_FILENAME = 'source_registry.json' -ISSUES_FILENAME = 'issues.json' -PROJECT_FILENAME = 'project.json' +TASKS_DIR = "tasks" +TASKS_INDEX_FILENAME = ".index.json" +TASKS_MEMORIES_FILENAME = "memories.md" +TASK_BINDINGS_DIR = "task_bindings" +SOURCE_REGISTRY_FILENAME = "source_registry.json" +ISSUES_FILENAME = "issues.json" +PROJECT_FILENAME = "project.json" # Mapping from page_type to subdirectory name under wiki/ PAGE_TYPE_DIRS = { - 'module': 'modules', - 'entity': 'entities', - 'concept': 'concepts', - 'source': 'sources', - 'comparison': 'comparisons', - 'query': 'queries', + "module": "modules", + "entity": "entities", + "concept": "concepts", + "source": "sources", + "comparison": "comparisons", + "query": "queries", # P2 (team-memory fusion): L2 work-method scene blocks — consolidated # reusable knowledge (SOP / judgment logic / taboos / principles) distilled # from confirmed notes via consolidate_notes. - 'scenario': 'scenarios', + "scenario": "scenarios", } # Files excluded from wiki index and search (system files) -WIKI_SYSTEM_FILES = {'index.md', 'log.md', 'overview.md', 'schema.yaml'} +WIKI_SYSTEM_FILES = {"index.md", "log.md", "overview.md", "schema.yaml"} # OKF v0.2 actor convention (§7): '/' for agents and tools # (e.g. ``reference_agent/gemini-2.5-pro``), 'human:' for people, # 'process:' for pipelines. Single source of truth for the actor string # used in `generated.by` / `verified[].by` fields. -ACTOR_NAME = 'codewiki' -OKF_VERSION = '0.2' +ACTOR_NAME = "codewiki" +OKF_VERSION = "0.2" def actor_id() -> str: """Return the OKF actor id for this tool, e.g. ``codewiki/5.2.0`` (§7).""" from codewiki import __version__ + return f"{ACTOR_NAME}/{__version__}" @@ -88,8 +88,11 @@ def _git_config_value(key: str) -> str: try: proc = subprocess.run( ["git", "config", key], - capture_output=True, text=True, timeout=2, - encoding="utf-8", errors="replace", + capture_output=True, + text=True, + timeout=2, + encoding="utf-8", + errors="replace", ) if proc.returncode == 0: return (proc.stdout or "").strip() @@ -221,27 +224,32 @@ def meta_resolve(base_dir, filename): # CLI context detection _CLI_CONTEXT = False + def set_cli_context(enabled: bool = True): """Set whether we're running in CLI context (vs web app).""" global _CLI_CONTEXT _CLI_CONTEXT = enabled + def is_cli_context() -> bool: """Check if running in CLI context.""" return _CLI_CONTEXT + # LLM services # In CLI mode, these will be loaded from ~/.codewiki/config.json + keyring # In web app mode, use environment variables -MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') -FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') -CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) -LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') -LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') +MAIN_MODEL = os.getenv("MAIN_MODEL", "claude-sonnet-4") +FALLBACK_MODEL_1 = os.getenv("FALLBACK_MODEL_1", "glm-4p5") +CLUSTER_MODEL = os.getenv("CLUSTER_MODEL", MAIN_MODEL) +LLM_BASE_URL = os.getenv("LLM_BASE_URL", "http://0.0.0.0:4000/") +LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-1234") + @dataclass class Config: """Configuration class for CodeWiki.""" + repo_path: str output_dir: str dependency_graph_dir: str @@ -264,77 +272,79 @@ class Config: max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE # Agent instructions for customization agent_instructions: Optional[Dict[str, Any]] = None - + @property def include_patterns(self) -> Optional[List[str]]: """Get file include patterns from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('include_patterns') + return self.agent_instructions.get("include_patterns") return None - + @property def exclude_patterns(self) -> Optional[List[str]]: """Get file exclude patterns from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('exclude_patterns') + return self.agent_instructions.get("exclude_patterns") return None - + @property def focus_modules(self) -> Optional[List[str]]: """Get focus modules from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('focus_modules') + return self.agent_instructions.get("focus_modules") return None - + @property def doc_type(self) -> Optional[str]: """Get documentation type from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('doc_type') + return self.agent_instructions.get("doc_type") return None - + @property def custom_instructions(self) -> Optional[str]: """Get custom instructions from agent instructions.""" if self.agent_instructions: - return self.agent_instructions.get('custom_instructions') + return self.agent_instructions.get("custom_instructions") return None - + def get_prompt_addition(self) -> str: """Generate prompt additions based on agent instructions.""" if not self.agent_instructions: return "" - + additions = [] - + if self.doc_type: doc_type_instructions = { - 'api': "Focus on API documentation: endpoints, parameters, return types, and usage examples.", - 'architecture': "Focus on architecture documentation: system design, component relationships, and data flow.", - 'user-guide': "Focus on user guide documentation: how to use features, step-by-step tutorials.", - 'developer': "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", - 'business': "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details.", - 'design': "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", + "api": "Focus on API documentation: endpoints, parameters, return types, and usage examples.", + "architecture": "Focus on architecture documentation: system design, component relationships, and data flow.", + "user-guide": "Focus on user guide documentation: how to use features, step-by-step tutorials.", + "developer": "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", + "business": "Focus on business logic documentation: describe business workflows, processing pipelines, state transitions, and domain rules. Emphasize WHAT the system does for users and WHY, trace end-to-end business scenarios through the code, and document domain-specific terminology. De-emphasize infrastructure and deployment details.", + "design": "Generate technical design documentation optimized for AI comprehension. For each module, describe in depth: (1) module responsibilities and boundaries, (2) detailed implementation logic and business rules, (3) data flow within and through the module, (4) interface contracts — inputs, outputs, and side effects, (5) internal layered design and component collaboration patterns, (6) relationships and dependencies with other modules, (7) constraints, assumptions, and edge cases. Use precise technical language. Include Mermaid diagrams for complex flows and interactions. Do not limit documentation length — let the content depth match the module's complexity.", } if self.doc_type.lower() in doc_type_instructions: additions.append(doc_type_instructions[self.doc_type.lower()]) else: additions.append(f"Focus on generating {self.doc_type} documentation.") - + if self.focus_modules: - additions.append(f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}") - + additions.append( + f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}" + ) + if self.custom_instructions: additions.append(f"Additional instructions: {self.custom_instructions}") - + return "\n".join(additions) if additions else "" - + @classmethod - def from_args(cls, args: argparse.Namespace) -> 'Config': + def from_args(cls, args: argparse.Namespace) -> "Config": """Create configuration from parsed arguments.""" repo_name = os.path.basename(os.path.normpath(args.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) - + sanitized_repo_name = "".join(c if c.isalnum() else "_" for c in repo_name) + return cls( repo_path=args.repo_path, output_dir=OUTPUT_BASE_DIR, @@ -345,9 +355,9 @@ def from_args(cls, args: argparse.Namespace) -> 'Config': llm_api_key=LLM_API_KEY, main_model=MAIN_MODEL, cluster_model=CLUSTER_MODEL, - fallback_model=FALLBACK_MODEL_1 + fallback_model=FALLBACK_MODEL_1, ) - + @classmethod def from_cli( cls, @@ -366,8 +376,8 @@ def from_cli( max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE, max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE, max_depth: int = MAX_DEPTH, - agent_instructions: Optional[Dict[str, Any]] = None - ) -> 'Config': + agent_instructions: Optional[Dict[str, Any]] = None, + ) -> "Config": """ Create configuration for CLI context. @@ -392,7 +402,7 @@ def from_cli( Returns: Config instance """ - repo_name = os.path.basename(os.path.normpath(repo_path)) + os.path.basename(os.path.normpath(repo_path)) base_output_dir = os.path.join(output_dir, "temp") return cls( @@ -413,5 +423,5 @@ def from_cli( max_tokens=max_tokens, max_token_per_module=max_token_per_module, max_token_per_leaf_module=max_token_per_leaf_module, - agent_instructions=agent_instructions - ) \ No newline at end of file + agent_instructions=agent_instructions, + ) diff --git a/codewiki/src/fe/__init__.py b/codewiki/src/fe/__init__.py index 9fa1624..63edc7e 100644 --- a/codewiki/src/fe/__init__.py +++ b/codewiki/src/fe/__init__.py @@ -13,14 +13,14 @@ from .routes import WebRoutes __all__ = [ - 'app', - 'main', - 'JobStatus', - 'JobStatusResponse', - 'RepositorySubmission', - 'CacheEntry', - 'CacheManager', - 'BackgroundWorker', - 'GitHubRepoProcessor', - 'WebRoutes' -] \ No newline at end of file + "app", + "main", + "JobStatus", + "JobStatusResponse", + "RepositorySubmission", + "CacheEntry", + "CacheManager", + "BackgroundWorker", + "GitHubRepoProcessor", + "WebRoutes", +] diff --git a/codewiki/src/fe/background_worker.py b/codewiki/src/fe/background_worker.py index 5e77991..6a2d321 100644 --- a/codewiki/src/fe/background_worker.py +++ b/codewiki/src/fe/background_worker.py @@ -4,7 +4,6 @@ """ import os -import json import time import threading import subprocess @@ -13,7 +12,6 @@ from pathlib import Path from queue import Queue from typing import Dict -from dataclasses import asdict from codewiki.src.be.documentation_generator import DocumentationGenerator from codewiki.src.config import Config, MAIN_MODEL @@ -23,9 +21,10 @@ from .config import WebAppConfig from codewiki.src.utils import file_manager + class BackgroundWorker: """Background worker for processing documentation generation jobs.""" - + def __init__(self, cache_manager: CacheManager, temp_dir: str = None): self.cache_manager = cache_manager self.temp_dir = temp_dir or WebAppConfig.TEMP_DIR @@ -34,7 +33,7 @@ def __init__(self, cache_manager: CacheManager, temp_dir: str = None): self.job_status: Dict[str, JobStatus] = {} self.jobs_file = Path(WebAppConfig.CACHE_DIR) / "jobs.json" self.load_job_statuses() - + def start(self): """Start the background worker thread.""" if not self.running: @@ -42,111 +41,118 @@ def start(self): thread = threading.Thread(target=self._worker_loop, daemon=True) thread.start() print("Background worker started") - + def stop(self): """Stop the background worker.""" self.running = False - + def add_job(self, job_id: str, job: JobStatus): """Add a job to the processing queue.""" self.job_status[job_id] = job self.processing_queue.put(job_id) - + def get_job_status(self, job_id: str) -> JobStatus: """Get job status by ID.""" return self.job_status.get(job_id) - + def get_all_jobs(self) -> Dict[str, JobStatus]: """Get all job statuses.""" return self.job_status - + def load_job_statuses(self): """Load job statuses from disk.""" if not self.jobs_file.exists(): # Try to reconstruct from cache if no job file exists self._reconstruct_jobs_from_cache() return - + try: data = file_manager.load_json(self.jobs_file) - + for job_id, job_data in data.items(): # Only load completed jobs to avoid inconsistent state - if job_data.get('status') == 'completed': + if job_data.get("status") == "completed": self.job_status[job_id] = JobStatus( - job_id=job_data['job_id'], - repo_url=job_data['repo_url'], - status=job_data['status'], - created_at=datetime.fromisoformat(job_data['created_at']), - started_at=datetime.fromisoformat(job_data['started_at']) if job_data.get('started_at') else None, - completed_at=datetime.fromisoformat(job_data['completed_at']) if job_data.get('completed_at') else None, - error_message=job_data.get('error_message'), - progress=job_data.get('progress', ''), - docs_path=job_data.get('docs_path') + job_id=job_data["job_id"], + repo_url=job_data["repo_url"], + status=job_data["status"], + created_at=datetime.fromisoformat(job_data["created_at"]), + started_at=datetime.fromisoformat(job_data["started_at"]) + if job_data.get("started_at") + else None, + completed_at=datetime.fromisoformat(job_data["completed_at"]) + if job_data.get("completed_at") + else None, + error_message=job_data.get("error_message"), + progress=job_data.get("progress", ""), + docs_path=job_data.get("docs_path"), ) - print(f"Loaded {len([j for j in self.job_status.values() if j.status == 'completed'])} completed jobs from disk") + print( + f"Loaded {len([j for j in self.job_status.values() if j.status == 'completed'])} completed jobs from disk" + ) except Exception as e: print(f"Error loading job statuses: {e}") - + def _reconstruct_jobs_from_cache(self): """Reconstruct job statuses from cache entries for backward compatibility.""" try: cache_entries = self.cache_manager.cache_index reconstructed_count = 0 - + for repo_hash, cache_entry in cache_entries.items(): # Extract repo info to create job_id from .github_processor import GitHubRepoProcessor + try: repo_info = GitHubRepoProcessor.get_repo_info(cache_entry.repo_url) - job_id = repo_info['full_name'].replace('/', '--') - + job_id = repo_info["full_name"].replace("/", "--") + # Only add if job doesn't already exist if job_id not in self.job_status: self.job_status[job_id] = JobStatus( job_id=job_id, repo_url=cache_entry.repo_url, - status='completed', + status="completed", created_at=cache_entry.created_at, completed_at=cache_entry.created_at, docs_path=cache_entry.docs_path, - progress="Reconstructed from cache" + progress="Reconstructed from cache", ) reconstructed_count += 1 except Exception as e: print(f"Failed to reconstruct job for {cache_entry.repo_url}: {e}") - + if reconstructed_count > 0: print(f"Reconstructed {reconstructed_count} job statuses from cache") self.save_job_statuses() - + except Exception as e: print(f"Error reconstructing jobs from cache: {e}") - + def save_job_statuses(self): """Save job statuses to disk.""" try: # Ensure cache directory exists self.jobs_file.parent.mkdir(parents=True, exist_ok=True) - + data = {} for job_id, job in self.job_status.items(): data[job_id] = { - 'job_id': job.job_id, - 'repo_url': job.repo_url, - 'status': job.status, - 'created_at': job.created_at.isoformat(), - 'started_at': job.started_at.isoformat() if job.started_at else None, - 'completed_at': job.completed_at.isoformat() if job.completed_at else None, - 'error_message': job.error_message, - 'progress': job.progress, - 'docs_path': job.docs_path + "job_id": job.job_id, + "repo_url": job.repo_url, + "status": job.status, + "created_at": job.created_at.isoformat(), + "started_at": job.started_at.isoformat() if job.started_at else None, + "completed_at": job.completed_at.isoformat() if job.completed_at else None, + "error_message": job.error_message, + "progress": job.progress, + "docs_path": job.docs_path, } - + file_manager.save_json(data, self.jobs_file) except Exception as e: print(f"Error saving job statuses: {e}") - + def _worker_loop(self): """Main worker loop.""" while self.running: @@ -159,62 +165,65 @@ def _worker_loop(self): except Exception as e: print(f"Worker error: {e}") time.sleep(1) - + def _process_job(self, job_id: str): """Process a single documentation generation job.""" if job_id not in self.job_status: return - + job = self.job_status[job_id] - + try: # Update job status - job.status = 'processing' + job.status = "processing" job.started_at = datetime.now() job.progress = "Starting repository clone..." job.main_model = MAIN_MODEL - + # Check cache first cached_docs = self.cache_manager.get_cached_docs(job.repo_url) if cached_docs and Path(cached_docs).exists(): - job.status = 'completed' + job.status = "completed" job.completed_at = datetime.now() job.docs_path = cached_docs job.progress = "Documentation retrieved from cache" if not job.main_model: # Only set if not already set job.main_model = MAIN_MODEL - + # Save job status to disk self.save_job_statuses() - + print(f"Job {job_id}: Using cached documentation") return - + # Clone repository repo_info = GitHubRepoProcessor.get_repo_info(job.repo_url) # Use repo full name for temp directory (already URL-safe since job_id is URL-safe) temp_repo_dir = os.path.join(self.temp_dir, job_id) - + job.progress = f"Cloning repository {repo_info['full_name']}..." - - if not GitHubRepoProcessor.clone_repository(repo_info['clone_url'], temp_repo_dir, job.commit_id): + + if not GitHubRepoProcessor.clone_repository( + repo_info["clone_url"], temp_repo_dir, job.commit_id + ): raise Exception("Failed to clone repository") - + # Generate documentation job.progress = "Analyzing repository structure..." - + # Create config for documentation generation (using env vars) import argparse + args = argparse.Namespace(repo_path=temp_repo_dir) config = Config.from_args(args) # Override docs_dir with job-specific directory config.docs_dir = os.path.join("output", "docs", f"{job_id}-docs") - + job.progress = "Generating documentation..." - + # Generate documentation doc_generator = DocumentationGenerator(config, job.commit_id) - + # Run the async documentation generation in a new event loop loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -222,35 +231,35 @@ def _process_job(self, job_id: str): loop.run_until_complete(doc_generator.run()) finally: loop.close() - + # Cache the results docs_path = os.path.abspath(config.docs_dir) self.cache_manager.add_to_cache(job.repo_url, docs_path) - + # Update job status - job.status = 'completed' + job.status = "completed" job.completed_at = datetime.now() job.docs_path = docs_path job.progress = "Documentation generation completed" - + # Save job status to disk self.save_job_statuses() - + print(f"Job {job_id}: Documentation generated successfully") - + except Exception as e: # Update job status with error - job.status = 'failed' + job.status = "failed" job.completed_at = datetime.now() job.error_message = str(e) job.progress = f"Failed: {str(e)}" - + print(f"Job {job_id}: Failed with error: {e}") - + finally: # Cleanup temporary repository - if 'temp_repo_dir' in locals() and os.path.exists(temp_repo_dir): + if "temp_repo_dir" in locals() and os.path.exists(temp_repo_dir): try: - subprocess.run(['rm', '-rf', temp_repo_dir], check=True) + subprocess.run(["rm", "-rf", temp_repo_dir], check=True) except Exception as e: - print(f"Failed to cleanup temp directory: {e}") \ No newline at end of file + print(f"Failed to cleanup temp directory: {e}") diff --git a/codewiki/src/fe/cache_manager.py b/codewiki/src/fe/cache_manager.py index d156051..2a853ef 100644 --- a/codewiki/src/fe/cache_manager.py +++ b/codewiki/src/fe/cache_manager.py @@ -15,14 +15,14 @@ class CacheManager: """Manages documentation cache.""" - + def __init__(self, cache_dir: str = None, cache_expiry_days: int = None): self.cache_dir = Path(cache_dir or WebAppConfig.CACHE_DIR) self.cache_expiry_days = cache_expiry_days or WebAppConfig.CACHE_EXPIRY_DAYS self.cache_dir.mkdir(parents=True, exist_ok=True) self.cache_index: Dict[str, CacheEntry] = {} self.load_cache_index() - + def load_cache_index(self): """Load cache index from disk.""" index_file = self.cache_dir / "cache_index.json" @@ -31,15 +31,15 @@ def load_cache_index(self): data = file_manager.load_json(index_file) for key, value in data.items(): self.cache_index[key] = CacheEntry( - repo_url=value['repo_url'], - repo_url_hash=value['repo_url_hash'], - docs_path=value['docs_path'], - created_at=datetime.fromisoformat(value['created_at']), - last_accessed=datetime.fromisoformat(value['last_accessed']) + repo_url=value["repo_url"], + repo_url_hash=value["repo_url_hash"], + docs_path=value["docs_path"], + created_at=datetime.fromisoformat(value["created_at"]), + last_accessed=datetime.fromisoformat(value["last_accessed"]), ) except Exception as e: print(f"Error loading cache index: {e}") - + def save_cache_index(self): """Save cache index to disk.""" index_file = self.cache_dir / "cache_index.json" @@ -47,28 +47,28 @@ def save_cache_index(self): data = {} for key, entry in self.cache_index.items(): data[key] = { - 'repo_url': entry.repo_url, - 'repo_url_hash': entry.repo_url_hash, - 'docs_path': entry.docs_path, - 'created_at': entry.created_at.isoformat(), - 'last_accessed': entry.last_accessed.isoformat() + "repo_url": entry.repo_url, + "repo_url_hash": entry.repo_url_hash, + "docs_path": entry.docs_path, + "created_at": entry.created_at.isoformat(), + "last_accessed": entry.last_accessed.isoformat(), } - + file_manager.save_json(data, index_file) except Exception as e: print(f"Error saving cache index: {e}") - + def get_repo_hash(self, repo_url: str) -> str: """Generate hash for repository URL.""" return hashlib.sha256(repo_url.encode()).hexdigest()[:16] - + def get_cached_docs(self, repo_url: str) -> Optional[str]: """Get cached documentation path if available.""" repo_hash = self.get_repo_hash(repo_url) - + if repo_hash in self.cache_index: entry = self.cache_index[repo_hash] - + # Check if cache is still valid if datetime.now() - entry.created_at < timedelta(days=self.cache_expiry_days): # Update last accessed @@ -78,42 +78,42 @@ def get_cached_docs(self, repo_url: str) -> Optional[str]: else: # Cache expired, remove it self.remove_from_cache(repo_url) - + return None - + def add_to_cache(self, repo_url: str, docs_path: str): """Add documentation to cache.""" repo_hash = self.get_repo_hash(repo_url) now = datetime.now() - + self.cache_index[repo_hash] = CacheEntry( repo_url=repo_url, repo_url_hash=repo_hash, docs_path=docs_path, created_at=now, - last_accessed=now + last_accessed=now, ) - + self.save_cache_index() - + def remove_from_cache(self, repo_url: str): """Remove documentation from cache.""" repo_hash = self.get_repo_hash(repo_url) if repo_hash in self.cache_index: del self.cache_index[repo_hash] self.save_cache_index() - + def cleanup_expired_cache(self): """Remove expired cache entries.""" expired_entries = [] cutoff = datetime.now() - timedelta(days=self.cache_expiry_days) - + for repo_hash, entry in self.cache_index.items(): if entry.created_at < cutoff: expired_entries.append(repo_hash) - + for repo_hash in expired_entries: del self.cache_index[repo_hash] - + if expired_entries: - self.save_cache_index() \ No newline at end of file + self.save_cache_index() diff --git a/codewiki/src/fe/config.py b/codewiki/src/fe/config.py index c77d3d4..37a208a 100644 --- a/codewiki/src/fe/config.py +++ b/codewiki/src/fe/config.py @@ -9,43 +9,39 @@ class WebAppConfig: """Configuration class for web application settings.""" - + # Directories CACHE_DIR = "./output/cache" TEMP_DIR = "./output/temp" OUTPUT_DIR = "./output" - + # Queue settings QUEUE_SIZE = 100 - + # Cache settings CACHE_EXPIRY_DAYS = 365 - + # Job cleanup settings JOB_CLEANUP_HOURS = 24000 RETRY_COOLDOWN_MINUTES = 3 - + # Server settings DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8000 - + # Git settings CLONE_TIMEOUT = 300 CLONE_DEPTH = 1 - + @classmethod def ensure_directories(cls): """Ensure all required directories exist.""" - directories = [ - cls.CACHE_DIR, - cls.TEMP_DIR, - cls.OUTPUT_DIR - ] - + directories = [cls.CACHE_DIR, cls.TEMP_DIR, cls.OUTPUT_DIR] + for directory in directories: Path(directory).mkdir(parents=True, exist_ok=True) - + @classmethod def get_absolute_path(cls, path: str) -> str: """Get absolute path for a given relative path.""" - return os.path.abspath(path) \ No newline at end of file + return os.path.abspath(path) diff --git a/codewiki/src/fe/github_processor.py b/codewiki/src/fe/github_processor.py index bc9084d..71d108c 100644 --- a/codewiki/src/fe/github_processor.py +++ b/codewiki/src/fe/github_processor.py @@ -13,81 +13,98 @@ class GitHubRepoProcessor: """Handles GitHub repository processing.""" - + @staticmethod def is_valid_github_url(url: str) -> bool: """Validate if the URL is a valid GitHub repository URL.""" try: parsed = urlparse(url) - if parsed.netloc.lower() not in ['github.com', 'www.github.com']: + if parsed.netloc.lower() not in ["github.com", "www.github.com"]: return False - - path_parts = parsed.path.strip('/').split('/') + + path_parts = parsed.path.strip("/").split("/") if len(path_parts) < 2: return False - + # Check if it's a valid repo path (owner/repo) return len(path_parts) >= 2 and all(part for part in path_parts[:2]) except Exception: return False - + @staticmethod def get_repo_info(url: str) -> Dict[str, str]: """Extract repository information from GitHub URL.""" parsed = urlparse(url) - path_parts = parsed.path.strip('/').split('/') - + path_parts = parsed.path.strip("/").split("/") + owner = path_parts[0] repo = path_parts[1] - + # Remove .git suffix if present - if repo.endswith('.git'): + if repo.endswith(".git"): repo = repo[:-4] - + return { - 'owner': owner, - 'repo': repo, - 'full_name': f"{owner}/{repo}", - 'clone_url': f"https://github.com/{owner}/{repo}.git" + "owner": owner, + "repo": repo, + "full_name": f"{owner}/{repo}", + "clone_url": f"https://github.com/{owner}/{repo}.git", } - + @staticmethod def clone_repository(clone_url: str, target_dir: str, commit_id: str = None) -> bool: """Clone a GitHub repository to the target directory, optionally checking out a specific commit.""" try: # Ensure target directory exists os.makedirs(os.path.dirname(target_dir), exist_ok=True) - + # If specific commit is requested, don't use shallow clone if commit_id: # Clone full repository to access specific commit - result = subprocess.run([ - 'git', 'clone', clone_url, target_dir - ], capture_output=True, text=True, timeout=WebAppConfig.CLONE_TIMEOUT) - + result = subprocess.run( + ["git", "clone", clone_url, target_dir], + capture_output=True, + text=True, + timeout=WebAppConfig.CLONE_TIMEOUT, + ) + if result.returncode != 0: print(f"Error cloning repository: {result.stderr}") return False - + # Checkout specific commit - result = subprocess.run([ - 'git', 'checkout', commit_id - ], cwd=target_dir, capture_output=True, text=True, timeout=30) - + result = subprocess.run( + ["git", "checkout", commit_id], + cwd=target_dir, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: print(f"Error checking out commit {commit_id}: {result.stderr}") return False else: # Clone repository with shallow depth (default behavior) - result = subprocess.run([ - 'git', 'clone', '--depth', str(WebAppConfig.CLONE_DEPTH), clone_url, target_dir - ], capture_output=True, text=True, timeout=WebAppConfig.CLONE_TIMEOUT) - + result = subprocess.run( + [ + "git", + "clone", + "--depth", + str(WebAppConfig.CLONE_DEPTH), + clone_url, + target_dir, + ], + capture_output=True, + text=True, + timeout=WebAppConfig.CLONE_TIMEOUT, + ) + if result.returncode != 0: print(f"Error cloning repository: {result.stderr}") return False - + return True except Exception as e: print(f"Error cloning repository: {e}") - return False \ No newline at end of file + return False diff --git a/codewiki/src/fe/models.py b/codewiki/src/fe/models.py index 253d736..b8d0301 100644 --- a/codewiki/src/fe/models.py +++ b/codewiki/src/fe/models.py @@ -11,11 +11,13 @@ class RepositorySubmission(BaseModel): """Pydantic model for repository submission form.""" + repo_url: HttpUrl class JobStatusResponse(BaseModel): """Pydantic model for job status API response.""" + job_id: str repo_url: str status: str @@ -32,6 +34,7 @@ class JobStatusResponse(BaseModel): @dataclass class JobStatus: """Tracks the status of a documentation generation job.""" + job_id: str repo_url: str status: str # 'queued', 'processing', 'completed', 'failed' @@ -48,8 +51,9 @@ class JobStatus: @dataclass class CacheEntry: """Represents a cached documentation result.""" + repo_url: str repo_url_hash: str docs_path: str created_at: datetime - last_accessed: datetime \ No newline at end of file + last_accessed: datetime diff --git a/codewiki/src/fe/routes.py b/codewiki/src/fe/routes.py index 3e1a8fd..d84b2df 100644 --- a/codewiki/src/fe/routes.py +++ b/codewiki/src/fe/routes.py @@ -24,45 +24,43 @@ class WebRoutes: """Handles all web routes for the application.""" - + def __init__(self, background_worker: BackgroundWorker, cache_manager: CacheManager): self.background_worker = background_worker self.cache_manager = cache_manager - + async def index_get(self, request: Request) -> HTMLResponse: """Main page with form for submitting GitHub repositories.""" # Clean up old jobs before displaying # self.cleanup_old_jobs() - + # Get recent jobs (last 10) all_jobs = self.background_worker.get_all_jobs() - recent_jobs = sorted( - all_jobs.values(), - key=lambda x: x.created_at, - reverse=True - )[:100] - + recent_jobs = sorted(all_jobs.values(), key=lambda x: x.created_at, reverse=True)[:100] + context = { "message": None, "message_type": None, "repo_url": "", "commit_id": "", - "recent_jobs": recent_jobs + "recent_jobs": recent_jobs, } - + return HTMLResponse(content=render_template(WEB_INTERFACE_TEMPLATE, context)) - - async def index_post(self, request: Request, repo_url: str = Form(...), commit_id: str = Form("")) -> HTMLResponse: + + async def index_post( + self, request: Request, repo_url: str = Form(...), commit_id: str = Form("") + ) -> HTMLResponse: """Handle repository submission.""" # Clean up old jobs before processing self.cleanup_old_jobs() - + message = None message_type = None - + repo_url = repo_url.strip() commit_id = commit_id.strip() if commit_id else "" - + if not repo_url: message = "Please enter a GitHub repository URL" message_type = "error" @@ -72,26 +70,28 @@ async def index_post(self, request: Request, repo_url: str = Form(...), commit_i else: # Normalize the repo URL for comparison normalized_repo_url = self._normalize_github_url(repo_url) - + # Get repo info for job ID generation repo_info = GitHubRepoProcessor.get_repo_info(normalized_repo_url) - job_id = self._repo_full_name_to_job_id(repo_info['full_name']) - + job_id = self._repo_full_name_to_job_id(repo_info["full_name"]) + # Check if already in queue, processing, or recently failed existing_job = self.background_worker.get_job_status(job_id) recent_cutoff = datetime.now() - timedelta(minutes=WebAppConfig.RETRY_COOLDOWN_MINUTES) - + if existing_job: - if existing_job.status in ['queued', 'processing']: + if existing_job.status in ["queued", "processing"]: pass # Will handle below - elif existing_job.status == 'failed' and existing_job.created_at > recent_cutoff: + elif existing_job.status == "failed" and existing_job.created_at > recent_cutoff: pass # Will handle below else: existing_job = None # Job is old or completed, can reuse - + if existing_job: - if existing_job.status in ['queued', 'processing']: - message = f"Repository is already being processed (Job ID: {existing_job.job_id})" + if existing_job.status in ["queued", "processing"]: + message = ( + f"Repository is already being processed (Job ID: {existing_job.job_id})" + ) else: message = f"Repository recently failed processing. Please wait a few minutes before retrying (Job ID: {existing_job.job_id})" message_type = "error" @@ -105,12 +105,12 @@ async def index_post(self, request: Request, repo_url: str = Form(...), commit_i job = JobStatus( job_id=job_id, repo_url=normalized_repo_url, # Use normalized URL - status='completed', + status="completed", created_at=datetime.now(), completed_at=datetime.now(), docs_path=cached_docs, progress="Retrieved from cache", - commit_id=commit_id if commit_id else None + commit_id=commit_id if commit_id else None, ) self.background_worker.job_status[job_id] = job else: @@ -119,72 +119,70 @@ async def index_post(self, request: Request, repo_url: str = Form(...), commit_i job = JobStatus( job_id=job_id, repo_url=normalized_repo_url, # Use normalized URL - status='queued', + status="queued", created_at=datetime.now(), progress="Waiting in queue...", - commit_id=commit_id if commit_id else None + commit_id=commit_id if commit_id else None, ) - + self.background_worker.add_job(job_id, job) message = f"Repository added to processing queue! Job ID: {job_id}" message_type = "success" repo_url = "" # Clear form - + except Exception as e: message = f"Failed to add repository to queue: {str(e)}\n{format_exc()}" message_type = "error" - + # Get recent jobs (last 10) all_jobs = self.background_worker.get_all_jobs() - recent_jobs = sorted( - all_jobs.values(), - key=lambda x: x.created_at, - reverse=True - ) - + recent_jobs = sorted(all_jobs.values(), key=lambda x: x.created_at, reverse=True) + context = { "message": message, "message_type": message_type, "repo_url": repo_url or "", "commit_id": commit_id or "", - "recent_jobs": recent_jobs + "recent_jobs": recent_jobs, } - + return HTMLResponse(content=render_template(WEB_INTERFACE_TEMPLATE, context)) - + async def get_job_status(self, job_id: str) -> JobStatusResponse: """API endpoint to get job status.""" job = self.background_worker.get_job_status(job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - + return JobStatusResponse(**asdict(job)) - + async def view_docs(self, job_id: str) -> RedirectResponse: """View generated documentation.""" job = self.background_worker.get_job_status(job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - - if job.status != 'completed' or not job.docs_path: + + if job.status != "completed" or not job.docs_path: raise HTTPException(status_code=404, detail="Documentation not available") - + docs_path = Path(job.docs_path) if not docs_path.exists(): raise HTTPException(status_code=404, detail="Documentation files not found") - + # Redirect to the documentation viewer return RedirectResponse(url=f"/static-docs/{job_id}/", status_code=status.HTTP_302_FOUND) - - async def serve_generated_docs(self, job_id: str, filename: str = "overview.md") -> HTMLResponse: + + async def serve_generated_docs( + self, job_id: str, filename: str = "overview.md" + ) -> HTMLResponse: """Serve generated documentation files.""" job = self.background_worker.get_job_status(job_id) docs_path = None repo_url = None - + if job: # Job status exists - use it - if job.status != 'completed' or not job.docs_path: + if job.status != "completed" or not job.docs_path: raise HTTPException(status_code=404, detail="Documentation not available") docs_path = Path(job.docs_path) repo_url = job.repo_url @@ -193,42 +191,43 @@ async def serve_generated_docs(self, job_id: str, filename: str = "overview.md") # Convert job_id back to repo full name and construct potential paths repo_full_name = self._job_id_to_repo_full_name(job_id) potential_repo_url = f"https://github.com/{repo_full_name}" - + # Check if documentation exists in cache cached_docs = self.cache_manager.get_cached_docs(potential_repo_url) if cached_docs and Path(cached_docs).exists(): docs_path = Path(cached_docs) repo_url = potential_repo_url - + # Recreate job status for consistency job = JobStatus( job_id=job_id, repo_url=potential_repo_url, - status='completed', + status="completed", created_at=datetime.now(), completed_at=datetime.now(), docs_path=cached_docs, progress="Loaded from cache", - commit_id=None # No commit info available from cache + commit_id=None, # No commit info available from cache ) self.background_worker.job_status[job_id] = job self.background_worker.save_job_statuses() else: raise HTTPException(status_code=404, detail="Documentation not found") - + if not docs_path or not docs_path.exists(): raise HTTPException(status_code=404, detail="Documentation files not found") - + # Load module tree module_tree = None from codewiki.src.config import meta_resolve + module_tree_file = Path(meta_resolve(docs_path, "module_tree.json")) if module_tree_file.exists(): try: module_tree = file_manager.load_json(module_tree_file) except Exception: pass - + # Load metadata metadata = None metadata_file = Path(meta_resolve(docs_path, "metadata.json")) @@ -237,22 +236,22 @@ async def serve_generated_docs(self, job_id: str, filename: str = "overview.md") metadata = file_manager.load_json(metadata_file) except Exception: pass - + # Serve the requested file file_path = docs_path / filename if not file_path.exists(): raise HTTPException(status_code=404, detail=f"File {filename} not found") - + try: content = file_manager.load_text(file_path) - + # Convert markdown to HTML (reuse from visualise_docs.py) from .visualise_docs import markdown_to_html, get_file_title from .templates import DOCS_VIEW_TEMPLATE - + html_content = markdown_to_html(content) title = get_file_title(file_path) - + context = { "repo_name": repo_url.split("/")[-1], "title": title, @@ -260,14 +259,16 @@ async def serve_generated_docs(self, job_id: str, filename: str = "overview.md") "navigation": module_tree, "current_page": filename, "job_id": job_id, - "metadata": metadata + "metadata": metadata, } - + return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - + except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading {filename}: {e}\n{format_exc()}") - + raise HTTPException( + status_code=500, detail=f"Error reading {filename}: {e}\n{format_exc()}" + ) + def _normalize_github_url(self, url: str) -> str: """Normalize GitHub URL for consistent comparison.""" try: @@ -276,25 +277,26 @@ def _normalize_github_url(self, url: str) -> str: return f"https://github.com/{repo_info['full_name']}" except Exception: # Fallback to basic normalization - return url.rstrip('/').lower() - + return url.rstrip("/").lower() + def _repo_full_name_to_job_id(self, full_name: str) -> str: """Convert repo full name to URL-safe job ID.""" - return full_name.replace('/', '--') - + return full_name.replace("/", "--") + def _job_id_to_repo_full_name(self, job_id: str) -> str: """Convert job ID back to repo full name.""" - return job_id.replace('--', '/') - + return job_id.replace("--", "/") + def cleanup_old_jobs(self): """Clean up old job status entries.""" cutoff = datetime.now() - timedelta(hours=WebAppConfig.JOB_CLEANUP_HOURS) all_jobs = self.background_worker.get_all_jobs() expired_jobs = [ - job_id for job_id, job in all_jobs.items() - if job.created_at < cutoff and job.status in ['completed', 'failed'] + job_id + for job_id, job in all_jobs.items() + if job.created_at < cutoff and job.status in ["completed", "failed"] ] - + for job_id in expired_jobs: if job_id in self.background_worker.job_status: - del self.background_worker.job_status[job_id] \ No newline at end of file + del self.background_worker.job_status[job_id] diff --git a/codewiki/src/fe/template_utils.py b/codewiki/src/fe/template_utils.py index 4d9d7e7..98754ee 100644 --- a/codewiki/src/fe/template_utils.py +++ b/codewiki/src/fe/template_utils.py @@ -9,10 +9,10 @@ class StringTemplateLoader(BaseLoader): """Custom Jinja2 loader for string templates.""" - + def __init__(self, template_string: str): self.template_string = template_string - + def get_source(self, environment, template): return self.template_string, None, lambda: True @@ -20,41 +20,41 @@ def get_source(self, environment, template): def render_template(template: str, context: Dict[str, Any]) -> str: """ Render template using Jinja2. - + Args: template: HTML template string with Jinja2 syntax context: Dictionary of variables to substitute - + Returns: Rendered HTML string """ # Create Jinja2 environment with string template env = Environment( loader=StringTemplateLoader(template), - autoescape=select_autoescape(['html', 'xml']), + autoescape=select_autoescape(["html", "xml"]), trim_blocks=True, - lstrip_blocks=True + lstrip_blocks=True, ) - + # Get template and render - jinja_template = env.get_template('') + jinja_template = env.get_template("") return jinja_template.render(**context) def render_navigation(module_tree: Dict[str, Any], current_page: str = "") -> str: """ Render navigation HTML from module tree structure. - + Args: module_tree: Dictionary representing the module tree current_page: Current page filename for highlighting - + Returns: HTML string for navigation """ if not module_tree: return "" - + nav_template = """ {%- for section_key, section_data in module_tree.items() %}
@@ -72,26 +72,23 @@ def render_navigation(module_tree: Dict[str, Any], current_page: str = "") -> st
{%- endfor %} """ - - return render_template(nav_template, { - 'module_tree': module_tree, - 'current_page': current_page - }) + + return render_template(nav_template, {"module_tree": module_tree, "current_page": current_page}) def render_job_list(jobs: list) -> str: """ Render job list HTML. - + Args: jobs: List of job objects - + Returns: HTML string for job list """ if not jobs: return "" - + job_list_template = """ {%- for job in jobs %}
@@ -110,5 +107,5 @@ def render_job_list(jobs: list) -> str:
{%- endfor %} """ - - return render_template(job_list_template, {'jobs': jobs}) \ No newline at end of file + + return render_template(job_list_template, {"jobs": jobs}) diff --git a/codewiki/src/fe/templates.py b/codewiki/src/fe/templates.py index 763562c..c767654 100644 --- a/codewiki/src/fe/templates.py +++ b/codewiki/src/fe/templates.py @@ -677,4 +677,4 @@ -""" \ No newline at end of file +""" diff --git a/codewiki/src/fe/visualise_docs.py b/codewiki/src/fe/visualise_docs.py index 33c1779..6f59ab0 100644 --- a/codewiki/src/fe/visualise_docs.py +++ b/codewiki/src/fe/visualise_docs.py @@ -25,20 +25,25 @@ from .templates import DOCS_VIEW_TEMPLATE from codewiki.src.utils import file_manager -app = FastAPI(title="Documentation Server", description="Simple documentation server for hosting markdown documentation folders") +app = FastAPI( + title="Documentation Server", + description="Simple documentation server for hosting markdown documentation folders", +) # Global variables to store configuration DOCS_FOLDER = None MODULE_TREE = None + def initialize_globals(): """Initialize global variables from environment or command line args if not already set.""" global DOCS_FOLDER, MODULE_TREE - + if DOCS_FOLDER is None: # Try to get from environment variable or use a default import os - docs_folder_path = os.environ.get('DOCS_FOLDER') + + docs_folder_path = os.environ.get("DOCS_FOLDER") if docs_folder_path and Path(docs_folder_path).exists(): DOCS_FOLDER = docs_folder_path MODULE_TREE = load_module_tree(Path(docs_folder_path)) @@ -47,6 +52,7 @@ def initialize_globals(): # The FastAPI endpoints will need to check if DOCS_FOLDER is None pass + # Markdown parser md = MarkdownIt() @@ -54,11 +60,12 @@ def initialize_globals(): def load_module_tree(docs_folder: Path) -> Optional[Dict]: """Load the module tree structure from module_tree.json.""" from codewiki.src.config import meta_resolve + tree_file = Path(meta_resolve(docs_folder, "module_tree.json")) if not tree_file.exists(): print(f"Warning: module_tree.json not found in {docs_folder}/.meta/") return None - + try: return file_manager.load_json(tree_file) except Exception as e: @@ -70,24 +77,25 @@ def markdown_to_html(content: str) -> str: """Convert markdown content to HTML, with special handling for mermaid diagrams.""" # First, convert markdown to HTML html = md.render(content) - + # Post-process to ensure mermaid code blocks are properly formatted # Look for code blocks with language-mermaid class and convert them to mermaid divs import re - + # Pattern to match mermaid code blocks pattern = r'
(.*?)
' - + def replace_mermaid(match): mermaid_code = match.group(1) # Decode HTML entities that might have been encoded import html + mermaid_code = html.unescape(mermaid_code) return f'
{mermaid_code}
' - + # Replace mermaid code blocks with proper mermaid divs html = re.sub(pattern, replace_mermaid, html, flags=re.DOTALL) - + return html @@ -95,44 +103,49 @@ def get_file_title(file_path: Path) -> str: """Extract title from markdown file, fallback to filename.""" try: content = file_manager.load_text(file_path) - first_line = content.split('\n')[0].strip() - if first_line.startswith('# '): + first_line = content.split("\n")[0].strip() + if first_line.startswith("# "): return first_line[2:].strip() except Exception: pass - + # Fallback to filename without extension - return file_path.stem.replace('_', ' ').title() + return file_path.stem.replace("_", " ").title() @app.get("/", response_class=HTMLResponse) async def index(): """Serve the overview page as the main page.""" initialize_globals() - + if DOCS_FOLDER is None: - raise HTTPException(status_code=500, detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.") - + raise HTTPException( + status_code=500, + detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.", + ) + overview_file = Path(DOCS_FOLDER) / "overview.md" - + if not overview_file.exists(): - raise HTTPException(status_code=404, detail="overview.md not found in the documentation folder") - + raise HTTPException( + status_code=404, detail="overview.md not found in the documentation folder" + ) + try: content = file_manager.load_text(overview_file) - + html_content = markdown_to_html(content) title = get_file_title(overview_file) - + context = { "title": title, "content": html_content, "navigation": MODULE_TREE, - "current_page": "overview.md" + "current_page": "overview.md", } - + return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - + except Exception as e: raise HTTPException(status_code=500, detail=f"Error reading overview.md: {e}") @@ -141,16 +154,19 @@ async def index(): async def serve_doc(filename: str): """Serve individual documentation files.""" initialize_globals() - + if DOCS_FOLDER is None: - raise HTTPException(status_code=500, detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.") - + raise HTTPException( + status_code=500, + detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.", + ) + # Security check: ensure we're only serving .md files and they exist in the docs folder - if not filename.endswith('.md'): + if not filename.endswith(".md"): raise HTTPException(status_code=404, detail="Only markdown files are supported") - + file_path = Path(DOCS_FOLDER) / filename - + # Ensure the file is within the docs folder (prevent directory traversal) try: file_path = file_path.resolve() @@ -159,25 +175,25 @@ async def serve_doc(filename: str): raise HTTPException(status_code=403, detail="Access denied") except Exception: raise HTTPException(status_code=403, detail="Invalid file path") - + if not file_path.exists(): raise HTTPException(status_code=404, detail=f"File {filename} not found") - + try: content = file_manager.load_text(file_path) - + html_content = markdown_to_html(content) title = get_file_title(file_path) - + context = { "title": title, "content": html_content, "navigation": MODULE_TREE, - "current_page": filename + "current_page": filename, } - + return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - + except Exception as e: raise HTTPException(status_code=500, detail=f"Error reading {filename}: {e}") @@ -195,75 +211,70 @@ def main(): "--docs-folder", type=str, required=True, - help="Path to the documentation folder containing markdown files and module_tree.json" + help="Path to the documentation folder containing markdown files and module_tree.json", ) parser.add_argument( - "--port", - type=int, - default=8000, - help="Port to run the server on (default: 8000)" + "--port", type=int, default=8000, help="Port to run the server on (default: 8000)" ) parser.add_argument( "--host", type=str, default="127.0.0.1", - help="Host to bind the server to (default: 127.0.0.1)" - ) - parser.add_argument( - "--debug", - action="store_true", - help="Run the server in debug mode" + help="Host to bind the server to (default: 127.0.0.1)", ) - + parser.add_argument("--debug", action="store_true", help="Run the server in debug mode") + args = parser.parse_args() - + # Validate docs folder docs_folder = Path(args.docs_folder) if not docs_folder.exists(): print(f"Error: Documentation folder '{docs_folder}' does not exist") sys.exit(1) - + if not docs_folder.is_dir(): print(f"Error: '{docs_folder}' is not a directory") sys.exit(1) - + # Check for overview.md overview_file = docs_folder / "overview.md" if not overview_file.exists(): print(f"Warning: overview.md not found in '{docs_folder}'") - + # Set global variables and environment variable for uvicorn reload global DOCS_FOLDER, MODULE_TREE DOCS_FOLDER = str(docs_folder.resolve()) MODULE_TREE = load_module_tree(docs_folder) - + # Set environment variable so uvicorn reload can pick it up import os - os.environ['DOCS_FOLDER'] = DOCS_FOLDER - - print(f"📚 Starting documentation server...") + + os.environ["DOCS_FOLDER"] = DOCS_FOLDER + + print("📚 Starting documentation server...") print(f"📁 Documentation folder: {DOCS_FOLDER}") print(f"🌐 Server running at: http://{args.host}:{args.port}") - print(f"📖 Main page: overview.md") - + print("📖 Main page: overview.md") + if MODULE_TREE: modules_count = len(MODULE_TREE) print(f"🗂️ Found {modules_count} main modules in module_tree.json") - + print("\nPress Ctrl+C to stop the server") - + try: import uvicorn + uvicorn.run( "visualise_docs:app", host=args.host, port=args.port, reload=args.debug, - log_level="debug" if args.debug else "info" + log_level="debug" if args.debug else "info", ) except KeyboardInterrupt: print("\n👋 Server stopped") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/codewiki/src/fe/web_app.py b/codewiki/src/fe/web_app.py index 6f5d846..e09da28 100644 --- a/codewiki/src/fe/web_app.py +++ b/codewiki/src/fe/web_app.py @@ -22,19 +22,14 @@ # Initialize FastAPI app app = FastAPI( - title="CodeWiki", - description="Generate comprehensive documentation for any GitHub repository" + title="CodeWiki", description="Generate comprehensive documentation for any GitHub repository" ) # Initialize components cache_manager = CacheManager( - cache_dir=WebAppConfig.CACHE_DIR, - cache_expiry_days=WebAppConfig.CACHE_EXPIRY_DAYS -) -background_worker = BackgroundWorker( - cache_manager=cache_manager, - temp_dir=WebAppConfig.TEMP_DIR + cache_dir=WebAppConfig.CACHE_DIR, cache_expiry_days=WebAppConfig.CACHE_EXPIRY_DAYS ) +background_worker = BackgroundWorker(cache_manager=cache_manager, temp_dir=WebAppConfig.TEMP_DIR) web_routes = WebRoutes(background_worker=background_worker, cache_manager=cache_manager) @@ -67,7 +62,7 @@ async def view_docs(job_id: str): @app.get("/static-docs/{job_id}/{filename:path}") async def serve_generated_docs(job_id: str, filename: str = "overview.md"): """Serve generated documentation files.""" - if not filename: + if not filename: filename = "overview.md" return await web_routes.serve_generated_docs(job_id, filename) @@ -75,7 +70,7 @@ async def serve_generated_docs(job_id: str, filename: str = "overview.md"): def main(): """Main function to run the web application.""" import uvicorn - + parser = argparse.ArgumentParser( description="CodeWiki Web Application - Generate documentation for GitHub repositories" ) @@ -83,46 +78,38 @@ def main(): "--host", type=str, default=WebAppConfig.DEFAULT_HOST, - help=f"Host to bind the server to (default: {WebAppConfig.DEFAULT_HOST})" + help=f"Host to bind the server to (default: {WebAppConfig.DEFAULT_HOST})", ) parser.add_argument( "--port", type=int, default=WebAppConfig.DEFAULT_PORT, - help=f"Port to run the server on (default: {WebAppConfig.DEFAULT_PORT})" - ) - parser.add_argument( - "--debug", - action="store_true", - help="Run the server in debug mode" - ) - parser.add_argument( - "--reload", - action="store_true", - help="Enable auto-reload for development" + help=f"Port to run the server on (default: {WebAppConfig.DEFAULT_PORT})", ) - + parser.add_argument("--debug", action="store_true", help="Run the server in debug mode") + parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development") + args = parser.parse_args() - + # Ensure required directories exist WebAppConfig.ensure_directories() - + # Start background worker background_worker.start() - - print(f"🚀 CodeWiki Web Application starting...") + + print("🚀 CodeWiki Web Application starting...") print(f"🌐 Server running at: http://{args.host}:{args.port}") print(f"📁 Cache directory: {WebAppConfig.get_absolute_path(WebAppConfig.CACHE_DIR)}") print(f"🗂️ Temp directory: {WebAppConfig.get_absolute_path(WebAppConfig.TEMP_DIR)}") print("\nPress Ctrl+C to stop the server") - + try: uvicorn.run( "fe.web_app:app", host=args.host, port=args.port, reload=args.reload, - log_level="debug" if args.debug else "info" + log_level="debug" if args.debug else "info", ) except KeyboardInterrupt: print("\n👋 Server stopped") @@ -130,4 +117,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/codewiki/src/frontmatter.py b/codewiki/src/frontmatter.py index 4ffef0a..1bb23c0 100644 --- a/codewiki/src/frontmatter.py +++ b/codewiki/src/frontmatter.py @@ -27,25 +27,55 @@ # OKF v0.2 standard top-level keys (see okf/SPEC.md §4/§5/§7). # Anything else written by producers should live under ``metadata``. -_OKF_STANDARD_KEYS = frozenset({ - "type", "title", "aliases", "description", - "status", "verified", "stale_after", "generated", - "tags", "sources", -}) +_OKF_STANDARD_KEYS = frozenset( + { + "type", + "title", + "aliases", + "description", + "status", + "verified", + "stale_after", + "generated", + "tags", + "sources", + } +) # Producer-private keys that were historically written at the top level and # must be folded under ``metadata`` by this helper. Kept here so lint rules # and other consumers share one definition of "private". -PRIVATE_FRONTMATTER_KEYS = frozenset({ - "resource", "generated_from", "category", "domain", "version", - "format", "decision", "decided_at", "severity", "root_cause", - "captured_at", "content_hash", "turn_count", "link_to", - "source_session", "keep_raw", "task_id", - # Note-specific fields historically written at the top level (notes/) - "date", "summary", "keywords", "origin", - "related_modules", "related_components", - "source_ref", "source_refs", "chunk_refs", -}) +PRIVATE_FRONTMATTER_KEYS = frozenset( + { + "resource", + "generated_from", + "category", + "domain", + "version", + "format", + "decision", + "decided_at", + "severity", + "root_cause", + "captured_at", + "content_hash", + "turn_count", + "link_to", + "source_session", + "keep_raw", + "task_id", + # Note-specific fields historically written at the top level (notes/) + "date", + "summary", + "keywords", + "origin", + "related_modules", + "related_components", + "source_ref", + "source_refs", + "chunk_refs", + } +) def _utc_now_iso() -> str: @@ -62,6 +92,7 @@ def _stale_after_iso(stale_days: Optional[int]) -> Optional[str]: def _default_actor() -> str: try: from codewiki.src.config import actor_id + return actor_id() except Exception: return "codewiki" @@ -73,12 +104,16 @@ def _schema_defaults(output_dir: Optional[Path] = None) -> Dict[str, Any]: Falls back to the generator's built-in defaults when the schema file is missing or unparseable. """ - defaults: Dict[str, Any] = {"default_stale_days": 90, "okf_tags": ["codewiki", "auto-generated"]} + defaults: Dict[str, Any] = { + "default_stale_days": 90, + "okf_tags": ["codewiki", "auto-generated"], + } if output_dir is None: return defaults try: from codewiki.src.config import SCHEMA_FILENAME import yaml + schema_path = output_dir / SCHEMA_FILENAME if not schema_path.is_file(): return defaults diff --git a/codewiki/src/utils.py b/codewiki/src/utils.py index 688a565..f3092c4 100644 --- a/codewiki/src/utils.py +++ b/codewiki/src/utils.py @@ -7,19 +7,20 @@ # ---------------------- File Manager --------------------- # ------------------------------------------------------------ + class FileManager: """Handles file I/O operations.""" - + @staticmethod def ensure_directory(path: str) -> None: """Create directory if it doesn't exist.""" os.makedirs(path, exist_ok=True) - + @staticmethod def save_json(data: Any, filepath: str) -> None: """Save data as JSON to file.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, indent=4, ensure_ascii=False) @staticmethod @@ -28,19 +29,20 @@ def load_json(filepath: str) -> Optional[Dict[str, Any]]: if not os.path.exists(filepath): return None - with open(filepath, 'r', encoding='utf-8') as f: + with open(filepath, "r", encoding="utf-8") as f: return json.load(f) @staticmethod def save_text(content: str, filepath: str) -> None: """Save text content to file.""" - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: f.write(content) @staticmethod def load_text(filepath: str) -> str: """Load text content from file.""" - with open(filepath, 'r', encoding='utf-8') as f: + with open(filepath, "r", encoding="utf-8") as f: return f.read() + file_manager = FileManager() diff --git "a/docs/LLM-Wiki-\346\211\251\345\261\225\346\226\271\346\241\210.md" "b/docs/LLM-Wiki-\346\211\251\345\261\225\346\226\271\346\241\210.md" index 8c2ccd4..59b68b8 100644 --- "a/docs/LLM-Wiki-\346\211\251\345\261\225\346\226\271\346\241\210.md" +++ "b/docs/LLM-Wiki-\346\211\251\345\261\225\346\226\271\346\241\210.md" @@ -254,19 +254,19 @@ def resolve_wiki_path(output_dir: str, schema: dict) -> dict: """所有 wiki 文件路径的唯一权威来源""" wiki = os.path.join(output_dir, "wiki") paths = { - "modules": os.path.join(wiki, "modules"), - "entities": os.path.join(wiki, "entities"), - "concepts": os.path.join(wiki, "concepts"), - "sources": os.path.join(wiki, "sources"), - "comparisons": os.path.join(wiki, "comparisons"), - "queries": os.path.join(wiki, "queries"), - "notes": os.path.join(output_dir, "notes"), - "raw_sources": os.path.join(output_dir, "raw", "sources"), - "index": os.path.join(wiki, "index.md"), - "log": os.path.join(wiki, "log.md"), - "overview": os.path.join(wiki, "overview.md"), - "schema": os.path.join(output_dir, "schema.yaml"), - "purpose": os.path.join(output_dir, "purpose.md"), + "modules": os.path.join(wiki, "modules"), + "entities": os.path.join(wiki, "entities"), + "concepts": os.path.join(wiki, "concepts"), + "sources": os.path.join(wiki, "sources"), + "comparisons": os.path.join(wiki, "comparisons"), + "queries": os.path.join(wiki, "queries"), + "notes": os.path.join(output_dir, "notes"), + "raw_sources": os.path.join(output_dir, "raw", "sources"), + "index": os.path.join(wiki, "index.md"), + "log": os.path.join(wiki, "log.md"), + "overview": os.path.join(wiki, "overview.md"), + "schema": os.path.join(output_dir, "schema.yaml"), + "purpose": os.path.join(output_dir, "purpose.md"), } # 从 schema.page_types 覆盖目录（用户可自定义） for ptype, config in schema.get("page_types", {}).items(): @@ -317,7 +317,7 @@ def _resolve_doc_path(filename, page_type, output_dir, schema): ```python def _extract_source_refs(content: str) -> tuple[list[str], list[str]]: """从正文中提取源文件引用和行号引用""" - pattern = r'\[\^src:([^:]+):(\d+-\d+)\]' + pattern = r"\[\^src:([^:]+):(\d+-\d+)\]" source_refs = set() chunk_refs = [] for match in re.finditer(pattern, content): @@ -691,11 +691,11 @@ def compute_health_score(issues: list) -> int: """0-100 分，扣分项：""" score = 100 weights = { - "error": 10, # 每个 error -10 - "warning": 3, # 每个 warning -3 - "info": 1, # 每个 info -1 - "orphan": 2, # 每个孤立页 -2 - "stale_source": 8, # 每个过时源 -8 + "error": 10, # 每个 error -10 + "warning": 3, # 每个 warning -3 + "info": 1, # 每个 info -1 + "orphan": 2, # 每个孤立页 -2 + "stale_source": 8, # 每个过时源 -8 } for issue in issues: score -= weights.get(issue["check"], 1) diff --git "a/docs/OKF v0.2 \346\224\271\350\277\233 Backlog\357\274\210CodeWiki-P....md" "b/docs/OKF v0.2 \346\224\271\350\277\233 Backlog\357\274\210CodeWiki-P....md" index 61ead02..539a2bd 100644 --- "a/docs/OKF v0.2 \346\224\271\350\277\233 Backlog\357\274\210CodeWiki-P....md" +++ "b/docs/OKF v0.2 \346\224\271\350\277\233 Backlog\357\274\210CodeWiki-P....md" @@ -65,9 +65,11 @@ from datetime import datetime, timedelta, timezone _TZ_CST = timezone(timedelta(hours=8)) + def _now_iso() -> str: return datetime.now(_TZ_CST).strftime("%Y-%m-%dT%H:%M:%S+08:00") + def inject_okf_frontmatter( body: str, *, @@ -78,7 +80,7 @@ def inject_okf_frontmatter( sources: list[dict] | None = None, status: str = "draft", extra: dict | None = None, - stale_days: int | None = None, # None = 不写；0 = 当下过期 + stale_days: int | None = None, # None = 不写；0 = 当下过期 ) -> str: fm: dict = { "type": type_, @@ -356,10 +358,9 @@ def _check_okf_conformance(output_dir: Path): md_files = list(output_dir.rglob("*.md")) # 排除暂存/调试目录 md_files = [ - f for f in md_files - if "/.meta/" not in str(f) - and "/.trash/" not in str(f) - and "/.hook-debug/" not in str(f) + f + for f in md_files + if "/.meta/" not in str(f) and "/.trash/" not in str(f) and "/.hook-debug/" not in str(f) ] # 保留文件豁免 frontmatter 检查 for md_file in md_files: diff --git "a/docs/OKF-v0.2-\351\200\202\351\205\215\346\226\271\346\241\210.md" "b/docs/OKF-v0.2-\351\200\202\351\205\215\346\226\271\346\241\210.md" index 512efa5..46c47bb 100644 --- "a/docs/OKF-v0.2-\351\200\202\351\205\215\346\226\271\346\241\210.md" +++ "b/docs/OKF-v0.2-\351\200\202\351\205\215\346\226\271\346\241\210.md" @@ -88,9 +88,11 @@ receipt/attester 运行时协议（v0.2 规范自身已 defer 到下个版本） ```python ACTOR_NAME = "codewiki" + def actor_id() -> str: from codewiki import __version__ - return f"{ACTOR_NAME}/{__version__}" # e.g. codewiki/5.2.0 + + return f"{ACTOR_NAME}/{__version__}" # e.g. codewiki/5.2.0 ``` 所有 `generated.by` / 默认 `verified.by` 统一走 `actor_id()`，避免版本号散落。 diff --git "a/docs/articles/CodeWiki-Plus\347\263\273\345\210\2273\357\274\232\344\273\273\345\212\241\347\256\241\347\220\206\344\270\216\347\273\217\351\252\214\350\256\260\345\277\206\346\217\220\345\217\226-\346\212\200\346\234\257\350\247\206\350\247\222.md" "b/docs/articles/CodeWiki-Plus\347\263\273\345\210\2273\357\274\232\344\273\273\345\212\241\347\256\241\347\220\206\344\270\216\347\273\217\351\252\214\350\256\260\345\277\206\346\217\220\345\217\226-\346\212\200\346\234\257\350\247\206\350\247\222.md" index 8932784..d6ed253 100644 --- "a/docs/articles/CodeWiki-Plus\347\263\273\345\210\2273\357\274\232\344\273\273\345\212\241\347\256\241\347\220\206\344\270\216\347\273\217\351\252\214\350\256\260\345\277\206\346\217\220\345\217\226-\346\212\200\346\234\257\350\247\206\350\247\222.md" +++ "b/docs/articles/CodeWiki-Plus\347\263\273\345\210\2273\357\274\232\344\273\273\345\212\241\347\256\241\347\220\206\344\270\216\347\273\217\351\252\214\350\256\260\345\277\206\346\217\220\345\217\226-\346\212\200\346\234\257\350\247\206\350\247\222.md" @@ -88,9 +88,9 @@ def main() -> None: if payload.get("hook_event_name") != "sessionStart": return - out = _resolve_output_dir(payload) # 定位 repowiki/ - tasks = _load_active_tasks(out) # 读 tasks/.index.json，过滤 status=active - bindings = _render_bindings(out) # 读 .meta/task_bindings/*.json + out = _resolve_output_dir(payload) # 定位 repowiki/ + tasks = _load_active_tasks(out) # 读 tasks/.index.json，过滤 status=active + bindings = _render_bindings(out) # 读 .meta/task_bindings/*.json lines = ["## [task-memory] 会话开始：请先关联任务", ""] lines.append("当前进行中的任务：") @@ -98,8 +98,7 @@ def main() -> None: lines.append(f"- {t['title']}（task_id={t['task_id']}）") # 标题直接内联 # ... 绑定表 + 弹框指引 + 硬性执行顺序 ... - print(json.dumps({"continue": True, "additionalContext": "\n".join(lines)}, - ensure_ascii=False)) + print(json.dumps({"continue": True, "additionalContext": "\n".join(lines)}, ensure_ascii=False)) ``` 输出结构是 IDE hook 的标准协议：`additionalContext` 字段的内容会被注入到 Agent 本次会话的系统提示里。 @@ -253,10 +252,16 @@ flowchart TB ```python _SYSTEM_PROMPT_MARKERS = ( - "system prompt", "you are", "your role", "you must never", - "available tools", "tool names marked", ... + "system prompt", + "you are", + "your role", + "you must never", + "available tools", + "tool names marked", + ..., ) + def _is_system_prompt(text: str) -> bool: if len(text) <= 1500: return False diff --git "a/docs/codebase-memory-mcp\350\267\250\346\234\215\345\212\241\345\210\206\346\236\220-\346\272\220\347\240\201\345\200\237\351\211\264\345\210\206\346\236\220.md" "b/docs/codebase-memory-mcp\350\267\250\346\234\215\345\212\241\345\210\206\346\236\220-\346\272\220\347\240\201\345\200\237\351\211\264\345\210\206\346\236\220.md" index cbf1572..64b4401 100644 --- "a/docs/codebase-memory-mcp\350\267\250\346\234\215\345\212\241\345\210\206\346\236\220-\346\272\220\347\240\201\345\200\237\351\211\264\345\210\206\346\236\220.md" +++ "b/docs/codebase-memory-mcp\350\267\250\346\234\215\345\212\241\345\210\206\346\236\220-\346\272\220\347\240\201\345\200\237\351\211\264\345\210\206\346\236\220.md" @@ -187,14 +187,16 @@ def match_cross_service_routes(workspace_routes): for qn, route in workspace_routes.items(): if route.callers and route.handler: for caller in route.callers: - cross_links.append(CrossServiceLink( - source_repo=caller.repo, - source_func=caller.name, - target_repo=route.handler.repo, - target_func=route.handler.name, - route=qn, - protocol="HTTP" - )) + cross_links.append( + CrossServiceLink( + source_repo=caller.repo, + source_func=caller.name, + target_repo=route.handler.repo, + target_func=route.handler.name, + route=qn, + protocol="HTTP", + ) + ) return cross_links ``` @@ -211,16 +213,17 @@ def match_cross_service_routes(workspace_routes): ```python import re + def canon_path(path: str) -> str: """将各种框架的路径参数语法统一为 {}""" # :name (Express/Rails) → {} - path = re.sub(r':([a-zA-Z_]\w*)', '{}', path) + path = re.sub(r":([a-zA-Z_]\w*)", "{}", path) # {name} (Spring/Axum) → {} - path = re.sub(r'\{[^}]+\}', '{}', path) + path = re.sub(r"\{[^}]+\}", "{}", path) # 或 (Flask) → {} - path = re.sub(r'<[^>]+>', '{}', path) + path = re.sub(r"<[^>]+>", "{}", path) # ${...} (JS template) → {} - path = re.sub(r'\$\{[^}]+\}', '{}', path) + path = re.sub(r"\$\{[^}]+\}", "{}", path) return path ``` diff --git "a/docs/codewiki-vs-codebase-memory-\345\212\237\350\203\275\345\257\271\351\275\220\347\240\224\347\251\266.md" "b/docs/codewiki-vs-codebase-memory-\345\212\237\350\203\275\345\257\271\351\275\220\347\240\224\347\251\266.md" index 98c0f49..45776d2 100644 --- "a/docs/codewiki-vs-codebase-memory-\345\212\237\350\203\275\345\257\271\351\275\220\347\240\224\347\251\266.md" +++ "b/docs/codewiki-vs-codebase-memory-\345\212\237\350\203\275\345\257\271\351\275\220\347\240\224\347\251\266.md" @@ -225,7 +225,7 @@ CodeWiki-CN 是 LLM Wiki 生成器：tree-sitter 解析代码 → LLM 生成 Mar ```python class Node(BaseModel): id, name, component_type, file_path, relative_path - depends_on: Set[str] # 唯一的"边"——扁平依赖集合 + depends_on: Set[str] # 唯一的"边"——扁平依赖集合 source_code, start_line, end_line has_docstring, docstring, parameters node_type, base_classes, class_name diff --git a/docs/plans/ember-bay-sparrow.md b/docs/plans/ember-bay-sparrow.md index d8784b4..5d0b1c6 100644 --- a/docs/plans/ember-bay-sparrow.md +++ b/docs/plans/ember-bay-sparrow.md @@ -38,6 +38,7 @@ def detect_services(repo_path: Path) -> Dict[str, Path]: ```python # 子服务检测 + 跨服务分析 from ...analysis.service_detector import detect_services + services = detect_services(repo_path) cross_service_info = {} if len(services) >= 2: @@ -46,7 +47,9 @@ if len(services) >= 2: # 2. 更新缓存中的 repo_name cache.batch_insert_routes(retagged_routes, incremental=False) # 3. 运行 CrossServiceMatcher - cross_service_info = _run_intra_repo_cross_service(repo_path, output_dir, services, retagged_routes) + cross_service_info = _run_intra_repo_cross_service( + repo_path, output_dir, services, retagged_routes + ) ``` 路由重分配逻辑（新增辅助函数 `_retag_routes_by_service`）： diff --git a/docs/plans/windy-mesa-stork.md b/docs/plans/windy-mesa-stork.md index 181ae50..aca811d 100644 --- a/docs/plans/windy-mesa-stork.md +++ b/docs/plans/windy-mesa-stork.md @@ -40,6 +40,7 @@ if session.docs_written > 0: try: from codewiki.mcp.tools.agents_md import write_agents_md + write_agents_md(session) except Exception: logger.debug("Failed to update AGENTS.md", exc_info=True) diff --git "a/docs/\350\267\250\346\234\215\345\212\241\350\260\203\347\224\250\345\210\206\346\236\220-\345\256\236\347\216\260\350\256\241\345\210\222.md" "b/docs/\350\267\250\346\234\215\345\212\241\350\260\203\347\224\250\345\210\206\346\236\220-\345\256\236\347\216\260\350\256\241\345\210\222.md" index 84cc2d7..7b6b5e0 100644 --- "a/docs/\350\267\250\346\234\215\345\212\241\350\260\203\347\224\250\345\210\206\346\236\220-\345\256\236\347\216\260\350\256\241\345\210\222.md" +++ "b/docs/\350\267\250\346\234\215\345\212\241\350\260\203\347\224\250\345\210\206\346\236\220-\345\256\236\347\216\260\350\256\241\345\210\222.md" @@ -58,26 +58,29 @@ ```python class RouteProtocol(str, Enum): HTTP = "http" - GRPC = "grpc" # Phase 2 - GRAPHQL = "graphql" # Phase 2 - MQ = "mq" # Phase 2 + GRPC = "grpc" # Phase 2 + GRAPHQL = "graphql" # Phase 2 + MQ = "mq" # Phase 2 + class RouteRole(str, Enum): - SERVER = "server" # 服务端路由处理器 - CLIENT = "client" # 客户端 HTTP 调用 + SERVER = "server" # 服务端路由处理器 + CLIENT = "client" # 客户端 HTTP 调用 + class RouteNode(BaseModel): - route_key: str # "__route__POST__/api/orders/{}" + route_key: str # "__route__POST__/api/orders/{}" protocol: RouteProtocol - method: Optional[str] # GET, POST, etc. - path: str # 规范化后的路径 + method: Optional[str] # GET, POST, etc. + path: str # 规范化后的路径 role: RouteRole - component_id: str # 关联的 Node ID + component_id: str # 关联的 Node ID repo_name: str file_path: str line_number: int = 0 framework: Optional[str] = None # fastapi, spring, express, etc. + class CrossServiceLink(BaseModel): route_key: str protocol: RouteProtocol @@ -89,6 +92,7 @@ class CrossServiceLink(BaseModel): server_component_id: str confidence: float = 1.0 + class WorkspaceTopology(BaseModel): repos: List[str] routes: List[RouteNode] @@ -105,14 +109,15 @@ class WorkspaceTopology(BaseModel): ```python def canonicalize_path(path: str) -> str: """统一路径参数语法：:id, {id}, , ${...} → {}""" - result = re.sub(r':([a-zA-Z_]\w*)', '{}', path) - result = re.sub(r'\{[^}]+\}', '{}', result) - result = re.sub(r'<[^>]+>', '{}', result) - result = re.sub(r'\$\{[^}]+\}', '{}', result) - if len(result) > 1 and result.endswith('/'): - result = result.rstrip('/') + result = re.sub(r":([a-zA-Z_]\w*)", "{}", path) + result = re.sub(r"\{[^}]+\}", "{}", result) + result = re.sub(r"<[^>]+>", "{}", result) + result = re.sub(r"\$\{[^}]+\}", "{}", result) + if len(result) > 1 and result.endswith("/"): + result = result.rstrip("/") return result + def make_route_key(method: str, path: str) -> str: return f"__route__{method.upper()}__{canonicalize_path(path)}" ``` @@ -293,12 +298,12 @@ class CrossServiceMatcher: ```python def path_matches_template(concrete: str, template: str) -> bool: """逐段比较：{} 段匹配任意非空段""" - c_segs = [s for s in concrete.split('/') if s] - t_segs = [s for s in template.split('/') if s] + c_segs = [s for s in concrete.split("/") if s] + t_segs = [s for s in template.split("/") if s] if len(c_segs) != len(t_segs): return False for c, t in zip(c_segs, t_segs): - if t == '{}': + if t == "{}": if not c: return False elif c != t: diff --git a/pyproject.toml b/pyproject.toml index a3f4b4b..c35a6a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,9 +70,9 @@ dev = [ "pytest>=7.4.0", "pytest-cov>=4.1.0", "pytest-asyncio>=0.21.0", - "black>=23.0.0", "mypy>=1.5.0", - "ruff>=0.1.0", + "ruff==0.16.3", + "pre-commit>=4.0.0", ] # Keep pip-compatible extra for `pip install -e .[dev]` fallback (uv canonical is dependency-groups above) @@ -81,9 +81,9 @@ dev = [ "pytest>=7.4.0", "pytest-cov>=4.1.0", "pytest-asyncio>=0.21.0", - "black>=23.0.0", "mypy>=1.5.0", - "ruff>=0.1.0", + "ruff==0.16.3", + "pre-commit>=4.0.0", ] [project.scripts] @@ -105,10 +105,6 @@ packages = ["codewiki"] managed = true default-groups = ["dev"] -[tool.black] -line-length = 100 -target-version = ['py312'] - [tool.mypy] python_version = "3.12" warn_return_any = true @@ -130,7 +126,11 @@ select = ["E4", "E7", "E9", "F"] # occurrences — relaxed in #17 until a dedicated cleanup pass. # E741/E731 were re-tightened after fixing all violations (post-#17 cleanup); # E501 is never active under this select, so it needs no ignore. -ignore = ["E701", "E702"] +# E402 (import not at top) is ignored globally because 9 files intentionally +# do sys.path manipulation before imports (e.g. codewiki/run_web_app.py:11-13, +# codewiki/cli/main.py:33). Fixing would require structural refactor, not +# low-value for pre-commit gate. +ignore = ["E701", "E702", "E402"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/repowiki/wiki/index.md b/repowiki/wiki/index.md index 6ba00d3..ec69ca6 100644 --- a/repowiki/wiki/index.md +++ b/repowiki/wiki/index.md @@ -6,7 +6,7 @@ aliases: - 知识笔记索引 --- - + # 项目文档索引 diff --git a/repowiki/wiki/modules/AnalysisPipeline.md b/repowiki/wiki/modules/AnalysisPipeline.md index cbcb3a7..08697dd 100644 --- a/repowiki/wiki/modules/AnalysisPipeline.md +++ b/repowiki/wiki/modules/AnalysisPipeline.md @@ -101,7 +101,9 @@ struct, _ = analyze_repository_structure_only("https://github.com/owner/repo") # 跨服务匹配 from codewiki.src.be.dependency_analyzer.analysis.cross_service_matcher import CrossServiceMatcher from codewiki.src.be.dependency_analyzer.analysis.topology_visualizer import TopologyVisualizer -m = CrossServiceMatcher(); m.add_repo_routes("svc-a", routes_a) + +m = CrossServiceMatcher() +m.add_repo_routes("svc-a", routes_a) topo = m.match() print(TopologyVisualizer().render_all(topo)) ``` diff --git a/repowiki/wiki/modules/AnalyzerModels.md b/repowiki/wiki/modules/AnalyzerModels.md index f0dd91d..0c8fa20 100644 --- a/repowiki/wiki/modules/AnalyzerModels.md +++ b/repowiki/wiki/modules/AnalyzerModels.md @@ -91,7 +91,8 @@ node = Node( file_path="src/api/orders.py", relative_path="src/api/orders.py", depends_on={"src/db/session.py::get_session"}, - start_line=10, end_line=42, + start_line=10, + end_line=42, language="python", component_id="src/api/orders.py::create_order", ) @@ -99,8 +100,14 @@ node = Node( result = AnalysisResult( repository=Repository(url=".", name="demo", clone_path="/tmp/demo", analysis_id="a1"), functions=[node], - relationships=[CallRelationship(caller=node.id, callee="src/db/session.py::get_session", call_line=21, is_resolved=True)], - file_tree={}, summary={"total_functions": 1}, visualization={}, + relationships=[ + CallRelationship( + caller=node.id, callee="src/db/session.py::get_session", call_line=21, is_resolved=True + ) + ], + file_tree={}, + summary={"total_functions": 1}, + visualization={}, ) print(result.model_dump_json()) ``` diff --git a/repowiki/wiki/modules/AnalyzerUtils.md b/repowiki/wiki/modules/AnalyzerUtils.md index 8a5efc7..1602930 100644 --- a/repowiki/wiki/modules/AnalyzerUtils.md +++ b/repowiki/wiki/modules/AnalyzerUtils.md @@ -116,13 +116,17 @@ flowchart TD ## 使用示例 ```python from codewiki.src.be.dependency_analyzer.utils import ( - external_symbols, path_canonicalizer, patterns, security, logging_config, + external_symbols, + path_canonicalizer, + patterns, + security, + logging_config, ) # 判定符号是否外部，避免垃圾边 -external_symbols.is_external_symbol("c", "printf") # True +external_symbols.is_external_symbol("c", "printf") # True external_symbols.is_external_symbol("cpp", "std::vector") # True -external_symbols.is_macro_name("MAX_SIZE") # True +external_symbols.is_macro_name("MAX_SIZE") # True # 路由键统一（Express 与 Spring 同一路径可比对） key = path_canonicalizer.make_route_key("GET", "/users/:id") diff --git a/repowiki/wiki/modules/CLI_Config.md b/repowiki/wiki/modules/CLI_Config.md index 87732c0..91603c1 100644 --- a/repowiki/wiki/modules/CLI_Config.md +++ b/repowiki/wiki/modules/CLI_Config.md @@ -148,27 +148,39 @@ flowchart TD ```python # 保存并校验配置 from codewiki.cli.config_manager import ConfigManager + cm = ConfigManager() -cm.save(api_key="sk-...", base_url="https://api.openai.com/v1", - main_model="gpt-4o", cluster_model="gpt-4o-mini") +cm.save( + api_key="sk-...", + base_url="https://api.openai.com/v1", + main_model="gpt-4o", + cluster_model="gpt-4o-mini", +) assert cm.is_configured() # 生成后端配置（桥接） from codewiki.src.config import Config + cfg = cm.get_config().to_backend_config( - repo_path="/repo", output_dir="docs", api_key=cm.get_api_key()) + repo_path="/repo", output_dir="docs", api_key=cm.get_api_key() +) # 创建文档分支并提交 from codewiki.cli.git_manager import GitManager + gm = GitManager("/repo") gm.create_documentation_branch(force=True) sha = gm.commit_documentation(Path("docs")) # 生成静态查看器 from codewiki.cli.html_generator import HTMLGenerator + HTMLGenerator().generate( - output_path=Path("docs/index.html"), title="My Repo", - docs_dir=Path("docs"), repository_url="https://github.com/u/r") + output_path=Path("docs/index.html"), + title="My Repo", + docs_dir=Path("docs"), + repository_url="https://github.com/u/r", +) ``` ## 扩展点 diff --git a/repowiki/wiki/modules/CLI_Utils.md b/repowiki/wiki/modules/CLI_Utils.md index 9fa9322..5f82065 100644 --- a/repowiki/wiki/modules/CLI_Utils.md +++ b/repowiki/wiki/modules/CLI_Utils.md @@ -131,8 +131,8 @@ from codewiki.cli.utils.fs import ensure_directory, safe_write from codewiki.cli.utils.progress import ProgressTracker from codewiki.cli.utils.api_errors import wrap_api_call -key = validate_api_key("sk-abcdef123456") # 通过 -print(mask_api_key(key)) # sk-ab...3456 +key = validate_api_key("sk-abcdef123456") # 通过 +print(mask_api_key(key)) # sk-ab...3456 out = ensure_directory("~/docs/wiki") safe_write(out / "overview.md", "# Wiki") diff --git a/repowiki/wiki/modules/GraphAndSort.md b/repowiki/wiki/modules/GraphAndSort.md index f017f6a..79ec1ce 100644 --- a/repowiki/wiki/modules/GraphAndSort.md +++ b/repowiki/wiki/modules/GraphAndSort.md @@ -106,15 +106,19 @@ components, leaf_nodes, routes = builder.build_dependency_graph() # 叶优先全序（底层依赖在前） from codewiki.src.be.dependency_analyzer.topo_sort import ( - build_graph_from_components, topological_sort + build_graph_from_components, + topological_sort, ) + graph = build_graph_from_components(components) -order = topological_sort(graph) # 文档生成按此顺序遍历 +order = topological_sort(graph) # 文档生成按此顺序遍历 # 文件变更影响分析 from codewiki.src.be.dependency_analyzer.topo_sort import ( - transitive_impact, resolve_files_to_components + transitive_impact, + resolve_files_to_components, ) + changed = resolve_files_to_components(components, ["src/api/handler.py"]) impact = transitive_impact(graph, set(changed), direction="depended_by", max_depth=5) ``` diff --git a/repowiki/wiki/modules/LLM_Backend.md b/repowiki/wiki/modules/LLM_Backend.md index ddf409e..0908e91 100644 --- a/repowiki/wiki/modules/LLM_Backend.md +++ b/repowiki/wiki/modules/LLM_Backend.md @@ -108,7 +108,7 @@ from codewiki.src.be.documentation_generator import DocumentationGenerator cfg = Config(repo_path="/path/to/repo", output_dir="/path/to/repo/wiki") gen = DocumentationGenerator(config=cfg, commit_id="abc123", no_cache=False) -await gen.run() # 生成各模块 .md + overview.md + metadata.json +await gen.run() # 生成各模块 .md + overview.md + metadata.json ``` ## 扩展点 diff --git a/repowiki/wiki/modules/MCP_Tools_Dependency.md b/repowiki/wiki/modules/MCP_Tools_Dependency.md index c6adb8d..86f45d1 100644 --- a/repowiki/wiki/modules/MCP_Tools_Dependency.md +++ b/repowiki/wiki/modules/MCP_Tools_Dependency.md @@ -100,11 +100,7 @@ flowchart TD result = handle_list_components({"mode": "summary"}) # 查询跨服务调用并追踪路由 -result = handle_query_cross_service({ - "service": "auth-service", - "method": "POST", - "path": "/login" -}) +result = handle_query_cross_service({"service": "auth-service", "method": "POST", "path": "/login"}) # 列出模块依赖 result = handle_list_dependencies({"target": "codewiki.core"}) diff --git a/repowiki/wiki/modules/RouteExtractors.md b/repowiki/wiki/modules/RouteExtractors.md index 3b74d19..21519c0 100644 --- a/repowiki/wiki/modules/RouteExtractors.md +++ b/repowiki/wiki/modules/RouteExtractors.md @@ -101,9 +101,11 @@ flowchart LR ## 使用示例 ```python from codewiki.src.be.dependency_analyzer.analyzers.route_extractors import get_extractor -from codewiki.src.be.dependency_analyzer.analyzers.route_extractors.mq_patterns import extract_mq_routes +from codewiki.src.be.dependency_analyzer.analyzers.route_extractors.mq_patterns import ( + extract_mq_routes, +) -extractor = get_extractor(".go") # -> extract_go_routes +extractor = get_extractor(".go") # -> extract_go_routes routes = extractor("svc/handler.go", src, "myrepo") routes += extract_mq_routes("svc/handler.go", src, "myrepo") for r in routes: diff --git a/repowiki/wiki/modules/SharedConfig.md b/repowiki/wiki/modules/SharedConfig.md index 095aa4a..9e362e8 100644 --- a/repowiki/wiki/modules/SharedConfig.md +++ b/repowiki/wiki/modules/SharedConfig.md @@ -83,10 +83,11 @@ cfg = from_cli_args( print(cfg.output_dir, cfg.main_model) # 元数据路径统一解析 -config_path = cfg.meta_resolve("config.json") # -> .../wiki/.meta/config.json +config_path = cfg.meta_resolve("config.json") # -> .../wiki/.meta/config.json # 文件读写经单例 from codewiki.src.utils import file_manager + file_manager.save_text(config_path, "# wiki config") data = file_manager.load_json(cfg.meta_resolve("metadata.json")) ``` diff --git a/scripts/_tmp_analyze_deleted.py b/scripts/_tmp_analyze_deleted.py index 18c0f8f..341aa16 100644 --- a/scripts/_tmp_analyze_deleted.py +++ b/scripts/_tmp_analyze_deleted.py @@ -1,7 +1,12 @@ -import subprocess, re, collections +import subprocess +import re +import collections + out = subprocess.run( ["git", "-c", "core.quotepath=false", "status", "--porcelain"], - capture_output=True, text=True, encoding="utf-8", + capture_output=True, + text=True, + encoding="utf-8", ).stdout deleted = [] for line in out.splitlines(): @@ -15,8 +20,13 @@ no_show = 0 for p in deleted: gitpath = p.replace("\\", "/") - r = subprocess.run(["git", "show", "HEAD:" + gitpath], - capture_output=True, text=True, encoding="utf-8", errors="replace") + r = subprocess.run( + ["git", "show", "HEAD:" + gitpath], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) if r.returncode != 0: no_show += 1 continue diff --git a/scripts/backfill_aliases.py b/scripts/backfill_aliases.py index 2738338..d36c355 100644 --- a/scripts/backfill_aliases.py +++ b/scripts/backfill_aliases.py @@ -71,7 +71,7 @@ def backfill_file(path: Path) -> bool: title = data.get("title") or path.stem aliases = [title] if isinstance(title, str) else [path.stem] - alias_line = f"aliases: [{', '.join('\"' + a.replace('\"', '\\\\\"') + '\"' for a in aliases)}]" + alias_line = f"aliases: [{', '.join('"' + a.replace('"', '\\\\"') + '"' for a in aliases)}]" lines = fm_text.split("\n") # Insert right after the opening delimiter block (end of first non-empty @@ -80,9 +80,7 @@ def backfill_file(path: Path) -> bool: new_fm = "\n".join(lines + [alias_line]) new_content = f"---\n{new_fm}\n---{body}" - fd, tmp_path = tempfile.mkstemp( - dir=str(path.parent), prefix=path.stem + ".", suffix=".tmp" - ) + fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), prefix=path.stem + ".", suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: f.write(new_content) diff --git a/scripts/migrate_freshness.py b/scripts/migrate_freshness.py index a2eef8f..3fcf1d3 100644 --- a/scripts/migrate_freshness.py +++ b/scripts/migrate_freshness.py @@ -82,7 +82,7 @@ def split_frontmatter(text: str): end = text.find("---", 3) if end < 0: return None, None - return text[3:end], text[end + 3:] + return text[3:end], text[end + 3 :] def parse_day(value: Any) -> Optional[datetime]: @@ -204,7 +204,9 @@ def main() -> int: parser = argparse.ArgumentParser( description="Backfill stale_after from historical verified events (idempotent)." ) - parser.add_argument("output_dir", help="repowiki output directory (contains notes/ and schema.yaml)") + parser.add_argument( + "output_dir", help="repowiki output directory (contains notes/ and schema.yaml)" + ) parser.add_argument("--dry-run", action="store_true", help="report without writing") args = parser.parse_args() diff --git a/scripts/migrate_okf.py b/scripts/migrate_okf.py index d2ddbe4..7c96c94 100644 --- a/scripts/migrate_okf.py +++ b/scripts/migrate_okf.py @@ -71,13 +71,14 @@ def split_frontmatter(content: str): data = None except Exception: data = None - return data, fm_lines, lines[i + 1:], i + return data, fm_lines, lines[i + 1 :], i return None, [], lines, None def actor_id() -> str: try: from codewiki.src.config import actor_id as _aid + return _aid() except Exception: return "codewiki" @@ -106,6 +107,7 @@ def title_of(body_lines, fallback: str) -> str: # YAML double-quoted scalar legal escape starters (§11 frontmatter must parse). _VALID_ESCAPE = re.compile(r'\\(?!["\\nrt0abfveN_LPxuU ])') + def repair_double_quoted_escapes(fm_lines: list) -> list: """Repair invalid YAML escapes in single-line ``key: "value"`` rows. @@ -126,8 +128,9 @@ def repair_double_quoted_escapes(fm_lines: list) -> list: return out -def migrate_file(path: Path, output_dir: Path, stale_days: int, dry_run: bool, - fold_private: bool = False) -> list: +def migrate_file( + path: Path, output_dir: Path, stale_days: int, dry_run: bool, fold_private: bool = False +) -> list: """Migrate one markdown file. Returns list of change descriptions.""" changes = [] content = path.read_text(encoding="utf-8") @@ -165,9 +168,16 @@ def migrate_file(path: Path, output_dir: Path, stale_days: int, dry_run: bool, # and later full regenerations therefore keep reading them unchanged. if fold_private and isinstance(data, dict): _OKF_STANDARD = { - "type", "title", "aliases", "description", - "status", "verified", "stale_after", "generated", - "tags", "sources", + "type", + "title", + "aliases", + "description", + "status", + "verified", + "stale_after", + "generated", + "tags", + "sources", } meta = dict(data.get("metadata") or {}) folded = [k for k in data if k not in _OKF_STANDARD and k != "metadata"] @@ -277,7 +287,9 @@ def migrate_file(path: Path, output_dir: Path, stale_days: int, dry_run: bool, new_content = re.sub( rf"^(status:\s*){old}(\s*)$", rf"\g<1>{new}\g<2>", - new_content, count=1, flags=re.MULTILINE, + new_content, + count=1, + flags=re.MULTILINE, ) if not dry_run: path.write_text(new_content, encoding="utf-8") @@ -312,15 +324,22 @@ def main() -> int: ap = argparse.ArgumentParser(description="Migrate a CodeWiki repowiki to OKF v0.2 conformance.") ap.add_argument("output_dir", help="repowiki output directory (contains wiki/ and schema.yaml)") ap.add_argument("--dry-run", action="store_true", help="report changes without writing") - ap.add_argument("--stale-days", type=int, default=90, help="days until stale_after (default 90)") - ap.add_argument("--fold-private", action="store_true", - help="fold producer-private top-level keys under metadata (OKF §4/§5)") + ap.add_argument( + "--stale-days", type=int, default=90, help="days until stale_after (default 90)" + ) + ap.add_argument( + "--fold-private", + action="store_true", + help="fold producer-private top-level keys under metadata (OKF §4/§5)", + ) args = ap.parse_args() output_dir = Path(args.output_dir).expanduser().resolve() wiki_dir = output_dir / "wiki" if not wiki_dir.is_dir(): - print(f"ERROR: {wiki_dir} not found — is this a repowiki output directory?", file=sys.stderr) + print( + f"ERROR: {wiki_dir} not found — is this a repowiki output directory?", file=sys.stderr + ) return 1 targets = [] @@ -346,9 +365,11 @@ def main() -> int: index_path = wiki_dir / "index.md" if index_path.is_file() and ensure_okf_version(index_path, args.dry_run): changed += 1 - print(f"{'[dry-run] ' if args.dry_run else ''}wiki/index.md: okf_version: \"0.2\"") + print(f'{"[dry-run] " if args.dry_run else ""}wiki/index.md: okf_version: "0.2"') - print(f"\nScanned {len(targets)} files, {'would change' if args.dry_run else 'changed'} {changed}.") + print( + f"\nScanned {len(targets)} files, {'would change' if args.dry_run else 'changed'} {changed}." + ) return 0 diff --git a/tests/conftest.py b/tests/conftest.py index 7457a11..b0826f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,7 +44,11 @@ def analyzed_repo(tmp_path): store = SessionStore() resp = json.loads( handle_analyze_repo( - {"repo_path": str(tmp_path), "output_dir": str(tmp_path / "repowiki"), "incremental": False}, + { + "repo_path": str(tmp_path), + "output_dir": str(tmp_path / "repowiki"), + "incremental": False, + }, store, ) ) diff --git a/tests/okf_regression_test.py b/tests/okf_regression_test.py index c789e79..8d55c03 100644 --- a/tests/okf_regression_test.py +++ b/tests/okf_regression_test.py @@ -88,10 +88,7 @@ def main(): # Minimal schema with OKF v0.2 conventions (output_dir / "schema.yaml").write_text( - "conventions:\n" - " okf_frontmatter: true\n" - ' okf_version: "0.2"\n' - " default_stale_days: 90\n", + 'conventions:\n okf_frontmatter: true\n okf_version: "0.2"\n default_stale_days: 90\n', encoding="utf-8", ) @@ -101,84 +98,158 @@ def main(): # ================================================================ print("[1] write_doc_file — OKF frontmatter 注入/补丁") - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_new.md", - "content": "# 新文档\n\n这是没有 frontmatter 的新文档。\n", - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_new.md", + "content": "# 新文档\n\n这是没有 frontmatter 的新文档。\n", + }, + store, + ) + ) + ) doc1 = Path(r.get("path", "")) check("write_doc_file", "新文档创建成功", r.get("status") == "created", str(r)[:200]) if doc1.exists(): fm1 = read_fm(doc1) check("write_doc_file", "注入 type", "type:" in fm1, fm1[:200]) - check("write_doc_file", "注入 generated", "generated:" in fm1 and "codewiki/" in fm1, fm1[:300]) + check( + "write_doc_file", + "注入 generated", + "generated:" in fm1 and "codewiki/" in fm1, + fm1[:300], + ) check("write_doc_file", "注入 stale_after(90d)", "stale_after:" in fm1, fm1[:300]) # 代码生成 wiki 场景默认 stable（OKF v0.2 生命周期） check("write_doc_file", "默认status=stable", "status: stable" in fm1, fm1[:300]) check("write_doc_file", "status仅注入一次", fm1.count("status:") == 1, fm1[:300]) # frontmatter_extra 显式 status 可覆盖默认 stable - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_draft_override.md", - "content": "# 覆盖测试\n\n显式指定 draft 状态。\n", - "frontmatter_extra": {"status": "draft"}, - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_draft_override.md", + "content": "# 覆盖测试\n\n显式指定 draft 状态。\n", + "frontmatter_extra": {"status": "draft"}, + }, + store, + ) + ) + ) over_doc = Path(r.get("path", "")) if over_doc.exists(): over_fm = read_fm(over_doc) - check("write_doc_file", "frontmatter_extra可覆盖为draft", - "status: draft" in over_fm, over_fm[:300]) + check( + "write_doc_file", + "frontmatter_extra可覆盖为draft", + "status: draft" in over_fm, + over_fm[:300], + ) check("write_doc_file", "覆盖时不重复注入", over_fm.count("status:") == 1, over_fm[:300]) # Agent-written frontmatter without type → patched (P0 fix) - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_agent.md", - "content": "---\ntitle: 自定义标题\ncustom_key: keep-me\n---\n# 代理文档\n\n正文。\n", - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_agent.md", + "content": "---\ntitle: 自定义标题\ncustom_key: keep-me\n---\n# 代理文档\n\n正文。\n", + }, + store, + ) + ) + ) doc2 = Path(r.get("path", "")) if doc2.exists(): fm2 = read_fm(doc2) check("write_doc_file", "已有frontmatter被补丁type", "type:" in fm2, fm2[:300]) check("write_doc_file", "保留代理自定义键", "custom_key: keep-me" in fm2, fm2[:300]) check("write_doc_file", "保留原标题", "自定义标题" in fm2, fm2[:300]) - check("write_doc_file", "补丁generated/stale_after", "generated:" in fm2 and "stale_after:" in fm2, fm2[:400]) + check( + "write_doc_file", + "补丁generated/stale_after", + "generated:" in fm2 and "stale_after:" in fm2, + fm2[:400], + ) check("write_doc_file", "缺失status补丁为stable", "status: stable" in fm2, fm2[:400]) # Agent-written frontmatter WITH type → untouched type, no dup - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_typed.md", - "content": "---\ntype: Concept\ntitle: 已有类型\n---\n# 已带类型\n\n正文。\n", - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_typed.md", + "content": "---\ntype: Concept\ntitle: 已有类型\n---\n# 已带类型\n\n正文。\n", + }, + store, + ) + ) + ) doc3 = Path(r.get("path", "")) if doc3.exists(): fm3 = read_fm(doc3) - check("write_doc_file", "已有type不重复注入", fm3.count("type:") == 1 and "Concept" in fm3, fm3[:300]) + check( + "write_doc_file", + "已有type不重复注入", + fm3.count("type:") == 1 and "Concept" in fm3, + fm3[:300], + ) # ================================================================ print("\n[2] edit_doc_file — frontmatter 存在时的编辑") - r = json.loads(asyncio.run(handle_edit_doc_file({ - "session_id": sid, "filename": "okf_new.md", - "command": "str_replace", "old_string": "# 新文档", "new_string": "# 新文档V2", - }, store))) + r = json.loads( + asyncio.run( + handle_edit_doc_file( + { + "session_id": sid, + "filename": "okf_new.md", + "command": "str_replace", + "old_string": "# 新文档", + "new_string": "# 新文档V2", + }, + store, + ) + ) + ) check("edit_doc_file", "str_replace成功", r.get("status") == "edited", str(r)[:200]) if doc1.exists(): - check("edit_doc_file", "frontmatter未被破坏", doc1.read_text(encoding="utf-8").startswith("---"), "") + check( + "edit_doc_file", + "frontmatter未被破坏", + doc1.read_text(encoding="utf-8").startswith("---"), + "", + ) check("edit_doc_file", "正文已更新", "# 新文档V2" in doc1.read_text(encoding="utf-8"), "") # ================================================================ print("\n[2b] doc_writer — frontmatter_extra 私有字段折叠 metadata") # session 模式：OKF 标准键留顶层，私有键(components/related_modules/date/severity/...)折叠 - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_fold.md", - "content": "# 折叠测试\n\n正文内容。\n", - "frontmatter_extra": { - "components": ["AuthService", "OrderService"], - "related_modules": ["auth", "order"], - "date": "2026-08-01", - "severity": "high", - "status": "stable", - "category": "backend", - }, - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_fold.md", + "content": "# 折叠测试\n\n正文内容。\n", + "frontmatter_extra": { + "components": ["AuthService", "OrderService"], + "related_modules": ["auth", "order"], + "date": "2026-08-01", + "severity": "high", + "status": "stable", + "category": "backend", + }, + }, + store, + ) + ) + ) fold_doc = Path(r.get("path", "")) if fold_doc.exists(): fmf = read_fm(fold_doc) @@ -189,21 +260,41 @@ def main(): print(" (fold yaml err:", e, ")") meta = fold_y.get("metadata") or {} check("doc_writer", "标准键status留在顶层", fold_y.get("status") == "stable", fmf[:400]) - check("doc_writer", "components折叠进metadata", - meta.get("components") == ["AuthService", "OrderService"], fmf[:600]) - check("doc_writer", "related_modules折叠进metadata", - meta.get("related_modules") == ["auth", "order"], fmf[:600]) + check( + "doc_writer", + "components折叠进metadata", + meta.get("components") == ["AuthService", "OrderService"], + fmf[:600], + ) + check( + "doc_writer", + "related_modules折叠进metadata", + meta.get("related_modules") == ["auth", "order"], + fmf[:600], + ) check("doc_writer", "date折叠进metadata", meta.get("date") == "2026-08-01", fmf[:600]) check("doc_writer", "severity折叠进metadata", meta.get("severity") == "high", fmf[:600]) check("doc_writer", "category折叠进metadata", meta.get("category") == "backend", fmf[:600]) - check("doc_writer", "顶层无components", - "components:" not in fmf.split("\nmetadata:")[0], fmf[:400]) + check( + "doc_writer", + "顶层无components", + "components:" not in fmf.split("\nmetadata:")[0], + fmf[:400], + ) # body 提取的 source_refs/chunk_refs 折叠进 metadata - r = json.loads(asyncio.run(handle_write_doc_file({ - "session_id": sid, "filename": "okf_srcfold.md", - "content": "# 来源折叠\n\n正文引用[^src:alpha:1-5]内容。\n", - }, store))) + r = json.loads( + asyncio.run( + handle_write_doc_file( + { + "session_id": sid, + "filename": "okf_srcfold.md", + "content": "# 来源折叠\n\n正文引用[^src:alpha:1-5]内容。\n", + }, + store, + ) + ) + ) src_doc = Path(r.get("path", "")) if src_doc.exists(): fms = read_fm(src_doc) @@ -213,30 +304,66 @@ def main(): src_y = {} print(" (src yaml err:", e, ")") src_meta = src_y.get("metadata") or {} - check("doc_writer", "source_refs折叠进metadata", - "alpha" in (src_meta.get("source_refs") or []), fms[:600]) - check("doc_writer", "chunk_refs折叠进metadata", - "alpha:1-5" in (src_meta.get("chunk_refs") or []), fms[:600]) - check("doc_writer", "顶层无source_refs", - not any(ln.startswith("source_refs:") for ln in fms.splitlines()), fms[:400]) + check( + "doc_writer", + "source_refs折叠进metadata", + "alpha" in (src_meta.get("source_refs") or []), + fms[:600], + ) + check( + "doc_writer", + "chunk_refs折叠进metadata", + "alpha:1-5" in (src_meta.get("chunk_refs") or []), + fms[:600], + ) + check( + "doc_writer", + "顶层无source_refs", + not any(ln.startswith("source_refs:") for ln in fms.splitlines()), + fms[:400], + ) # edit 后 _resync_source_refs 保持折叠 - r = json.loads(asyncio.run(handle_edit_doc_file({ - "session_id": sid, "filename": "okf_srcfold.md", - "command": "str_replace", "old_string": "# 来源折叠", "new_string": "# 来源折叠V2", - }, store))) + r = json.loads( + asyncio.run( + handle_edit_doc_file( + { + "session_id": sid, + "filename": "okf_srcfold.md", + "command": "str_replace", + "old_string": "# 来源折叠", + "new_string": "# 来源折叠V2", + }, + store, + ) + ) + ) if src_doc.exists(): fme = read_fm(src_doc) - check("doc_writer", "edit后source_refs仍折叠(缩进)", - any(ln.lstrip().startswith("source_refs:") and ln.startswith(" ") - for ln in fme.splitlines()), fme[:500]) - check("doc_writer", "edit后顶层无source_refs", - not any(ln.strip().startswith("source_refs:") for ln in fme.splitlines() - if not ln.startswith(" ")), fme[:400]) + check( + "doc_writer", + "edit后source_refs仍折叠(缩进)", + any( + ln.lstrip().startswith("source_refs:") and ln.startswith(" ") + for ln in fme.splitlines() + ), + fme[:500], + ) + check( + "doc_writer", + "edit后顶层无source_refs", + not any( + ln.strip().startswith("source_refs:") + for ln in fme.splitlines() + if not ln.startswith(" ") + ), + fme[:400], + ) # sessionless 模式 _inject_lightweight_frontmatter 同样折叠 light = _inject_lightweight_frontmatter( - "okf_light.md", "# 轻量\n\n正文。\n", + "okf_light.md", + "# 轻量\n\n正文。\n", page_type="module", frontmatter_extra={"components": ["X"], "status": "stable", "severity": "low"}, ) @@ -246,22 +373,49 @@ def main(): except Exception as e: light_y = {} print(" (light yaml err:", e, ")") - check("doc_writer", "sessionless标准键status顶层", light_y.get("status") == "stable", light_fm[:400]) + check( + "doc_writer", + "sessionless标准键status顶层", + light_y.get("status") == "stable", + light_fm[:400], + ) light_meta = light_y.get("metadata") or {} - check("doc_writer", "sessionless components折叠", light_meta.get("components") == ["X"], light_fm[:500]) - check("doc_writer", "sessionless severity折叠", light_meta.get("severity") == "low", light_fm[:500]) + check( + "doc_writer", + "sessionless components折叠", + light_meta.get("components") == ["X"], + light_fm[:500], + ) + check( + "doc_writer", + "sessionless severity折叠", + light_meta.get("severity") == "low", + light_fm[:500], + ) # knowledge_loop._extract_frontmatter 支持 metadata 回退 - _fm_folded = ("---\ntype: note\ntitle: X\nmetadata:\n" - " origin: human\n date: \"2026-08-01\"\n---\n\n正文。\n") - check("doc_writer", "_extract_frontmatter读折叠origin", - _extract_frontmatter(_fm_folded, "origin") == "human", - _extract_frontmatter(_fm_folded, "origin") or "") - check("doc_writer", "_extract_frontmatter读折叠date", - _extract_frontmatter(_fm_folded, "date") == "2026-08-01", - _extract_frontmatter(_fm_folded, "date") or "") - check("doc_writer", "_extract_frontmatter顶层优先", - _extract_frontmatter("---\ntitle: Y\norigin: top\n---\n\n正文。\n", "origin") == "top", "") + _fm_folded = ( + "---\ntype: note\ntitle: X\nmetadata:\n" + ' origin: human\n date: "2026-08-01"\n---\n\n正文。\n' + ) + check( + "doc_writer", + "_extract_frontmatter读折叠origin", + _extract_frontmatter(_fm_folded, "origin") == "human", + _extract_frontmatter(_fm_folded, "origin") or "", + ) + check( + "doc_writer", + "_extract_frontmatter读折叠date", + _extract_frontmatter(_fm_folded, "date") == "2026-08-01", + _extract_frontmatter(_fm_folded, "date") or "", + ) + check( + "doc_writer", + "_extract_frontmatter顶层优先", + _extract_frontmatter("---\ntitle: Y\norigin: top\n---\n\n正文。\n", "origin") == "top", + "", + ) # ================================================================ print("\n[2c] migrate_okf --fold-private — 行手术折叠且不churn未动键") @@ -270,59 +424,89 @@ def main(): mig_src.mkdir(parents=True) mig_file = mig_src / "Legacy.md" mig_file.write_text( - '---\n' - 'type: Module\n' + "---\n" + "type: Module\n" 'title: "Legacy"\n' - 'status: candidate\n' + "status: candidate\n" "generated: { by: human:alice, at: '2026-08-01T00:00:00Z' }\n" - 'related_modules:\n' - ' - auth\n' - ' - order\n' - 'severity: high\n' - 'date: 2026-08-01\n' - 'category: backend\n' + "related_modules:\n" + " - auth\n" + " - order\n" + "severity: high\n" + "date: 2026-08-01\n" + "category: backend\n" 'source_refs: ["README_CN"]\n' - '---\n' - '\n' - '# Legacy\n\n正文。\n', + "---\n" + "\n" + "# Legacy\n\n正文。\n", encoding="utf-8", ) mig_changes = migrate_file(mig_file, mig_dir, 90, False, True) mig_fm = read_fm(mig_file) - check("migrate_okf", "私有键已折叠", "folded metadata:" in " ".join(mig_changes), str(mig_changes)) - check("migrate_okf", "顶层无私有键", - not any(ln.startswith(k + ":") for ln in mig_fm.splitlines() - for k in ("related_modules", "severity", "date", "category", "source_refs")), - mig_fm[:400]) + check( + "migrate_okf", "私有键已折叠", "folded metadata:" in " ".join(mig_changes), str(mig_changes) + ) + check( + "migrate_okf", + "顶层无私有键", + not any( + ln.startswith(k + ":") + for ln in mig_fm.splitlines() + for k in ("related_modules", "severity", "date", "category", "source_refs") + ), + mig_fm[:400], + ) try: mig_y = _yaml.safe_load(mig_fm) except Exception as e: mig_y = {} print(" (migrate yaml err:", e, ")") mig_meta = mig_y.get("metadata") or {} - check("migrate_okf", "metadata含related_modules", - mig_meta.get("related_modules") == ["auth", "order"], mig_fm[:600]) + check( + "migrate_okf", + "metadata含related_modules", + mig_meta.get("related_modules") == ["auth", "order"], + mig_fm[:600], + ) check("migrate_okf", "metadata含severity", mig_meta.get("severity") == "high", mig_fm[:600]) check("migrate_okf", "metadata含date", mig_meta.get("date") == "2026-08-01", mig_fm[:600]) - check("migrate_okf", "metadata含source_refs", - mig_meta.get("source_refs") == ["README_CN"], mig_fm[:600]) + check( + "migrate_okf", + "metadata含source_refs", + mig_meta.get("source_refs") == ["README_CN"], + mig_fm[:600], + ) check("migrate_okf", "未动键格式保持(title引号)", 'title: "Legacy"' in mig_fm, mig_fm[:300]) - check("migrate_okf", "未动键格式保持(generated flow映射)", - "generated: { by: human:alice" in mig_fm, mig_fm[:300]) - check("migrate_okf", "status映射candidate→draft", - mig_y.get("status") == "draft", mig_fm[:300]) - check("migrate_okf", "折叠值为单行JSON(行式读取兼容)", - any(ln.startswith(" source_refs: [") for ln in mig_fm.splitlines()), mig_fm[:400]) + check( + "migrate_okf", + "未动键格式保持(generated flow映射)", + "generated: { by: human:alice" in mig_fm, + mig_fm[:300], + ) + check("migrate_okf", "status映射candidate→draft", mig_y.get("status") == "draft", mig_fm[:300]) + check( + "migrate_okf", + "折叠值为单行JSON(行式读取兼容)", + any(ln.startswith(" source_refs: [") for ln in mig_fm.splitlines()), + mig_fm[:400], + ) check("migrate_okf", "补丁键stale_after已补齐", "stale_after" in mig_fm, mig_fm[:300]) mig_changes2 = migrate_file(mig_file, mig_dir, 90, False, True) check("migrate_okf", "幂等-二次运行无改动", not mig_changes2, str(mig_changes2)) # ================================================================ print("\n[3] ingest_note — 生命周期默认值与归一化") - r = json.loads(handle_ingest_note({ - "session_id": sid, "note_type": "decision", - "title": "OKF回归测试决策", "content": "这是OKF回归测试的决策笔记内容。", - }, store)) + r = json.loads( + handle_ingest_note( + { + "session_id": sid, + "note_type": "decision", + "title": "OKF回归测试决策", + "content": "这是OKF回归测试的决策笔记内容。", + }, + store, + ) + ) note1 = Path(r.get("note_path", "")) check("ingest_note", "默认status=draft", r.get("note_status") == "draft", str(r)[:200]) check("ingest_note", "返回draft提示", "draft" in r.get("hint", ""), str(r.get("hint"))[:150]) @@ -331,31 +515,62 @@ def main(): check("ingest_note", "写入generated", "generated:" in fm, fm[:300]) check("ingest_note", "写入stale_after", "stale_after:" in fm, fm[:300]) - r = json.loads(handle_ingest_note({ - "session_id": sid, "note_type": "lesson", - "title": "显式stable笔记", "content": "这条笔记显式指定stable状态。", - "status": "stable", - }, store)) + r = json.loads( + handle_ingest_note( + { + "session_id": sid, + "note_type": "lesson", + "title": "显式stable笔记", + "content": "这条笔记显式指定stable状态。", + "status": "stable", + }, + store, + ) + ) check("ingest_note", "显式status=stable接受", r.get("note_status") == "stable", str(r)[:200]) - r = json.loads(handle_ingest_note({ - "session_id": sid, "note_type": "pitfall", - "title": "旧词汇candidate笔记", "content": "这条笔记用旧词汇candidate。", - "status": "candidate", - }, store)) + r = json.loads( + handle_ingest_note( + { + "session_id": sid, + "note_type": "pitfall", + "title": "旧词汇candidate笔记", + "content": "这条笔记用旧词汇candidate。", + "status": "candidate", + }, + store, + ) + ) note3 = Path(r.get("note_path", "")) - check("ingest_note", "legacy candidate归一化为draft", r.get("note_status") == "draft", str(r)[:200]) + check( + "ingest_note", + "legacy candidate归一化为draft", + r.get("note_status") == "draft", + str(r)[:200], + ) # ================================================================ print("\n[4] confirm_note — stable升级与verified事件") - r = json.loads(handle_confirm_note({ - "session_id": sid, "note_file": note1.name, "by": "human:tester", - }, store)) + r = json.loads( + handle_confirm_note( + { + "session_id": sid, + "note_file": note1.name, + "by": "human:tester", + }, + store, + ) + ) check("confirm_note", "升级为stable", r.get("status") == "stable", str(r)[:200]) fm = read_fm(note1) - check("confirm_note", "记录verified事件(human:tester)", - "verified:" in fm and "human:tester" in fm, fm[:500]) + check( + "confirm_note", + "记录verified事件(human:tester)", + "verified:" in fm and "human:tester" in fm, + fm[:500], + ) import re as _re + m = _re.search(r"stale_after:\s*['\"]?(\d{4}-\d{2}-\d{2})", fm) renewed = False if m: @@ -373,29 +588,49 @@ def main(): legacy.write_text( "---\ntype: lesson\ntitle: 旧格式笔记\ndate: 2020-01-01\n" "status: confirmed\nverified: {by: human:old-reviewer, at: 2020-01-02T00:00:00Z}\n" - "---\n\n旧格式正文。\n", encoding="utf-8") + "---\n\n旧格式正文。\n", + encoding="utf-8", + ) r = json.loads(handle_confirm_note({"session_id": sid, "note_file": legacy.name}, store)) fm = read_fm(legacy) check("confirm_note", "legacy confirmed→stable", r.get("status") == "stable", str(r)[:200]) try: data = _yaml.safe_load(fm) vlist = data.get("verified") - ok = isinstance(vlist, list) and len(vlist) == 2 and \ - vlist[0].get("by") == "human:old-reviewer" and "codewiki/" in str(vlist[1].get("by", "")) + ok = ( + isinstance(vlist, list) + and len(vlist) == 2 + and vlist[0].get("by") == "human:old-reviewer" + and "codewiki/" in str(vlist[1].get("by", "")) + ) except Exception as e: ok, data = False, str(e) check("confirm_note", "bare verified映射转列表并追加", ok, str(data)[:300]) # ================================================================ print("\n[5] reject_note — deprecated标记") - r = json.loads(handle_ingest_note({ - "session_id": sid, "note_type": "general", - "title": "待否决笔记", "content": "这条笔记将被否决。", - }, store)) + r = json.loads( + handle_ingest_note( + { + "session_id": sid, + "note_type": "general", + "title": "待否决笔记", + "content": "这条笔记将被否决。", + }, + store, + ) + ) note4 = Path(r.get("note_path", "")) - r = json.loads(handle_reject_note({ - "session_id": sid, "note_file": note4.name, "reason": "回归测试否决", - }, store)) + r = json.loads( + handle_reject_note( + { + "session_id": sid, + "note_file": note4.name, + "reason": "回归测试否决", + }, + store, + ) + ) check("reject_note", "标记deprecated", r.get("status") == "deprecated", str(r)[:200]) fm = read_fm(note4) check("reject_note", "保留否决原因", "回归测试否决" in fm, fm[:400]) @@ -406,23 +641,35 @@ def main(): legacy2 = notes_dir / "2020-02-02-legacy-confirmed.md" legacy2.write_text( "---\ntype: decision\ntitle: 旧确认笔记\ndate: 2020-02-02\n" - "status: confirmed\ntags: [\"legacyquery\"]\n---\n\n旧确认笔记正文 uniqueword-zeta。\n", - encoding="utf-8") + 'status: confirmed\ntags: ["legacyquery"]\n---\n\n旧确认笔记正文 uniqueword-zeta。\n', + encoding="utf-8", + ) try: from codewiki.mcp.tools.wiki_search import build_full_index + build_full_index(output_dir, session=store.get(sid)) except Exception as e: print(" (index rebuild warning:", e, ")") - r = json.loads(handle_query_wiki({ - "session_id": sid, "query": "legacyquery 旧确认笔记", - }, store)) + r = json.loads( + handle_query_wiki( + { + "session_id": sid, + "query": "legacyquery 旧确认笔记", + }, + store, + ) + ) results = r.get("results", []) legacy_hits = [x for x in results if "旧确认笔记" in str(x.get("title", ""))] check("query_wiki", "legacy confirmed可检索", len(legacy_hits) > 0, str(results)[:300]) if legacy_hits: - check("query_wiki", "legacy confirmed无[unconfirmed]前缀", - "[unconfirmed]" not in legacy_hits[0].get("title", ""), str(legacy_hits[0])[:200]) + check( + "query_wiki", + "legacy confirmed无[unconfirmed]前缀", + "[unconfirmed]" not in legacy_hits[0].get("title", ""), + str(legacy_hits[0])[:200], + ) r = json.loads(handle_query_wiki({"session_id": sid, "query": "待否决笔记"}, store)) rej_hits = [x for x in r.get("results", []) if "待否决" in str(x.get("title", ""))] @@ -432,46 +679,88 @@ def main(): conf_hits = [x for x in r.get("results", []) if "OKF回归测试决策" in str(x.get("title", ""))] check("query_wiki", "stable笔记可检索", len(conf_hits) > 0, str(r.get("results"))[:300]) if conf_hits: - check("query_wiki", "trust_tier=human-reviewed", - conf_hits[0].get("trust_tier") == "human-reviewed", str(conf_hits[0])[:300]) + check( + "query_wiki", + "trust_tier=human-reviewed", + conf_hits[0].get("trust_tier") == "human-reviewed", + str(conf_hits[0])[:300], + ) # ================================================================ print("\n[7] ingest_source — 外部文档登记与sources注入") src_file = base / "ext_spec.md" src_file.write_text("# 外部规范\n\n这是外部规范文档的正文内容。\n", encoding="utf-8") rel_target = doc1.as_posix().split("/repowiki/", 1)[1] - r = json.loads(handle_ingest_source({ - "session_id": sid, "source_ref": str(src_file), "name": "ext-spec", - "source_type": "md", "description": "外部规范", - "related_pages": [rel_target], - }, store)) + r = json.loads( + handle_ingest_source( + { + "session_id": sid, + "source_ref": str(src_file), + "name": "ext-spec", + "source_type": "md", + "description": "外部规范", + "related_pages": [rel_target], + }, + store, + ) + ) check("ingest_source", "登记成功", r.get("status") == "ingested", str(r)[:200]) stored = output_dir / r.get("stored_at", "") check("ingest_source", "文件已入库", stored.exists(), str(stored)) if stored.exists(): fm = read_fm(stored) - check("ingest_source", "md源注入type/status/generated", - "type:" in fm and "status:" in fm and "generated:" in fm, fm[:400]) + check( + "ingest_source", + "md源注入type/status/generated", + "type:" in fm and "status:" in fm and "generated:" in fm, + fm[:400], + ) fm1 = read_fm(doc1) - check("ingest_source", "相关页面注入sources条目", - "sources:" in fm1 and "ext-spec" in fm1, fm1[:600]) + check( + "ingest_source", + "相关页面注入sources条目", + "sources:" in fm1 and "ext-spec" in fm1, + fm1[:600], + ) - r = json.loads(handle_ingest_source({ - "session_id": sid, "source_ref": str(src_file), "name": "ext-spec-dup", - }, store)) + r = json.loads( + handle_ingest_source( + { + "session_id": sid, + "source_ref": str(src_file), + "name": "ext-spec-dup", + }, + store, + ) + ) check("ingest_source", "重复内容检测", r.get("status") == "duplicate", str(r)[:200]) # ================================================================ print("\n[8] retract_source — dry_run与引用清理") - r = json.loads(handle_retract_source({ - "session_id": sid, "name": "ext-spec", "mode": "remove_refs", "dry_run": True, - }, store)) + r = json.loads( + handle_retract_source( + { + "session_id": sid, + "name": "ext-spec", + "mode": "remove_refs", + "dry_run": True, + }, + store, + ) + ) check("retract_source", "dry_run返回预览", r.get("status") == "dry_run", str(r)[:200]) check("retract_source", "dry_run预告清理refs", r.get("would_clean_refs", 0) >= 1, str(r)[:200]) - r = json.loads(handle_retract_source({ - "session_id": sid, "name": "ext-spec", "mode": "remove_refs", - }, store)) + r = json.loads( + handle_retract_source( + { + "session_id": sid, + "name": "ext-spec", + "mode": "remove_refs", + }, + store, + ) + ) check("retract_source", "撤回成功", r.get("status") == "retracted", str(r)[:200]) check("retract_source", "清理了页面sources引用", r.get("cleaned_refs", 0) >= 1, str(r)[:200]) fm1 = read_fm(doc1) @@ -483,18 +772,31 @@ def main(): print("\n[9] batch_ingest — 批量笔记+源") src2 = base / "batch_doc.md" src2.write_text("# 批量文档\n\n批量导入测试内容。\n", encoding="utf-8") - r = json.loads(handle_batch_ingest({ - "session_id": sid, - "items": [ - {"kind": "note", "note_type": "general", "title": "批量笔记一", - "content": "批量导入的笔记内容。"}, - {"kind": "source", "source_ref": str(src2), "name": "batch-doc"}, - ], - }, store)) + r = json.loads( + handle_batch_ingest( + { + "session_id": sid, + "items": [ + { + "kind": "note", + "note_type": "general", + "title": "批量笔记一", + "content": "批量导入的笔记内容。", + }, + {"kind": "source", "source_ref": str(src2), "name": "batch-doc"}, + ], + }, + store, + ) + ) check("batch_ingest", "批量完成", r.get("status") == "completed", str(r)[:200]) items = r.get("results", []) - check("batch_ingest", "2项全部ok", - len(items) == 2 and all(i.get("status") == "ok" for i in items), str(items)[:300]) + check( + "batch_ingest", + "2项全部ok", + len(items) == 2 and all(i.get("status") == "ok" for i in items), + str(items)[:300], + ) # ================================================================ print("\n[10] lint_wiki — okf_conformance 检测能力") @@ -502,11 +804,14 @@ def main(): bad_dir.mkdir(parents=True, exist_ok=True) (bad_dir / "bad_nofm.md").write_text("# 无frontmatter\n\n正文。\n", encoding="utf-8") (bad_dir / "bad_legacy.md").write_text( - "---\ntype: Module\nstatus: confirmed\n---\n# 旧状态\n", encoding="utf-8") + "---\ntype: Module\nstatus: confirmed\n---\n# 旧状态\n", encoding="utf-8" + ) (bad_dir / "bad_expired.md").write_text( - "---\ntype: Module\nstale_after: 2020-01-01\n---\n# 过期\n", encoding="utf-8") + "---\ntype: Module\nstale_after: 2020-01-01\n---\n# 过期\n", encoding="utf-8" + ) (bad_dir / "bad_verified.md").write_text( - "---\ntype: Module\nverified: oops-not-a-list\n---\n# 坏verified\n", encoding="utf-8") + "---\ntype: Module\nverified: oops-not-a-list\n---\n# 坏verified\n", encoding="utf-8" + ) r = json.loads(handle_lint_wiki({"session_id": sid, "checks": ["all"]}, store)) lint_data = json.loads(Path(r["file"]).read_text(encoding="utf-8")) if "file" in r else r @@ -515,16 +820,33 @@ def main(): check("lint_wiki", "含okf_conformance", "okf_conformance" in checks_run, str(checks_run)) okf_issues = [i for i in lint_data.get("issues", []) if i.get("check") == "okf_conformance"] kinds = {i.get("file", ""): i.get("message", "") for i in okf_issues} - check("lint_wiki", "检出无frontmatter(error)", - any(i.get("severity") == "error" and "bad_nofm" in i.get("file", "") for i in okf_issues), - str(kinds)[:300]) - check("lint_wiki", "检出legacy status(warning)", - any("bad_legacy" in i.get("file", "") and "Legacy status" in i.get("message", "") for i in okf_issues), - str(kinds)[:300]) - check("lint_wiki", "检出过期stale_after", - any("bad_expired" in i.get("file", "") for i in okf_issues), str(kinds)[:300]) - check("lint_wiki", "检出坏verified", - any("bad_verified" in i.get("file", "") for i in okf_issues), str(kinds)[:300]) + check( + "lint_wiki", + "检出无frontmatter(error)", + any(i.get("severity") == "error" and "bad_nofm" in i.get("file", "") for i in okf_issues), + str(kinds)[:300], + ) + check( + "lint_wiki", + "检出legacy status(warning)", + any( + "bad_legacy" in i.get("file", "") and "Legacy status" in i.get("message", "") + for i in okf_issues + ), + str(kinds)[:300], + ) + check( + "lint_wiki", + "检出过期stale_after", + any("bad_expired" in i.get("file", "") for i in okf_issues), + str(kinds)[:300], + ) + check( + "lint_wiki", + "检出坏verified", + any("bad_verified" in i.get("file", "") for i in okf_issues), + str(kinds)[:300], + ) for b in ("bad_nofm.md", "bad_legacy.md", "bad_expired.md", "bad_verified.md"): (bad_dir / b).unlink(missing_ok=True) @@ -550,17 +872,27 @@ def main(): print("\n[12] get_prompt — OKF v0.2 规范段") r = json.loads(handle_get_prompt({"prompt_type": "system_leaf", "session_id": sid}, store)) prompt_text = r.get("content", "") or json.dumps(r, ensure_ascii=False) - check("get_prompt", "含OKF v0.2合规段", "OKF (Open Knowledge Format) v0.2" in prompt_text, prompt_text[:200]) + check( + "get_prompt", + "含OKF v0.2合规段", + "OKF (Open Knowledge Format) v0.2" in prompt_text, + prompt_text[:200], + ) check("get_prompt", "含actor约定", "human:" in prompt_text, "") check("get_prompt", "含status词汇表", "draft | stable | deprecated" in prompt_text, "") check("get_prompt", "含stale_after说明", "stale_after" in prompt_text, "") # output_dir-only call (no session/repo_path): the most direct locator - r = json.loads(handle_get_prompt( - {"prompt_type": "system_leaf", "output_dir": str(output_dir)}, store)) + r = json.loads( + handle_get_prompt({"prompt_type": "system_leaf", "output_dir": str(output_dir)}, store) + ) prompt_text2 = r.get("content", "") or json.dumps(r, ensure_ascii=False) - check("get_prompt", "仅output_dir也注入OKF段", - "OKF (Open Knowledge Format) v0.2" in prompt_text2, prompt_text2[:200]) + check( + "get_prompt", + "仅output_dir也注入OKF段", + "OKF (Open Knowledge Format) v0.2" in prompt_text2, + prompt_text2[:200], + ) # ================================================================ print("\n[13] index/log — §8/§9 格式") @@ -568,7 +900,9 @@ def main(): check("index", "index.md存在", idx.exists(), "") if idx.exists(): it = idx.read_text(encoding="utf-8") - check("index", "frontmatter含okf_version", "okf_version" in read_fm(idx), read_fm(idx)[:150]) + check( + "index", "frontmatter含okf_version", "okf_version" in read_fm(idx), read_fm(idx)[:150] + ) check("index", "§8 bullet格式", "* [" in it, it[:300]) log = output_dir / "wiki" / "log.md" if log.exists(): @@ -580,12 +914,20 @@ def main(): # ================================================================ print("\n[14] schema_generator — 默认conventions") from codewiki.mcp.tools.schema_generator import generate_schema + gen_dir = base / "gen_schema" gen_dir.mkdir() schema = generate_schema("demo", {}, ["python"], gen_dir, ["core", "utils"]) conv = schema.get("conventions", {}) - check("schema_generator", "okf_version默认0.2", conv.get("okf_version") == "0.2", str(conv)[:200]) - check("schema_generator", "default_stale_days默认90", conv.get("default_stale_days") == 90, str(conv)[:200]) + check( + "schema_generator", "okf_version默认0.2", conv.get("okf_version") == "0.2", str(conv)[:200] + ) + check( + "schema_generator", + "default_stale_days默认90", + conv.get("default_stale_days") == 90, + str(conv)[:200], + ) # ================================================================ print("\n[15] close_session — 收尾") diff --git a/tests/smoke_test_mcp.py b/tests/smoke_test_mcp.py index a22484a..eb0b29a 100644 --- a/tests/smoke_test_mcp.py +++ b/tests/smoke_test_mcp.py @@ -55,27 +55,38 @@ def main(): # -- 1. analyze_repo -- print("[1] analyze_repo") - result = json.loads(handle_analyze_repo({ - "repo_path": REPO_PATH, - "output_dir": output_dir, - }, store)) + result = json.loads( + handle_analyze_repo( + { + "repo_path": REPO_PATH, + "output_dir": output_dir, + }, + store, + ) + ) check("returns session_id", "session_id" in result, str(result)[:200]) check("returns workspace_dir", "workspace_dir" in result, str(result.keys())) check("returns stats", "stats" in result, str(result.keys())) check("returns files", "files" in result, str(result.keys())) - check("stats has total_components", - "total_components" in result.get("stats", {}), - str(result.get("stats"))) - check("stats has total_leaf_nodes", - "total_leaf_nodes" in result.get("stats", {}), - str(result.get("stats"))) + check( + "stats has total_components", + "total_components" in result.get("stats", {}), + str(result.get("stats")), + ) + check( + "stats has total_leaf_nodes", + "total_leaf_nodes" in result.get("stats", {}), + str(result.get("stats")), + ) session_id = result.get("session_id") workspace_dir = result.get("workspace_dir") check("session_id is non-empty", session_id and len(session_id) == 12, str(session_id)) - check("workspace_dir exists on disk", - workspace_dir and Path(workspace_dir).is_dir(), - str(workspace_dir)) + check( + "workspace_dir exists on disk", + workspace_dir and Path(workspace_dir).is_dir(), + str(workspace_dir), + ) # -- 2. Workspace files + list_components -- print("\n[2] Workspace files + list_components") @@ -90,12 +101,21 @@ def main(): check("summary has languages", "languages" in summary, str(summary.keys())) # Use list_components tool to get component index - lc_result = json.loads(handle_list_components({ - "session_id": session_id, - }, store)) + lc_result = json.loads( + handle_list_components( + { + "session_id": session_id, + }, + store, + ) + ) check("list_components returns file", "file" in lc_result, str(lc_result.keys())[:200]) check("list_components returns total", "total" in lc_result, str(lc_result.keys())[:200]) - check("list_components total > 0", lc_result.get("total", 0) > 0, f"total={lc_result.get('total')}") + check( + "list_components total > 0", + lc_result.get("total", 0) > 0, + f"total={lc_result.get('total')}", + ) # Read the workspace file with full component list comp_list_file = Path(lc_result["file"]) @@ -106,20 +126,37 @@ def main(): check("component_index non-empty", len(comp_index) > 0, f"len={len(comp_index)}") if comp_index: first = comp_index[0] - check("component has id/type/file", - all(k in first for k in ("id", "type", "file")), - str(first.keys())) + check( + "component has id/type/file", + all(k in first for k in ("id", "type", "file")), + str(first.keys()), + ) # Test summary mode print("\n[2b] list_components summary mode") - summ_result = json.loads(handle_list_components({ - "session_id": session_id, - "summary": True, - }, store)) + summ_result = json.loads( + handle_list_components( + { + "session_id": session_id, + "summary": True, + }, + store, + ) + ) check("summary returns file", "file" in summ_result, str(summ_result.keys())[:200]) - check("summary returns total_files", "total_files" in summ_result, str(summ_result.keys())[:200]) - check("summary returns mode", summ_result.get("mode") == "summary", f"mode={summ_result.get('mode')}") - check("summary total_files > 0", summ_result.get("total_files", 0) > 0, f"total_files={summ_result.get('total_files')}") + check( + "summary returns total_files", "total_files" in summ_result, str(summ_result.keys())[:200] + ) + check( + "summary returns mode", + summ_result.get("mode") == "summary", + f"mode={summ_result.get('mode')}", + ) + check( + "summary total_files > 0", + summ_result.get("total_files", 0) > 0, + f"total_files={summ_result.get('total_files')}", + ) # Read the summary file and validate structure summ_file = Path(summ_result["file"]) @@ -137,22 +174,31 @@ def main(): # Summary should be smaller than full list summ_size = summ_file.stat().st_size full_size = comp_list_file.stat().st_size - check("summary smaller than full", summ_size < full_size, f"summary={summ_size}, full={full_size}") + check( + "summary smaller than full", summ_size < full_size, f"summary={summ_size}, full={full_size}" + ) # -- 3. read_code_components (writes to workspace files) -- print("\n[3] read_code_components") if comp_index: ids = [c["id"] for c in comp_index[:5]] - read_result = json.loads(handle_read_code_components({ - "session_id": session_id, - "component_ids": ids, - }, store)) + read_result = json.loads( + handle_read_code_components( + { + "session_id": session_id, + "component_ids": ids, + }, + store, + ) + ) check("returns written count", "written" in read_result, str(read_result.keys())) check("returns source_dir", "source_dir" in read_result, str(read_result.keys())) check("returns files mapping", "files" in read_result, str(read_result.keys())) - check("written == requested", - read_result.get("written") == len(ids), - f"written={read_result.get('written')}, requested={len(ids)}") + check( + "written == requested", + read_result.get("written") == len(ids), + f"written={read_result.get('written')}, requested={len(ids)}", + ) # Verify source files exist on disk source_dir = Path(read_result["source_dir"]) @@ -161,40 +207,63 @@ def main(): src_file = source_dir / fname if src_file.exists(): content = src_file.read_text(encoding="utf-8") - check(f"source file has content ({fname})", - len(content) > 0, f"empty: {fname}") - check(f"source file has header ({fname})", - "Component:" in content, f"no header: {fname[:50]}") + check(f"source file has content ({fname})", len(content) > 0, f"empty: {fname}") + check( + f"source file has header ({fname})", + "Component:" in content, + f"no header: {fname[:50]}", + ) break # just check first one # -- 4. read_code_components no cap (removed 20-component limit) -- print("\n[4] read_code_components no cap") if len(comp_index) > 20: many_ids = [c["id"] for c in comp_index[:30]] - many_result = json.loads(handle_read_code_components({ - "session_id": session_id, - "component_ids": many_ids, - }, store)) - check("no 20-component cap", - many_result.get("written") == 30, - f"written={many_result.get('written')}") + many_result = json.loads( + handle_read_code_components( + { + "session_id": session_id, + "component_ids": many_ids, + }, + store, + ) + ) + check( + "no 20-component cap", + many_result.get("written") == 30, + f"written={many_result.get('written')}", + ) # -- 5. write_doc_file path traversal guard -- print("\n[5] write_doc_file path traversal guard") - traversal_write = json.loads(asyncio.run(handle_write_doc_file_wrapper({ - "session_id": session_id, - "filename": "../../evil.md", - "content": "pwned", - }, store))) + traversal_write = json.loads( + asyncio.run( + handle_write_doc_file_wrapper( + { + "session_id": session_id, + "filename": "../../evil.md", + "content": "pwned", + }, + store, + ) + ) + ) check("rejects ../../evil.md", "error" in traversal_write, str(traversal_write)) # -- 6. write_doc_file normal write -- print("\n[6] write_doc_file normal write") - normal_write = json.loads(asyncio.run(handle_write_doc_file_wrapper({ - "session_id": session_id, - "filename": "test_doc.md", - "content": "# Test\n\n```mermaid\ngraph TD\n A[Hello] --> B[World]\n```\n", - }, store))) + normal_write = json.loads( + asyncio.run( + handle_write_doc_file_wrapper( + { + "session_id": session_id, + "filename": "test_doc.md", + "content": "# Test\n\n```mermaid\ngraph TD\n A[Hello] --> B[World]\n```\n", + }, + store, + ) + ) + ) check("creates test_doc.md", normal_write.get("status") == "created", str(normal_write)) # write_doc_file routes pages by page_type (default: wiki/modules/); use returned path doc_file = Path(normal_write.get("path") or (Path(output_dir) / "test_doc.md")) @@ -202,26 +271,42 @@ def main(): # -- 7. edit_doc_file str_replace -- print("\n[7] edit_doc_file str_replace") - edit_result = json.loads(asyncio.run(handle_edit_doc_file_wrapper({ - "session_id": session_id, - "filename": "test_doc.md", - "command": "str_replace", - "old_string": "# Test", - "new_string": "# Test Edited", - }, store))) + edit_result = json.loads( + asyncio.run( + handle_edit_doc_file_wrapper( + { + "session_id": session_id, + "filename": "test_doc.md", + "command": "str_replace", + "old_string": "# Test", + "new_string": "# Test Edited", + }, + store, + ) + ) + ) check("edits file", edit_result.get("status") == "edited", str(edit_result)) edited_content = doc_file.read_text() check("content updated", "# Test Edited" in edited_content, edited_content[:100]) # -- 8. edit_doc_file undo -- print("\n[8] edit_doc_file undo") - undo_result = json.loads(asyncio.run(handle_edit_doc_file_wrapper({ - "session_id": session_id, - "filename": "test_doc.md", - "command": "undo", - }, store))) + undo_result = json.loads( + asyncio.run( + handle_edit_doc_file_wrapper( + { + "session_id": session_id, + "filename": "test_doc.md", + "command": "undo", + }, + store, + ) + ) + ) check("undone", undo_result.get("status") == "undone", str(undo_result)) - check("mermaid_validation in undo", "mermaid_validation" in undo_result, str(undo_result.keys())) + check( + "mermaid_validation in undo", "mermaid_validation" in undo_result, str(undo_result.keys()) + ) undone_content = doc_file.read_text() check("content reverted", "# Test\n" in undone_content, undone_content[:100]) @@ -238,44 +323,75 @@ def main(): # -- 10. list_dependencies -- print("\n[10] list_dependencies (LLM Wiki)") - deps_result = json.loads(handle_list_dependencies({ - "session_id": session_id, - "direction": "both", - "limit": 10, - }, store)) - check("returns file or deps", "file" in deps_result or "dependencies" in deps_result, str(deps_result.keys())[:200]) + deps_result = json.loads( + handle_list_dependencies( + { + "session_id": session_id, + "direction": "both", + "limit": 10, + }, + store, + ) + ) + check( + "returns file or deps", + "file" in deps_result or "dependencies" in deps_result, + str(deps_result.keys())[:200], + ) # Data may be in workspace file (file-side-channel) if "file" in deps_result: deps_file = Path(deps_result["file"]) deps_data = json.loads(deps_file.read_text(encoding="utf-8")) - check("deps file has dependencies", "dependencies" in deps_data, str(deps_data.keys())[:200]) + check( + "deps file has dependencies", "dependencies" in deps_data, str(deps_data.keys())[:200] + ) check("returns total_deps", "total_deps" in deps_result, str(deps_result.keys())[:200]) if deps_data.get("dependencies"): first_dep = deps_data["dependencies"][0] - check("dep has source/target", "source" in first_dep and "target" in first_dep, str(first_dep.keys())) + check( + "dep has source/target", + "source" in first_dep and "target" in first_dep, + str(first_dep.keys()), + ) else: check("returns dependencies", "dependencies" in deps_result, str(deps_result.keys())[:200]) check("returns total_deps", "total_deps" in deps_result, str(deps_result.keys())[:200]) if deps_result.get("dependencies"): first_dep = deps_result["dependencies"][0] - check("dep has source/target", "source" in first_dep and "target" in first_dep, str(first_dep.keys())) + check( + "dep has source/target", + "source" in first_dep and "target" in first_dep, + str(first_dep.keys()), + ) # Module-level dependencies - deps_module = json.loads(handle_list_dependencies({ - "session_id": session_id, - "module_level": True, - "limit": 5, - }, store)) - check("module_level works", - "file" in deps_module or "pagination" in deps_module, - str(deps_module.keys())[:200]) + deps_module = json.loads( + handle_list_dependencies( + { + "session_id": session_id, + "module_level": True, + "limit": 5, + }, + store, + ) + ) + check( + "module_level works", + "file" in deps_module or "pagination" in deps_module, + str(deps_module.keys())[:200], + ) # -- 11. lint_wiki -- print("\n[11] lint_wiki (LLM Wiki)") - lint_result = json.loads(handle_lint_wiki({ - "session_id": session_id, - "checks": ["all"], - }, store)) + lint_result = json.loads( + handle_lint_wiki( + { + "session_id": session_id, + "checks": ["all"], + }, + store, + ) + ) check("returns total_issues", "total_issues" in lint_result, str(lint_result.keys())[:200]) check("returns by_severity", "by_severity" in lint_result, str(lint_result.keys())[:200]) check("returns summary", "summary" in lint_result, str(lint_result.keys())[:200]) @@ -284,32 +400,46 @@ def main(): lint_file = Path(lint_result["file"]) lint_data = json.loads(lint_file.read_text(encoding="utf-8")) check("returns issues list", "issues" in lint_data, str(lint_data.keys())[:200]) - check("checks_run includes all", - len(lint_data.get("checks_run", [])) > 0, - str(lint_data.get("checks_run"))) + check( + "checks_run includes all", + len(lint_data.get("checks_run", [])) > 0, + str(lint_data.get("checks_run")), + ) else: check("returns issues list", "issues" in lint_result, str(lint_result.keys())[:200]) - check("checks_run includes all", - len(lint_result.get("checks_run", [])) > 0, - str(lint_result.get("checks_run"))) + check( + "checks_run includes all", + len(lint_result.get("checks_run", [])) > 0, + str(lint_result.get("checks_run")), + ) # Lint without session (output_dir mode) - lint_nosess = json.loads(handle_lint_wiki({ - "output_dir": output_dir, - "checks": ["broken_links"], - }, store)) - check("lint works without session", - "total_issues" in lint_nosess, - str(lint_nosess.keys())[:200]) + lint_nosess = json.loads( + handle_lint_wiki( + { + "output_dir": output_dir, + "checks": ["broken_links"], + }, + store, + ) + ) + check( + "lint works without session", "total_issues" in lint_nosess, str(lint_nosess.keys())[:200] + ) # -- 12. ingest_note -- print("\n[12] ingest_note (LLM Wiki)") - note_result = json.loads(handle_ingest_note({ - "session_id": session_id, - "note_type": "decision", - "title": "Smoke test decision", - "content": "This is a test decision note for the smoke test. We chose to use MCP tools for documentation generation.", - }, store)) + note_result = json.loads( + handle_ingest_note( + { + "session_id": session_id, + "note_type": "decision", + "title": "Smoke test decision", + "content": "This is a test decision note for the smoke test. We chose to use MCP tools for documentation generation.", + }, + store, + ) + ) check("note ingested", note_result.get("status") == "ingested", str(note_result)) check("note_path exists", "note_path" in note_result, str(note_result.keys())[:200]) if note_result.get("note_path"): @@ -324,39 +454,55 @@ def main(): check("wiki index.md exists", wiki_index_path.exists(), str(wiki_index_path)) # Duplicate protection - note_dup = json.loads(handle_ingest_note({ - "session_id": session_id, - "note_type": "decision", - "title": "Smoke test decision", - "content": "This is a different content for duplicate detection.", - }, store)) - check("duplicate handled (still ingested)", - note_dup.get("status") == "ingested", - str(note_dup)) + note_dup = json.loads( + handle_ingest_note( + { + "session_id": session_id, + "note_type": "decision", + "title": "Smoke test decision", + "content": "This is a different content for duplicate detection.", + }, + store, + ) + ) + check("duplicate handled (still ingested)", note_dup.get("status") == "ingested", str(note_dup)) # -- 13. query_wiki -- print("\n[13] query_wiki (LLM Wiki)") - query_result = json.loads(handle_query_wiki({ - "session_id": session_id, - "query": "test decision MCP", - "include_notes": True, - }, store)) + query_result = json.loads( + handle_query_wiki( + { + "session_id": session_id, + "query": "test decision MCP", + "include_notes": True, + }, + store, + ) + ) check("returns results", "results" in query_result, str(query_result.keys())[:200]) - check("returns context_package", "context_package" in query_result, str(query_result.keys())[:200]) + check( + "returns context_package", "context_package" in query_result, str(query_result.keys())[:200] + ) check("returns keywords", "keywords" in query_result, str(query_result.keys())[:200]) # Should find the ingested note note_results = [r for r in query_result.get("results", []) if r.get("source") == "note"] - check("finds ingested note", len(note_results) > 0, - f"note results: {len(note_results)}, total: {len(query_result.get('results', []))}") + check( + "finds ingested note", + len(note_results) > 0, + f"note results: {len(note_results)}, total: {len(query_result.get('results', []))}", + ) # Query without session (output_dir mode) - query_nosess = json.loads(handle_query_wiki({ - "output_dir": output_dir, - "query": "test", - }, store)) - check("query works without session", - "results" in query_nosess, - str(query_nosess.keys())[:200]) + query_nosess = json.loads( + handle_query_wiki( + { + "output_dir": output_dir, + "query": "test", + }, + store, + ) + ) + check("query works without session", "results" in query_nosess, str(query_nosess.keys())[:200]) # -- 13b. OKF v0.2 lifecycle & conformance -- print("\n[13b] OKF v0.2 lifecycle & conformance") @@ -376,28 +522,45 @@ def main(): check("note has stale_after", "stale_after:" in okf_note, okf_note[:400]) # confirm_note promotes draft -> stable and records a verified event - confirm_result = json.loads(handle_confirm_note({ - "session_id": session_id, - "note_file": okf_note_path.name, - "by": "human:smoke-tester", - }, store)) - check("confirm_note returns stable", confirm_result.get("status") == "stable", str(confirm_result)) + confirm_result = json.loads( + handle_confirm_note( + { + "session_id": session_id, + "note_file": okf_note_path.name, + "by": "human:smoke-tester", + }, + store, + ) + ) + check( + "confirm_note returns stable", confirm_result.get("status") == "stable", str(confirm_result) + ) okf_note2 = okf_note_path.read_text(encoding="utf-8") check("note promoted to stable", "status: stable" in okf_note2, okf_note2[:400]) - check("note records verified event", - "verified:" in okf_note2 and "human:smoke-tester" in okf_note2, - okf_note2[:400]) + check( + "note records verified event", + "verified:" in okf_note2 and "human:smoke-tester" in okf_note2, + okf_note2[:400], + ) # reject_note marks the duplicate note as deprecated dup_note_path = Path(note_dup.get("note_path", "")) if dup_note_path.exists(): - reject_result = json.loads(handle_reject_note({ - "session_id": session_id, - "note_file": dup_note_path.name, - "reason": "smoke-test cleanup", - }, store)) - check("reject_note returns deprecated", - reject_result.get("status") == "deprecated", str(reject_result)) + reject_result = json.loads( + handle_reject_note( + { + "session_id": session_id, + "note_file": dup_note_path.name, + "reason": "smoke-test cleanup", + }, + store, + ) + ) + check( + "reject_note returns deprecated", + reject_result.get("status") == "deprecated", + str(reject_result), + ) dup_note = dup_note_path.read_text(encoding="utf-8") check("rejected note marked deprecated", "status: deprecated" in dup_note, dup_note[:400]) @@ -406,10 +569,15 @@ def main(): check("index.md declares okf_version", "okf_version" in okf_index, okf_index[:200]) # okf_conformance lint check runs standalone without error - lint_okf = json.loads(handle_lint_wiki({ - "output_dir": output_dir, - "checks": ["okf_conformance"], - }, store)) + lint_okf = json.loads( + handle_lint_wiki( + { + "output_dir": output_dir, + "checks": ["okf_conformance"], + }, + store, + ) + ) check("okf_conformance check runs", "total_issues" in lint_okf, str(lint_okf.keys())[:200]) # The freshly generated docs/notes are conformant: no errors expected okf_errors = 0 @@ -437,7 +605,9 @@ def main(): # -- 15. SessionStore thread safety -- print("\n[15] SessionStore thread safety") import threading + errors = [] + def worker(): try: for _ in range(20): @@ -446,6 +616,7 @@ def worker(): store.remove(s.session_id) except Exception as e: errors.append(str(e)) + threads = [threading.Thread(target=worker) for _ in range(5)] for t in threads: t.start() @@ -466,15 +637,27 @@ def worker(): print("\n[17] capture_conversation (team-memory fusion)") conv = [ {"role": "user", "content": "How do I add a new MCP tool?"}, - {"role": "assistant", "content": "Register it in registry.py via _register(Tool(...), ...)."}, + { + "role": "assistant", + "content": "Register it in registry.py via _register(Tool(...), ...).", + }, {"role": "user", "content": "Thanks, got it."}, ] - cap_result = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": conv, - "link_to": "registry.py", - }, store)) - check("capture_conversation returns captured", cap_result.get("status") == "captured", str(cap_result)) + cap_result = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv, + "link_to": "registry.py", + }, + store, + ) + ) + check( + "capture_conversation returns captured", + cap_result.get("status") == "captured", + str(cap_result), + ) check("capture reports turn_count", cap_result.get("turn_count") == 3, str(cap_result)) conv_path = Path(output_dir) / cap_result.get("stored_at", "") check("conversation file written to raw/", conv_path.exists(), str(conv_path)) @@ -485,15 +668,22 @@ def worker(): check("raw file records link_to", "link_to:" in conv_text, conv_text[:200]) # Deduplication: capturing the same conversation again yields duplicate - cap_dup = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": conv, - "link_to": "registry.py", - }, store)) + cap_dup = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv, + "link_to": "registry.py", + }, + store, + ) + ) check("duplicate capture detected", cap_dup.get("status") == "duplicate", str(cap_dup)) - check("only one raw conv file after dedup", - len(list((Path(output_dir) / "raw").glob("conv-*.md"))) == 1, - "expected exactly 1") + check( + "only one raw conv file after dedup", + len(list((Path(output_dir) / "raw").glob("conv-*.md"))) == 1, + "expected exactly 1", + ) # Session-scoped supersede: Stop / PreCompact re-fire the same IDE session # with a growing transcript. Re-capturing the same source_session_id must @@ -503,43 +693,70 @@ def worker(): {"role": "user", "content": "supersede check turn 1"}, {"role": "assistant", "content": "answer 1"}, ] - cap_s1 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": conv_s1, - "source_session_id": "ide-sess-supersede", - }, store)) - check("first session capture ok", - cap_s1.get("status") == "captured" and not cap_s1.get("superseded"), str(cap_s1)) + cap_s1 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv_s1, + "source_session_id": "ide-sess-supersede", + }, + store, + ) + ) + check( + "first session capture ok", + cap_s1.get("status") == "captured" and not cap_s1.get("superseded"), + str(cap_s1), + ) _n_after_s1 = len(list((Path(output_dir) / "raw").glob("conv-*.md"))) - cap_s2 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": conv_s1 + [{"role": "user", "content": "supersede check turn 2"}], - "source_session_id": "ide-sess-supersede", - }, store)) + cap_s2 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv_s1 + [{"role": "user", "content": "supersede check turn 2"}], + "source_session_id": "ide-sess-supersede", + }, + store, + ) + ) check("same-session recapture supersedes", cap_s2.get("superseded") is True, str(cap_s2)) - check("supersede keeps a single raw file", - len(list((Path(output_dir) / "raw").glob("conv-*.md"))) == _n_after_s1, - f"expected {_n_after_s1}") + check( + "supersede keeps a single raw file", + len(list((Path(output_dir) / "raw").glob("conv-*.md"))) == _n_after_s1, + f"expected {_n_after_s1}", + ) _sup_path = Path(output_dir) / cap_s2.get("stored_at", "") if _sup_path.exists(): - check("superseded file has the longer transcript", - "turn_count: 3" in _sup_path.read_text(encoding="utf-8"), - str(cap_s2)) + check( + "superseded file has the longer transcript", + "turn_count: 3" in _sup_path.read_text(encoding="utf-8"), + str(cap_s2), + ) # Identical re-capture of the superseded content still hits hash dedup - cap_s3 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": conv_s1 + [{"role": "user", "content": "supersede check turn 2"}], - "source_session_id": "ide-sess-supersede", - }, store)) + cap_s3 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv_s1 + [{"role": "user", "content": "supersede check turn 2"}], + "source_session_id": "ide-sess-supersede", + }, + store, + ) + ) check("identical recapture is a duplicate", cap_s3.get("status") == "duplicate", str(cap_s3)) # query_wiki should NOT surface raw captures (raw/ is excluded from index) - q_after = json.loads(handle_query_wiki({ - "output_dir": output_dir, - "query": "add a new MCP tool", - }, store)) + q_after = json.loads( + handle_query_wiki( + { + "output_dir": output_dir, + "query": "add a new MCP tool", + }, + store, + ) + ) raw_hits = [r for r in q_after.get("results", []) if "raw" in str(r.get("path", ""))] check("query_wiki excludes raw captures", len(raw_hits) == 0, f"raw hits: {len(raw_hits)}") @@ -549,40 +766,61 @@ def worker(): _raw_before = list((Path(output_dir) / "raw").glob("conv-*.md")) async def _fake_llm(prompt, system): - return json.dumps({ - "notes": [{ - "title": "Adding an MCP tool requires registry.py registration", - "note_type": "decision", - "related_modules": ["mcp"], - "tags": ["mcp"], - "content": "## Background\nUser asked how to add an MCP tool.\n## Decision\nRegister via _register(Tool(...), handler_path=..., mode='thread') in registry.py.", - }] - }) - - dist = json.loads(handle_distill_conversation({ - "output_dir": output_dir, - "llm": _fake_llm, - }, store)) + return json.dumps( + { + "notes": [ + { + "title": "Adding an MCP tool requires registry.py registration", + "note_type": "decision", + "related_modules": ["mcp"], + "tags": ["mcp"], + "content": "## Background\nUser asked how to add an MCP tool.\n## Decision\nRegister via _register(Tool(...), handler_path=..., mode='thread') in registry.py.", + } + ] + } + ) + + dist = json.loads( + handle_distill_conversation( + { + "output_dir": output_dir, + "llm": _fake_llm, + }, + store, + ) + ) check("distill returns completed", dist.get("status") == "completed", str(dist)) check("distill created >=1 note", dist.get("notes_created", 0) >= 1, str(dist)) if _raw_before: # raw files captured in [17] should be deleted after distillation _raw_after = list((Path(output_dir) / "raw").glob("conv-*.md")) - check("raw captures deleted after distill", len(_raw_after) == 0, - f"remaining: {[p.name for p in _raw_after]}") + check( + "raw captures deleted after distill", + len(_raw_after) == 0, + f"remaining: {[p.name for p in _raw_after]}", + ) # a draft note should now exist and be queryable with [unconfirmed] prefix - q_note = json.loads(handle_query_wiki({ - "output_dir": output_dir, - "query": "registry.py registration MCP tool", - }, store)) - draft_hit = [r for r in q_note.get("results", []) if "registry" in str(r.get("title", "")).lower() - or "unconfirmed" in str(r)] + q_note = json.loads( + handle_query_wiki( + { + "output_dir": output_dir, + "query": "registry.py registration MCP tool", + }, + store, + ) + ) + draft_hit = [ + r + for r in q_note.get("results", []) + if "registry" in str(r.get("title", "")).lower() or "unconfirmed" in str(r) + ] check("distilled draft note is queryable", len(draft_hit) >= 1, f"hits: {len(draft_hit)}") # golden-set: LLM JSON parser must extract structured notes (anti-hallucination) from codewiki.mcp.tools.distill_conversation import _parse_llm_notes + golden = ( '```json\n{"notes":[{"title":"Use status=draft","note_type":"decision",' '"related_modules":["notes"],"content":"## Decision\\nX"}]}\n```' @@ -590,8 +828,14 @@ async def _fake_llm(prompt, system): parsed = _parse_llm_notes(golden) check("golden parse yields one note", len(parsed) == 1, str(parsed)) if parsed: - check("golden note_type preserved", parsed[0].get("note_type") == "decision", str(parsed[0])) - check("golden strips markdown fences", parsed[0].get("title") == "Use status=draft", str(parsed[0])) + check( + "golden note_type preserved", parsed[0].get("note_type") == "decision", str(parsed[0]) + ) + check( + "golden strips markdown fences", + parsed[0].get("title") == "Use status=draft", + str(parsed[0]), + ) bad = _parse_llm_notes("totally not json") check("non-json yields no notes (no hallucinated draft)", bad == [], str(bad)) @@ -600,57 +844,97 @@ async def _fake_llm(prompt, system): conv_c1 = [ {"role": "user", "content": "Why does analyze_repo hang inside MCP?"}, - {"role": "assistant", "content": "git subprocess inherited the MCP stdin pipe; pass stdin=DEVNULL."}, + { + "role": "assistant", + "content": "git subprocess inherited the MCP stdin pipe; pass stdin=DEVNULL.", + }, ] conv_c2 = [ {"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi, how can I help?"}, ] - cap_c1 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, "conversation": conv_c1, - "source_session_id": "mode-c-sess-1", - }, store)) - cap_c2 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, "conversation": conv_c2, - "source_session_id": "mode-c-sess-2", - }, store)) + cap_c1 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv_c1, + "source_session_id": "mode-c-sess-1", + }, + store, + ) + ) + cap_c2 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": conv_c2, + "source_session_id": "mode-c-sess-2", + }, + store, + ) + ) cid1 = cap_c1.get("conversation_id") cid2 = cap_c2.get("conversation_id") - check("mode-C captures ready", - cap_c1.get("status") == "captured" and cap_c2.get("status") == "captured", - f"{cap_c1} / {cap_c2}") + check( + "mode-C captures ready", + cap_c1.get("status") == "captured" and cap_c2.get("status") == "captured", + f"{cap_c1} / {cap_c2}", + ) - prep = json.loads(handle_distill_conversation({"output_dir": output_dir, "mode": "prepare"}, store)) + prep = json.loads( + handle_distill_conversation({"output_dir": output_dir, "mode": "prepare"}, store) + ) check("prepare returns prepared", prep.get("status") == "prepared", str(prep)[:200]) - check("prepare exposes system prompt", - "knowledge distillation engine" in prep.get("system_prompt", ""), "") + check( + "prepare exposes system prompt", + "knowledge distillation engine" in prep.get("system_prompt", ""), + "", + ) ids = {c.get("conversation_id") for c in prep.get("captures", [])} check("prepare lists both captures", {cid1, cid2} <= ids, str(ids)) - check("prepare carries transcripts", - any("stdin=DEVNULL" in c.get("transcript", "") for c in prep.get("captures", [])), "") + check( + "prepare carries transcripts", + any("stdin=DEVNULL" in c.get("transcript", "") for c in prep.get("captures", [])), + "", + ) # The test plays the host agent: one real note for c1, nothing for c2 - submit = json.loads(handle_distill_conversation({ - "output_dir": output_dir, - "mode": "submit", - "distilled": { - cid1: {"notes": [{ - "title": "MCP subprocesses must set stdin=DEVNULL", - "note_type": "pitfall", - "related_modules": ["mcp"], - "tags": ["mcp", "subprocess"], - "content": ("## Background\nanalyze_repo hung inside MCP.\n" - "## Root cause\ngit inherited the MCP stdin pipe.\n" - "## Fix\nPass stdin=subprocess.DEVNULL to every subprocess call."), - }]}, - cid2: {"notes": []}, - }, - }, store)) + submit = json.loads( + handle_distill_conversation( + { + "output_dir": output_dir, + "mode": "submit", + "distilled": { + cid1: { + "notes": [ + { + "title": "MCP subprocesses must set stdin=DEVNULL", + "note_type": "pitfall", + "related_modules": ["mcp"], + "tags": ["mcp", "subprocess"], + "content": ( + "## Background\nanalyze_repo hung inside MCP.\n" + "## Root cause\ngit inherited the MCP stdin pipe.\n" + "## Fix\nPass stdin=subprocess.DEVNULL to every subprocess call." + ), + } + ] + }, + cid2: {"notes": []}, + }, + }, + store, + ) + ) check("submit returns completed", submit.get("status") == "completed", str(submit)[:200]) check("submit created exactly 1 note", submit.get("notes_created") == 1, str(submit)[:200]) by_cid = {r.get("conversation_id"): r for r in submit.get("distilled", [])} check("c1 distilled", by_cid.get(cid1, {}).get("status") == "completed", str(by_cid.get(cid1))) - check("c2 no_knowledge", by_cid.get(cid2, {}).get("status") == "no_knowledge", str(by_cid.get(cid2))) + check( + "c2 no_knowledge", + by_cid.get(cid2, {}).get("status") == "no_knowledge", + str(by_cid.get(cid2)), + ) check("c1 raw deleted", not (Path(output_dir) / "raw" / f"{cid1}.md").exists(), str(cid1)) _c2_path = Path(output_dir) / "raw" / f"{cid2}.md" # no_knowledge raws are noise and cleaned up by distill_conversation @@ -659,36 +943,65 @@ async def _fake_llm(prompt, system): check("c2 raw deleted on no_knowledge", not _c2_path.exists(), str(_c2_path)) # Missing extraction result leaves the raw file untouched (still pending) - cap_c3 = json.loads(handle_capture_conversation({ - "output_dir": output_dir, - "conversation": [{"role": "user", "content": "pending leftover"}], - "source_session_id": "mode-c-sess-3", - }, store)) + cap_c3 = json.loads( + handle_capture_conversation( + { + "output_dir": output_dir, + "conversation": [{"role": "user", "content": "pending leftover"}], + "source_session_id": "mode-c-sess-3", + }, + store, + ) + ) cid3 = cap_c3.get("conversation_id") - sub2 = json.loads(handle_distill_conversation({ - "output_dir": output_dir, "mode": "submit", - "distilled": {"conv-nonexistent": {"notes": []}}, - }, store)) + sub2 = json.loads( + handle_distill_conversation( + { + "output_dir": output_dir, + "mode": "submit", + "distilled": {"conv-nonexistent": {"notes": []}}, + }, + store, + ) + ) by_cid2 = {r.get("conversation_id"): r for r in sub2.get("distilled", [])} - check("missing result reported", - by_cid2.get(cid3, {}).get("status") == "missing_result", str(sub2)[:200]) + check( + "missing result reported", + by_cid2.get(cid3, {}).get("status") == "missing_result", + str(sub2)[:200], + ) _c3_path = Path(output_dir) / "raw" / f"{cid3}.md" - check("raw untouched on missing result", - _c3_path.exists() and "status: pending" in _c3_path.read_text(encoding="utf-8"), - str(_c3_path)) + check( + "raw untouched on missing result", + _c3_path.exists() and "status: pending" in _c3_path.read_text(encoding="utf-8"), + str(_c3_path), + ) # Error paths - bad_mode = json.loads(handle_distill_conversation({"output_dir": output_dir, "mode": "bogus"}, store)) + bad_mode = json.loads( + handle_distill_conversation({"output_dir": output_dir, "mode": "bogus"}, store) + ) check("invalid mode rejected", "error" in bad_mode, str(bad_mode)) - no_map = json.loads(handle_distill_conversation({"output_dir": output_dir, "mode": "submit"}, store)) + no_map = json.loads( + handle_distill_conversation({"output_dir": output_dir, "mode": "submit"}, store) + ) check("submit without distilled rejected", "error" in no_map, str(no_map)) # The Mode-C draft note should be queryable like any other draft - q_c = json.loads(handle_query_wiki({ - "output_dir": output_dir, "query": "stdin DEVNULL MCP subprocess", - }, store)) - hit_c = [r for r in q_c.get("results", []) - if "DEVNULL" in str(r.get("title", "")) or "stdin" in str(r)] + q_c = json.loads( + handle_query_wiki( + { + "output_dir": output_dir, + "query": "stdin DEVNULL MCP subprocess", + }, + store, + ) + ) + hit_c = [ + r + for r in q_c.get("results", []) + if "DEVNULL" in str(r.get("title", "")) or "stdin" in str(r) + ] check("mode-C draft note queryable", len(hit_c) >= 1, f"hits: {len(hit_c)}") # -- Summary -- diff --git a/tests/telemetry_seed.py b/tests/telemetry_seed.py index c5486d3..cd56fcb 100644 --- a/tests/telemetry_seed.py +++ b/tests/telemetry_seed.py @@ -7,6 +7,7 @@ so existing test semantics (hit counts, last_hit dates, adoption counts, distinct keys) carry over unchanged. """ + from __future__ import annotations import json diff --git a/tests/test_adoption.py b/tests/test_adoption.py index 3aea116..d183291 100644 --- a/tests/test_adoption.py +++ b/tests/test_adoption.py @@ -10,6 +10,7 @@ - capture integration: declared docs persisted + adoption_nudge only when search traces exist without any declaration """ + from __future__ import annotations import json @@ -54,7 +55,9 @@ def test_path_normalisation(self): '', ) assert extract_adopted_docs(turns) == [ - "notes/abs.md", "notes/rel.md", "notes/win.md", + "notes/abs.md", + "notes/rel.md", + "notes/win.md", ] def test_rejects_traversal_and_empty(self): @@ -66,7 +69,7 @@ def test_rejects_traversal_and_empty(self): def test_invalid_json_skipped(self): turns = _turns( '', - '', + "", '', ) assert extract_adopted_docs(turns) == [] @@ -86,8 +89,10 @@ def test_existence_filter(self): turns = _turns( '', ) + def _exists(p): return p == "exists.md" + assert extract_adopted_docs(turns, existing=_exists) == ["exists.md"] def test_prose_mention_does_not_match(self): @@ -110,7 +115,8 @@ def test_insert_and_count(self, tmp_path): n = record_adoption_events(tmp_path, "tester/sess-1", ["notes/a.md", "notes/b.md"]) assert n == 2 assert load_adoption_counts(tmp_path) == { - "notes/a.md": 1, "notes/b.md": 1, + "notes/a.md": 1, + "notes/b.md": 1, } def test_idempotent_same_key(self, tmp_path): @@ -124,7 +130,8 @@ def test_new_doc_same_key_counts(self, tmp_path): n = record_adoption_events(tmp_path, "tester/sess-1", ["notes/a.md", "notes/b.md"]) assert n == 1 assert load_adoption_counts(tmp_path) == { - "notes/a.md": 1, "notes/b.md": 1, + "notes/a.md": 1, + "notes/b.md": 1, } def test_different_sessions_accumulate(self, tmp_path): @@ -204,13 +211,17 @@ def test_declared_docs_persisted(self, tmp_path): _make_doc(tmp_path, "wiki/modules/m.md") turns = [ {"role": "user", "content": "question"}, - {"role": "assistant", "content": - "answer\n'}, + { + "role": "assistant", + "content": "answer\n', + }, {"role": "user", "content": "thanks"}, {"role": "assistant", "content": "done"}, ] - result = json.loads(handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore())) + result = json.loads( + handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore()) + ) assert result["adopted_docs"] == ["notes/pitfall-a.md", "wiki/modules/m.md"] assert result["adoption_inserted"] == 2 counts = load_adoption_counts(tmp_path) @@ -220,8 +231,10 @@ def test_supersede_no_double_count(self, tmp_path): _make_doc(tmp_path, "notes/pitfall-a.md") turns1 = [ {"role": "user", "content": "q1"}, - {"role": "assistant", "content": - 'a1\n'}, + { + "role": "assistant", + "content": 'a1\n', + }, ] # re-capture same session with an extended transcript turns2 = turns1 + [ @@ -236,20 +249,25 @@ def test_missing_path_dropped(self, tmp_path): _make_doc(tmp_path, "notes/exists.md") turns = [ {"role": "user", "content": "q"}, - {"role": "assistant", "content": - 'a\n'}, + { + "role": "assistant", + "content": 'a\n', + }, ] - result = json.loads(handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore())) + result = json.loads( + handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore()) + ) assert result["adopted_docs"] == ["notes/exists.md"] assert load_adoption_counts(tmp_path) == {"notes/exists.md": 1} def test_nudge_when_search_traces_without_declaration(self, tmp_path): turns = [ {"role": "user", "content": "search it"}, - {"role": "assistant", "content": - "based on context_package from query_wiki ..."}, + {"role": "assistant", "content": "based on context_package from query_wiki ..."}, ] - result = json.loads(handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore())) + result = json.loads( + handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore()) + ) assert result.get("adoption_nudge") is True assert "adopted_docs" not in result @@ -257,11 +275,15 @@ def test_no_nudge_when_declared(self, tmp_path): _make_doc(tmp_path, "notes/a.md") turns = [ {"role": "user", "content": "search it"}, - {"role": "assistant", "content": - "based on context_package...\n" - ''}, + { + "role": "assistant", + "content": "based on context_package...\n" + '', + }, ] - result = json.loads(handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore())) + result = json.loads( + handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore()) + ) assert "adoption_nudge" not in result def test_no_nudge_when_no_search_traces(self, tmp_path): @@ -269,5 +291,7 @@ def test_no_nudge_when_no_search_traces(self, tmp_path): {"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi there"}, ] - result = json.loads(handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore())) + result = json.loads( + handle_capture_conversation(_capture_args(tmp_path, turns, "s1"), SessionStore()) + ) assert "adoption_nudge" not in result diff --git a/tests/test_authority_p0.py b/tests/test_authority_p0.py index 86c3d04..c2055ec 100644 --- a/tests/test_authority_p0.py +++ b/tests/test_authority_p0.py @@ -6,6 +6,7 @@ BEFORE the note title floor. Similarity-oriented consumers (distill dedup recall) are exempt via apply_authority=False. """ + from pathlib import Path from codewiki.mcp.cache import AnalysisCache, _doc_authority @@ -15,8 +16,7 @@ # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _mk_note(notes_dir: Path, name: str, title: str, ntype: str, status: str, - body: str) -> Path: +def _mk_note(notes_dir: Path, name: str, title: str, ntype: str, status: str, body: str) -> Path: notes_dir.mkdir(parents=True, exist_ok=True) p = notes_dir / name p.write_text( @@ -32,14 +32,14 @@ def _mk_note(notes_dir: Path, name: str, title: str, ntype: str, status: str, def test_authority_note_type_and_status(): decision_stable = "---\ntype: decision\ntitle: T\nstatus: stable\n---\nbody" lesson_draft = "---\ntype: lesson\ntitle: T\nstatus: draft\n---\nbody" - assert _doc_authority("notes/a.md", "note", decision_stable) == 1.2 # +0.15 +0.05 - assert _doc_authority("notes/b.md", "note", lesson_draft) == 0.85 # +0.10 -0.25 + assert _doc_authority("notes/a.md", "note", decision_stable) == 1.2 # +0.15 +0.05 + assert _doc_authority("notes/b.md", "note", lesson_draft) == 0.85 # +0.10 -0.25 def test_authority_deprecated_clamped_low(): - dep = "---\ntype: pitfall\ntitle: T\nstatus: deprecated\n---\nbody" # +0.12 -0.35 + dep = "---\ntype: pitfall\ntitle: T\nstatus: deprecated\n---\nbody" # +0.12 -0.35 assert abs(_doc_authority("notes/c.md", "note", dep) - 0.77) < 1e-9 - bare_dep = "---\ntitle: T\nstatus: deprecated\n---\nbody" # -0.35 -> clamp 0.7 + bare_dep = "---\ntitle: T\nstatus: deprecated\n---\nbody" # -0.35 -> clamp 0.7 assert _doc_authority("notes/d.md", "note", bare_dep) == 0.7 @@ -77,8 +77,10 @@ def test_sqlite_search_orders_by_authority(tmp_path): # Exemption: identical bodies -> identical raw BM25 scores, authority 1.0 raw = cache.search("gateway timeout retry", output_dir=od, apply_authority=False) by_file = {r["file"]: r for r in raw} - assert by_file["notes/n-decision.md"]["relevance_score"] == \ - by_file["notes/n-lesson.md"]["relevance_score"] + assert ( + by_file["notes/n-decision.md"]["relevance_score"] + == by_file["notes/n-lesson.md"]["relevance_score"] + ) assert all(r["authority"] == 1.0 for r in raw) finally: cache.close() @@ -104,16 +106,23 @@ def test_legacy_search_orders_by_authority(tmp_path): def test_update_file_refreshes_authority_after_status_change(tmp_path): od = tmp_path / "repowiki" - p = _mk_note(od / "notes", "n.md", "cache invalidation strategy", "lesson", "draft", - "cache invalidation strategy body text") + p = _mk_note( + od / "notes", + "n.md", + "cache invalidation strategy", + "lesson", + "draft", + "cache invalidation strategy body text", + ) wiki_search.build_full_index(od) res = wiki_search.search(od, "cache invalidation strategy") assert res and res[0]["file"] == "notes/n.md" assert res[0]["authority"] == 0.85 # Promote draft -> stable (mirrors _apply_status_to_file rewriting status) - p.write_text(p.read_text(encoding="utf-8").replace("status: draft", "status: stable"), - encoding="utf-8") + p.write_text( + p.read_text(encoding="utf-8").replace("status: draft", "status: stable"), encoding="utf-8" + ) wiki_search.update_file(od, p) res2 = wiki_search.search(od, "cache invalidation strategy") assert res2 and res2[0]["authority"] == 1.15 # lesson +0.10, stable +0.05 diff --git a/tests/test_change_analysis.py b/tests/test_change_analysis.py index 720b466..827f3e2 100644 --- a/tests/test_change_analysis.py +++ b/tests/test_change_analysis.py @@ -122,6 +122,7 @@ def test_parse_unified_diff_deleted_file() -> None: # Unit tests: changed-component location (line span matching) # ------------------------------------------------------------------ + class _FakeMeta: def __init__(self, rel: str, start: int, end: int): self.relative_path = rel @@ -172,6 +173,7 @@ def test_locate_changed_components_untracked() -> None: # Unit tests: test suggestion heuristics # ------------------------------------------------------------------ + def test_test_candidates_for() -> None: cands = _test_candidates_for("src/auth/login.py") assert "src/auth/test_login.py" in cands @@ -184,7 +186,9 @@ def test_test_candidates_for() -> None: def test_suggest_tests_filesystem(tmp_path) -> None: (tmp_path / "service").mkdir() (tmp_path / "service" / "order.py").write_text("def order(): pass\n", encoding="utf-8") - (tmp_path / "service" / "test_order.py").write_text("def test_order(): pass\n", encoding="utf-8") + (tmp_path / "service" / "test_order.py").write_text( + "def test_order(): pass\n", encoding="utf-8" + ) (tmp_path / "tests").mkdir() (tmp_path / "tests" / "test_order.py").write_text("def test_order(): pass\n", encoding="utf-8") @@ -250,7 +254,11 @@ def analyzed_repo(tmp_path): store = SessionStore() resp = json.loads( handle_analyze_repo( - {"repo_path": str(tmp_path), "output_dir": str(tmp_path / "repowiki"), "incremental": False}, + { + "repo_path": str(tmp_path), + "output_dir": str(tmp_path / "repowiki"), + "incremental": False, + }, store, ) ) diff --git a/tests/test_consolidation_p2.py b/tests/test_consolidation_p2.py index 122bac2..b6a39fb 100644 --- a/tests/test_consolidation_p2.py +++ b/tests/test_consolidation_p2.py @@ -9,6 +9,7 @@ validation errors, capacity enforcement, counter reset - lint scenario_capacity / scenario_orphan checks """ + import json from pathlib import Path @@ -18,7 +19,10 @@ from codewiki.mcp.tools import aggregation_state as agg from codewiki.mcp.tools import note_consolidation as cons from codewiki.mcp.tools.knowledge_loop import ( - handle_ingest_note, handle_confirm_note, handle_reject_note, handle_wiki_stats, + handle_ingest_note, + handle_confirm_note, + handle_reject_note, + handle_wiki_stats, ) from codewiki.mcp.tools.wiki_lint import handle_lint_wiki @@ -26,43 +30,58 @@ # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _set_thresholds(repo: str, cons_t: int = 3, hint_interval: int = 2, - max_scenes: int = 15): +def _set_thresholds(repo: str, cons_t: int = 3, hint_interval: int = 2, max_scenes: int = 15): od = Path(repo) / "repowiki" od.mkdir(parents=True, exist_ok=True) - schema = {"conventions": {"aggregation": { - "consolidation_threshold": cons_t, - "doctrine_threshold": 50, - "hint_interval": hint_interval, - "max_scenarios": max_scenes, - }}} + schema = { + "conventions": { + "aggregation": { + "consolidation_threshold": cons_t, + "doctrine_threshold": 50, + "hint_interval": hint_interval, + "max_scenarios": max_scenes, + } + } + } (od / "schema.yaml").write_text(yaml.safe_dump(schema), encoding="utf-8") -def _ingest(repo: str, title: str, note_type: str = "decision", - content: str = "## Background\nbody") -> str: +def _ingest( + repo: str, title: str, note_type: str = "decision", content: str = "## Background\nbody" +) -> str: store = SessionStore() - r = json.loads(handle_ingest_note({ - "output_dir": f"{repo}/repowiki", - "title": title, - "note_type": note_type, - "content": content, - "status": "draft", - }, store)) + r = json.loads( + handle_ingest_note( + { + "output_dir": f"{repo}/repowiki", + "title": title, + "note_type": note_type, + "content": content, + "status": "draft", + }, + store, + ) + ) assert r.get("status") in ("ingested", "already_exists"), r return Path(r["note_path"]).name def _confirm(repo: str, note_file: str) -> dict: store = SessionStore() - return json.loads(handle_confirm_note({ - "output_dir": f"{repo}/repowiki", - "note_file": note_file, - }, store)) + return json.loads( + handle_confirm_note( + { + "output_dir": f"{repo}/repowiki", + "note_file": note_file, + }, + store, + ) + ) -def _write_scenario(repo: str, name: str, body: str = "## Work context\nx", - with_provenance: bool = True) -> str: +def _write_scenario( + repo: str, name: str, body: str = "## Work context\nx", with_provenance: bool = True +) -> str: sdir = Path(repo) / "repowiki" / "wiki" / "scenarios" sdir.mkdir(parents=True, exist_ok=True) meta = {"heat": 1} @@ -127,8 +146,7 @@ def test_wiki_stats_exposes_aggregation_section(tmp_path): nf = _ingest(repo, "Stats visibility note") _confirm(repo, nf) store = SessionStore() - resp = json.loads(handle_wiki_stats( - {"output_dir": f"{repo}/repowiki"}, store)) + resp = json.loads(handle_wiki_stats({"output_dir": f"{repo}/repowiki"}, store)) # no retrieval stats DB yet → early return path must still carry counters assert "aggregation" in resp assert resp["aggregation"]["notes_since_last_consolidation"] == 1 @@ -139,12 +157,15 @@ def test_get_task_context_exposes_aggregation(tmp_path): repo = str(tmp_path) _set_thresholds(repo) from codewiki.mcp.tools.task_manager import handle_create_task, handle_get_task_context + store = SessionStore() - r = json.loads(handle_create_task( - {"output_dir": f"{repo}/repowiki", "title": "P2 smoke task"}, store)) + r = json.loads( + handle_create_task({"output_dir": f"{repo}/repowiki", "title": "P2 smoke task"}, store) + ) task_id = r["task"]["id"] - resp = json.loads(handle_get_task_context( - {"output_dir": f"{repo}/repowiki", "task_id": task_id}, store)) + resp = json.loads( + handle_get_task_context({"output_dir": f"{repo}/repowiki", "task_id": task_id}, store) + ) assert resp["ok"] is True assert "aggregation" in resp assert resp["aggregation"]["notes_since_last_consolidation"] == 0 @@ -158,11 +179,12 @@ def test_prepare_lists_only_pending_confirmed_notes(tmp_path): _set_thresholds(repo) stable = _ingest(repo, "Stable candidate note") _confirm(repo, stable) - _ingest(repo, "Draft only note") # stays draft + _ingest(repo, "Draft only note") # stays draft rejected = _ingest(repo, "Rejected candidate note") store = SessionStore() - handle_reject_note({"output_dir": f"{repo}/repowiki", - "note_file": rejected, "reason": "noise"}, store) + handle_reject_note( + {"output_dir": f"{repo}/repowiki", "note_file": rejected, "reason": "noise"}, store + ) resp = _consolidate(repo, {"mode": "prepare"}) assert resp["status"] == "prepared" @@ -197,15 +219,26 @@ def test_submit_records_provenance_and_resets_counter(tmp_path): state = agg.load_state(Path(repo) / "repowiki") assert state["notes_since_last_consolidation"] == 2 - scen = _write_scenario(repo, "redis-运维方法", body="## Core SOP\ncheck pool", - with_provenance=False) - resp = _consolidate(repo, {"mode": "submit", "report": {"scenarios": [{ - "file": scen, - "action": "created", - "source_notes": [f"notes/{n1}", f"notes/{n2}"], - "summary": "Redis 连接池运维方法汇总", - "heat": 1, - }]}}) + scen = _write_scenario( + repo, "redis-运维方法", body="## Core SOP\ncheck pool", with_provenance=False + ) + resp = _consolidate( + repo, + { + "mode": "submit", + "report": { + "scenarios": [ + { + "file": scen, + "action": "created", + "source_notes": [f"notes/{n1}", f"notes/{n2}"], + "summary": "Redis 连接池运维方法汇总", + "heat": 1, + } + ] + }, + }, + ) assert resp["status"] == "completed", resp assert resp["counters"]["notes_since_last_consolidation"] == 0 @@ -231,11 +264,19 @@ def test_submit_deleted_cleans_soft_deleted_files(tmp_path): p = Path(repo) / "repowiki" / scen fm_text = p.read_text(encoding="utf-8") end = fm_text.find("---", 3) - p.write_text(fm_text[:end + 3] + "\n[DELETED]\n", encoding="utf-8") - - resp = _consolidate(repo, {"mode": "submit", "report": {"scenarios": [ - {"file": scen, "action": "deleted"}, - ]}}) + p.write_text(fm_text[: end + 3] + "\n[DELETED]\n", encoding="utf-8") + + resp = _consolidate( + repo, + { + "mode": "submit", + "report": { + "scenarios": [ + {"file": scen, "action": "deleted"}, + ] + }, + }, + ) assert resp["status"] == "completed" assert scen.replace("/", "/") in [r.replace("\\", "/") for r in resp["removed_deleted"]] assert not p.exists() @@ -245,9 +286,17 @@ def test_submit_deleted_requires_marker(tmp_path): repo = str(tmp_path) _set_thresholds(repo) scen = _write_scenario(repo, "alive-scene") - resp = _consolidate(repo, {"mode": "submit", "report": {"scenarios": [ - {"file": scen, "action": "deleted"}, - ]}}) + resp = _consolidate( + repo, + { + "mode": "submit", + "report": { + "scenarios": [ + {"file": scen, "action": "deleted"}, + ] + }, + }, + ) assert resp["status"] == "error" assert (Path(repo) / "repowiki" / scen).exists() @@ -257,9 +306,17 @@ def test_submit_validation_error_keeps_counter(tmp_path): _set_thresholds(repo) nf = _ingest(repo, "Counter guard note") _confirm(repo, nf) - resp = _consolidate(repo, {"mode": "submit", "report": {"scenarios": [ - {"file": "wiki/scenarios/ghost.md", "action": "created", "source_notes": []}, - ]}}) + resp = _consolidate( + repo, + { + "mode": "submit", + "report": { + "scenarios": [ + {"file": "wiki/scenarios/ghost.md", "action": "created", "source_notes": []}, + ] + }, + }, + ) assert resp["status"] == "error" state = agg.load_state(Path(repo) / "repowiki") assert state["notes_since_last_consolidation"] == 1 # NOT reset @@ -271,9 +328,15 @@ def test_submit_capacity_exceeded_blocks_reset(tmp_path): nf = _ingest(repo, "Capacity guard note") _confirm(repo, nf) files = [_write_scenario(repo, f"cap-{i}") for i in range(3)] - resp = _consolidate(repo, {"mode": "submit", "report": {"scenarios": [ - {"file": f, "action": "updated", "source_notes": []} for f in files - ]}}) + resp = _consolidate( + repo, + { + "mode": "submit", + "report": { + "scenarios": [{"file": f, "action": "updated", "source_notes": []} for f in files] + }, + }, + ) assert resp["status"] == "capacity_exceeded" assert resp["capacity"]["current"] == 3 state = agg.load_state(Path(repo) / "repowiki") @@ -291,9 +354,12 @@ def test_lint_scenario_capacity_and_orphan(tmp_path): _write_scenario(repo, "over-3", with_provenance=False) store = SessionStore() - resp = json.loads(handle_lint_wiki( - {"output_dir": f"{repo}/repowiki", - "checks": ["scenario_capacity", "scenario_orphan"]}, store)) + resp = json.loads( + handle_lint_wiki( + {"output_dir": f"{repo}/repowiki", "checks": ["scenario_capacity", "scenario_orphan"]}, + store, + ) + ) checks = {i["check"] for i in resp["issues"]} assert "scenario_capacity" in checks cap = [i for i in resp["issues"] if i["check"] == "scenario_capacity"][0] diff --git a/tests/test_distill_cleanup.py b/tests/test_distill_cleanup.py index bef80a7..a3300c0 100644 --- a/tests/test_distill_cleanup.py +++ b/tests/test_distill_cleanup.py @@ -4,6 +4,7 @@ in repowiki/raw/ must be cleaned up so the transient staging area doesn't accumulate noise. Only keep_raw (explicit opt-in) preserves the raw file. """ + import json from pathlib import Path @@ -13,9 +14,11 @@ from codewiki.src.config import RAW_DIR -def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello", - keep_raw: bool = False) -> str: +def _write_raw( + repo: str, cid: str, body: str = "user: hi\nassistant: hello", keep_raw: bool = False +) -> str: from pathlib import Path + raw_dir = Path(repo) / "repowiki" / RAW_DIR raw_dir.mkdir(parents=True, exist_ok=True) p = raw_dir / f"conv-{cid}.md" @@ -23,11 +26,9 @@ def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello", p.write_text( "---\n" "type: conversation\n" - f"conversation_id: \"{cid}\"\n" + f'conversation_id: "{cid}"\n' "status: pending\n" - "origin: conversation\n" - + extra + - "---\n\n" + body, + "origin: conversation\n" + extra + "---\n\n" + body, encoding="utf-8", ) return str(p) @@ -35,11 +36,14 @@ def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello", def _submit(repo: str, distilled: dict): store = SessionStore() - out = distill.handle_distill_conversation({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "distilled": distilled, - }, store) + out = distill.handle_distill_conversation( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "distilled": distilled, + }, + store, + ) data = json.loads(out) # submit aggregates per-conversation results under data["distilled"], keyed # by the full filename stem (e.g. "conv-"). @@ -62,6 +66,7 @@ def test_no_knowledge_raw_is_cleaned_up(tmp_path): assert r["status"] == "no_knowledge" # raw file should be gone import os + assert not os.path.exists(raw_path) assert r["deleted_raw"] is True @@ -78,6 +83,7 @@ def test_keep_raw_preserves_no_knowledge_file(tmp_path): r = by_cid[_cid(cid)] assert r["status"] == "no_knowledge" import os + assert not os.path.exists(raw_path) # left the raw/ staging queue... assert r["deleted_raw"] is False # ...but was preserved via the L0 archive @@ -93,17 +99,27 @@ def test_produced_knowledge_archives_raw(tmp_path): cid = "produced-001" raw_path = _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "sample note", - "note_type": "pitfall", - "content": "## background\n\nsomething reusable", - "related_modules": ["x"], - "tags": ["t"], - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "sample note", + "note_type": "pitfall", + "content": "## background\n\nsomething reusable", + "related_modules": ["x"], + "tags": ["t"], + } + ] + } + }, + ) r = by_cid[_cid(cid)] assert r["status"] == "completed" import os + assert not os.path.exists(raw_path) # left raw/ staging queue assert r["deleted_raw"] is False assert r["archived_raw"] == f"conversations/{Path(raw_path).name}" @@ -117,20 +133,22 @@ def test_produced_knowledge_archives_raw(tmp_path): # sessionStart catch-up: task_id scoping # --------------------------------------------------------------------------- # + def _write_raw_tasked(repo: str, cid: str, task_id: str) -> str: """Write a pending raw capture bound to a task (frontmatter carries task_id).""" from pathlib import Path + raw_dir = Path(repo) / "repowiki" / RAW_DIR raw_dir.mkdir(parents=True, exist_ok=True) p = raw_dir / f"conv-{cid}.md" p.write_text( "---\n" "type: conversation\n" - f"conversation_id: \"{cid}\"\n" + f'conversation_id: "{cid}"\n' "status: pending\n" "origin: conversation\n" - f"task_id: \"{task_id}\"\n" - "captured_at: \"2026-08-16T00:00:00Z\"\n" + f'task_id: "{task_id}"\n' + 'captured_at: "2026-08-16T00:00:00Z"\n' "---\n\nuser: hi\nassistant: hello", encoding="utf-8", ) @@ -140,6 +158,7 @@ def _write_raw_tasked(repo: str, cid: str, task_id: str) -> str: def _write_raw_index(repo: str, entries: list) -> None: """Write raw/.index.json in the shape capture_conversation maintains.""" from pathlib import Path + raw_dir = Path(repo) / "repowiki" / RAW_DIR raw_dir.mkdir(parents=True, exist_ok=True) (raw_dir / ".index.json").write_text( @@ -162,14 +181,32 @@ def test_prepare_scopes_to_task_via_index(self, tmp_path): _write_raw_tasked(repo, "a", "task-one") _write_raw_tasked(repo, "b", "task-two") _write_raw_tasked(repo, "c", "") # unbound capture - _write_raw_index(repo, [ - {"relpath": "conv-a.md", "content_hash": "h1", "source_session": "s1", - "status": "pending", "task_id": "task-one"}, - {"relpath": "conv-b.md", "content_hash": "h2", "source_session": "s2", - "status": "pending", "task_id": "task-two"}, - {"relpath": "conv-c.md", "content_hash": "h3", "source_session": "s3", - "status": "pending", "task_id": ""}, - ]) + _write_raw_index( + repo, + [ + { + "relpath": "conv-a.md", + "content_hash": "h1", + "source_session": "s1", + "status": "pending", + "task_id": "task-one", + }, + { + "relpath": "conv-b.md", + "content_hash": "h2", + "source_session": "s2", + "status": "pending", + "task_id": "task-two", + }, + { + "relpath": "conv-c.md", + "content_hash": "h3", + "source_session": "s3", + "status": "pending", + "task_id": "", + }, + ], + ) data = _prepare(repo, "task-one") assert data["status"] == "prepared" @@ -178,10 +215,18 @@ def test_prepare_scopes_to_task_via_index(self, tmp_path): def test_prepare_no_match_is_noop(self, tmp_path): repo = str(tmp_path) _write_raw_tasked(repo, "a", "task-one") - _write_raw_index(repo, [ - {"relpath": "conv-a.md", "content_hash": "h1", "source_session": "s1", - "status": "pending", "task_id": "task-one"}, - ]) + _write_raw_index( + repo, + [ + { + "relpath": "conv-a.md", + "content_hash": "h1", + "source_session": "s1", + "status": "pending", + "task_id": "task-one", + }, + ], + ) data = _prepare(repo, "some-other-task") assert data["status"] == "noop" @@ -201,27 +246,46 @@ def test_submit_ignores_other_tasks_extractions(self, tmp_path): repo = str(tmp_path) _write_raw_tasked(repo, "a", "task-one") _write_raw_tasked(repo, "b", "task-two") - _write_raw_index(repo, [ - {"relpath": "conv-a.md", "content_hash": "h1", "source_session": "s1", - "status": "pending", "task_id": "task-one"}, - {"relpath": "conv-b.md", "content_hash": "h2", "source_session": "s2", - "status": "pending", "task_id": "task-two"}, - ]) + _write_raw_index( + repo, + [ + { + "relpath": "conv-a.md", + "content_hash": "h1", + "source_session": "s1", + "status": "pending", + "task_id": "task-one", + }, + { + "relpath": "conv-b.md", + "content_hash": "h2", + "source_session": "s2", + "status": "pending", + "task_id": "task-two", + }, + ], + ) store = SessionStore() - out = distill.handle_distill_conversation({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "task_id": "task-one", - # Extraction for conv-b (another task) must be ignored; conv-a has - # no extraction yet, so nothing should be processed. - "distilled": {"conv-b": {"notes": []}}, - }, store) + out = distill.handle_distill_conversation( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "task_id": "task-one", + # Extraction for conv-b (another task) must be ignored; conv-a has + # no extraction yet, so nothing should be processed. + "distilled": {"conv-b": {"notes": []}}, + }, + store, + ) data = json.loads(out) # conv-a is in scope but has no extraction yet → missing_result (raw # untouched); conv-b (another task) must not be processed at all. assert [r["conversation_id"] for r in data["distilled"]] == ["conv-a"] assert data["distilled"][0]["status"] == "missing_result" import os + assert os.path.exists(f"{repo}/repowiki/{RAW_DIR}/conv-b.md") - assert "status: pending" in Path(f"{repo}/repowiki/{RAW_DIR}/conv-b.md").read_text(encoding="utf-8") + assert "status: pending" in Path(f"{repo}/repowiki/{RAW_DIR}/conv-b.md").read_text( + encoding="utf-8" + ) diff --git a/tests/test_distill_p1.py b/tests/test_distill_p1.py index 63e574b..ec63f29 100644 --- a/tests/test_distill_p1.py +++ b/tests/test_distill_p1.py @@ -10,6 +10,7 @@ until the agent re-submits with a dedup_action: store / skip / update / merge. """ + import json from pathlib import Path @@ -28,7 +29,7 @@ def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello") -> p.write_text( "---\n" "type: conversation\n" - f"conversation_id: \"{cid}\"\n" + f'conversation_id: "{cid}"\n' "status: pending\n" "origin: conversation\n" "---\n\n" + body, @@ -37,18 +38,15 @@ def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello") -> return str(p) -def _write_note(repo: str, filename: str, title: str, - note_type: str = "pitfall", body: str = "existing body") -> str: +def _write_note( + repo: str, filename: str, title: str, note_type: str = "pitfall", body: str = "existing body" +) -> str: """Create a pre-existing note with unquoted frontmatter title (dedup target).""" notes_dir = Path(repo) / "repowiki" / "notes" notes_dir.mkdir(parents=True, exist_ok=True) p = notes_dir / filename p.write_text( - "---\n" - f"type: {note_type}\n" - f"title: {title}\n" - "status: stable\n" - "---\n\n" + body + "\n", + f"---\ntype: {note_type}\ntitle: {title}\nstatus: stable\n---\n\n" + body + "\n", encoding="utf-8", ) return f"notes/{filename}" @@ -56,11 +54,14 @@ def _write_note(repo: str, filename: str, title: str, def _submit(repo: str, distilled: dict): store = SessionStore() - out = distill.handle_distill_conversation({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "distilled": distilled, - }, store) + out = distill.handle_distill_conversation( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "distilled": distilled, + }, + store, + ) data = json.loads(out) by_cid = {r["conversation_id"]: r for r in data.get("distilled", [])} return data, by_cid @@ -83,7 +84,7 @@ def _notes_of_type(repo: str, note_type: str): # --------------------------------------------------------------------------- # def test_parse_priority_clamps_and_rejects_invalid(): assert distill._parse_priority(None) is None - assert distill._parse_priority(True) is None # bool is not a priority + assert distill._parse_priority(True) is None # bool is not a priority assert distill._parse_priority("abc") is None assert distill._parse_priority(75) == 75 assert distill._parse_priority("82") == 82 @@ -106,12 +107,21 @@ def test_low_priority_note_is_dropped(tmp_path): repo = str(tmp_path) cid = "prio-low" _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "Trivial formatting tweak", - "note_type": "general", - "priority": 50, - "content": "low value content", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "Trivial formatting tweak", + "note_type": "general", + "priority": 50, + "content": "low value content", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["status"] == "no_knowledge" or res["status"] == "completed" entry = res["notes"][0] @@ -125,20 +135,27 @@ def test_priority_maps_to_severity(tmp_path): repo = str(tmp_path) cid = "prio-high" _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [ - { - "title": "Never delete production data without backup", - "note_type": "pitfall", - "priority": 95, - "content": "## Root cause\ndata loss risk", - }, + _data, by_cid = _submit( + repo, { - "title": "Prefer incremental index rebuild", - "note_type": "decision", - "priority": 75, - "content": "## Decision\nincremental rebuild", + cid: { + "notes": [ + { + "title": "Never delete production data without backup", + "note_type": "pitfall", + "priority": 95, + "content": "## Root cause\ndata loss risk", + }, + { + "title": "Prefer incremental index rebuild", + "note_type": "decision", + "priority": 75, + "content": "## Decision\nincremental rebuild", + }, + ] + } }, - ]}}) + ) res = by_cid[f"conv-{cid}"] assert res["status"] == "completed" highs = _notes_of_type(repo, "pitfall") @@ -151,11 +168,20 @@ def test_note_without_priority_ingests_without_severity(tmp_path): repo = str(tmp_path) cid = "prio-none" _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "Legacy note has no priority field", - "note_type": "general", - "content": "backward compatible", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "Legacy note has no priority field", + "note_type": "general", + "content": "backward compatible", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["notes"][0]["status"] in ("ingested", "draft", "already_exists") notes = list((Path(repo) / "repowiki" / "notes").glob("*.md")) @@ -170,13 +196,22 @@ def test_scene_written_to_metadata(tmp_path): repo = str(tmp_path) cid = "scene-001" _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "BM25 threshold tuning for short notes", - "note_type": "decision", - "priority": 80, - "scene": "围绕检索质量调优", - "content": "## Decision\nlower threshold", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "BM25 threshold tuning for short notes", + "note_type": "decision", + "priority": 80, + "scene": "围绕检索质量调优", + "content": "## Decision\nlower threshold", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["status"] == "completed" notes = list((Path(repo) / "repowiki" / "notes").glob("*.md")) @@ -193,14 +228,24 @@ def test_strong_duplicate_still_suppressed(tmp_path): cid = "strong-dup" _write_raw(repo, cid) # identical title + same type => Jaccard 1.0, strong duplicate - _write_note(repo, "2026-01-01-redis-pool.md", "Redis connection pool timeout", - note_type="pitfall") - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "Redis connection pool timeout", - "note_type": "pitfall", - "priority": 85, - "content": "dup content", - }]}}) + _write_note( + repo, "2026-01-01-redis-pool.md", "Redis connection pool timeout", note_type="pitfall" + ) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "Redis connection pool timeout", + "note_type": "pitfall", + "priority": 85, + "content": "dup content", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] entry = res["notes"][0] assert entry["status"] == "suppressed" @@ -215,20 +260,30 @@ def test_strong_duplicate_still_suppressed(tmp_path): def _weak_conflict_setup(repo: str, cid: str): """Existing note shares 3/7 title tokens (sim≈0.43, weak band, diff type).""" _write_raw(repo, cid) - return _write_note(repo, "2026-01-01-alpha.md", - "alpha beta gamma delta epsilon", note_type="pitfall") + return _write_note( + repo, "2026-01-01-alpha.md", "alpha beta gamma delta epsilon", note_type="pitfall" + ) def test_weak_conflict_holds_and_retains_raw(tmp_path): repo = str(tmp_path) cid = "weak-conflict" _weak_conflict_setup(repo, cid) - data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", # 3/7 ≈ 0.43, different type - "note_type": "lesson", - "priority": 85, - "content": "new knowledge, maybe overlapping", - }]}}) + data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", # 3/7 ≈ 0.43, different type + "note_type": "lesson", + "priority": 85, + "content": "new knowledge, maybe overlapping", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["status"] == "conflicts_pending" entry = res["notes"][0] @@ -247,18 +302,36 @@ def test_conflict_resolved_with_store(tmp_path): repo = str(tmp_path) cid = "resolve-store" _weak_conflict_setup(repo, cid) - _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "genuinely new", - }]}}) + _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "genuinely new", + } + ] + } + }, + ) # second pass: agent adjudicates 'store' - data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "genuinely new", - "dedup_action": "store", - }]}}) + data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "genuinely new", + "dedup_action": "store", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["status"] == "completed" assert len(_notes_of_type(repo, "lesson")) == 1 @@ -270,17 +343,35 @@ def test_conflict_resolved_with_skip(tmp_path): repo = str(tmp_path) cid = "resolve-skip" _weak_conflict_setup(repo, cid) - _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "maybe dup", - }]}}) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "maybe dup", - "dedup_action": "skip", - }]}}) + _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "maybe dup", + } + ] + } + }, + ) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "maybe dup", + "dedup_action": "skip", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["notes"][0]["status"] == "skipped" assert _notes_of_type(repo, "lesson") == [] @@ -291,18 +382,36 @@ def test_conflict_resolved_with_update_replaces_body(tmp_path): repo = str(tmp_path) cid = "resolve-update" target = _weak_conflict_setup(repo, cid) - _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "maybe dup", - }]}}) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "NEW SUPERSEDING BODY", - "dedup_action": "update", - "target": target, - }]}}) + _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "maybe dup", + } + ] + } + }, + ) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "NEW SUPERSEDING BODY", + "dedup_action": "update", + "target": target, + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["notes"][0]["status"] == "updated" text = (Path(repo) / "repowiki" / target).read_text(encoding="utf-8") @@ -320,18 +429,36 @@ def test_conflict_resolved_with_merge_appends_section(tmp_path): repo = str(tmp_path) cid = "resolve-merge" target = _weak_conflict_setup(repo, cid) - _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "complementary knowledge", - }]}}) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "complementary knowledge", - "dedup_action": "merge", - "target": target, - }]}}) + _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "complementary knowledge", + } + ] + } + }, + ) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "complementary knowledge", + "dedup_action": "merge", + "target": target, + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["notes"][0]["status"] == "merged" text = (Path(repo) / "repowiki" / target).read_text(encoding="utf-8") @@ -346,11 +473,20 @@ def test_update_without_target_reports_error(tmp_path): repo = str(tmp_path) cid = "resolve-no-target" _weak_conflict_setup(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "alpha beta gamma zeta eta", - "note_type": "lesson", - "content": "x", - "dedup_action": "update", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "x", + "dedup_action": "update", + } + ] + } + }, + ) res = by_cid[f"conv-{cid}"] assert res["notes"][0]["status"] == "target_required" diff --git a/tests/test_doctrine_p3.py b/tests/test_doctrine_p3.py index abfae86..df035cd 100644 --- a/tests/test_doctrine_p3.py +++ b/tests/test_doctrine_p3.py @@ -8,6 +8,7 @@ - query_wiki(mode='overview') injects doctrine + scene navigation - consolidate_notes submit cascade: doctrine_hint when doctrine counter is due """ + import json from pathlib import Path @@ -18,34 +19,48 @@ from codewiki.mcp.tools import doctrine as doc_tool from codewiki.mcp.tools import note_consolidation as cons from codewiki.mcp.tools.knowledge_loop import ( - handle_ingest_note, handle_confirm_note, handle_query_wiki, + handle_ingest_note, + handle_confirm_note, + handle_query_wiki, ) # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _set_thresholds(repo: str, cons_t: int = 10, doctrine_t: int = 50, - max_scenes: int = 15, cap: int = 1200): +def _set_thresholds( + repo: str, cons_t: int = 10, doctrine_t: int = 50, max_scenes: int = 15, cap: int = 1200 +): od = Path(repo) / "repowiki" od.mkdir(parents=True, exist_ok=True) - schema = {"conventions": {"aggregation": { - "consolidation_threshold": cons_t, - "doctrine_threshold": doctrine_t, - "hint_interval": 5, - "max_scenarios": max_scenes, - "doctrine_max_chars": cap, - }}} + schema = { + "conventions": { + "aggregation": { + "consolidation_threshold": cons_t, + "doctrine_threshold": doctrine_t, + "hint_interval": 5, + "max_scenarios": max_scenes, + "doctrine_max_chars": cap, + } + } + } (od / "schema.yaml").write_text(yaml.safe_dump(schema), encoding="utf-8") def _ingest_and_confirm(repo: str, title: str) -> str: store = SessionStore() - r = json.loads(handle_ingest_note({ - "output_dir": f"{repo}/repowiki", - "title": title, "note_type": "decision", - "content": "## Background\nbody", "status": "draft", - }, store)) + r = json.loads( + handle_ingest_note( + { + "output_dir": f"{repo}/repowiki", + "title": title, + "note_type": "decision", + "content": "## Background\nbody", + "status": "draft", + }, + store, + ) + ) nf = Path(r["note_path"]).name handle_confirm_note({"output_dir": f"{repo}/repowiki", "note_file": nf}, store) return nf @@ -54,13 +69,18 @@ def _ingest_and_confirm(repo: str, title: str) -> str: def _write_scenario(repo: str, name: str) -> str: sdir = Path(repo) / "repowiki" / "wiki" / "scenarios" sdir.mkdir(parents=True, exist_ok=True) - fm = {"type": "Scenario", "title": name, "status": "draft", - "generated": {"by": "test/agent", "at": "2020-01-01T00:00:00Z"}, - "metadata": {"heat": 1, "summary": f"summary of {name}", - "source_notes": ["notes/seed.md"]}} + fm = { + "type": "Scenario", + "title": name, + "status": "draft", + "generated": {"by": "test/agent", "at": "2020-01-01T00:00:00Z"}, + "metadata": {"heat": 1, "summary": f"summary of {name}", "source_notes": ["notes/seed.md"]}, + } p = sdir / f"{name}.md" - p.write_text("---\n" + yaml.safe_dump(fm, allow_unicode=True) + - "---\n\n## Core SOP\ndo the thing\n", encoding="utf-8") + p.write_text( + "---\n" + yaml.safe_dump(fm, allow_unicode=True) + "---\n\n## Core SOP\ndo the thing\n", + encoding="utf-8", + ) return f"wiki/scenarios/{name}.md" @@ -146,8 +166,7 @@ def test_submit_does_not_keep_backups(tmp_path): repo = str(tmp_path) _set_thresholds(repo) for i in range(2): - resp = _refresh(repo, {"mode": "submit", - "content": f"# Doctrine v{i}\nthesis {i}"}) + resp = _refresh(repo, {"mode": "submit", "content": f"# Doctrine v{i}\nthesis {i}"}) assert resp["status"] == "completed" bdir = Path(repo) / "repowiki" / "wiki" / ".backup" assert not bdir.exists() or not list(bdir.glob("doctrine-*.md")) @@ -161,15 +180,25 @@ def test_query_wiki_overview_injects_doctrine_and_navigation(tmp_path): repo = str(tmp_path) _set_thresholds(repo) _write_scenario(repo, "nav-scene") - _refresh(repo, {"mode": "submit", - "content": "# Team Operating Doctrine\n> Operating Thesis: always lint before release"}) + _refresh( + repo, + { + "mode": "submit", + "content": "# Team Operating Doctrine\n> Operating Thesis: always lint before release", + }, + ) store = SessionStore() - resp = json.loads(handle_query_wiki({ - "output_dir": f"{repo}/repowiki", - "mode": "overview", - "query": "", - }, store)) + resp = json.loads( + handle_query_wiki( + { + "output_dir": f"{repo}/repowiki", + "mode": "overview", + "query": "", + }, + store, + ) + ) assert "doctrine" in resp assert "always lint before release" in resp["doctrine"] assert "scene_navigation" in resp @@ -187,14 +216,24 @@ def test_consolidate_submit_cascades_doctrine_hint(tmp_path): scen = _write_scenario(repo, "cascade-scene") store = SessionStore() - resp = json.loads(cons.handle_consolidate_notes({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "report": {"scenarios": [{ - "file": scen, "action": "updated", - "source_notes": [f"notes/{n1}", f"notes/{n2}"], - }]}, - }, store)) + resp = json.loads( + cons.handle_consolidate_notes( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "report": { + "scenarios": [ + { + "file": scen, + "action": "updated", + "source_notes": [f"notes/{n1}", f"notes/{n2}"], + } + ] + }, + }, + store, + ) + ) assert resp["status"] == "completed" # doctrine counter (2) >= threshold (2) → cascade hint present assert "doctrine_hint" in resp @@ -211,13 +250,23 @@ def test_consolidate_no_doctrine_hint_below_threshold(tmp_path): scen = _write_scenario(repo, "quiet-scene") store = SessionStore() - resp = json.loads(cons.handle_consolidate_notes({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "report": {"scenarios": [{ - "file": scen, "action": "updated", - "source_notes": [f"notes/{n1}"], - }]}, - }, store)) + resp = json.loads( + cons.handle_consolidate_notes( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "report": { + "scenarios": [ + { + "file": scen, + "action": "updated", + "source_notes": [f"notes/{n1}"], + } + ] + }, + }, + store, + ) + ) assert resp["status"] == "completed" assert "doctrine_hint" not in resp diff --git a/tests/test_freshness.py b/tests/test_freshness.py index 6ea37dc..187220a 100644 --- a/tests/test_freshness.py +++ b/tests/test_freshness.py @@ -9,6 +9,7 @@ - backfill script is idempotent and converts verified[] into freshness - wiki_stats exposes freshness {due, fresh} reusing the same judgment """ + from __future__ import annotations import json @@ -40,15 +41,21 @@ def _mk_wiki(tmp_path, freshness=None, default_stale_days=90) -> Path: conv = {"default_stale_days": default_stale_days} if freshness is not None: conv["freshness"] = freshness - (od / "schema.yaml").write_text( - yaml.safe_dump({"conventions": conv}), encoding="utf-8" - ) + (od / "schema.yaml").write_text(yaml.safe_dump({"conventions": conv}), encoding="utf-8") return od -def _write_note(od: Path, name: str, *, ntype="decision", status="stable", - stale_after=None, date=None, verified=None, - title="T") -> Path: +def _write_note( + od: Path, + name: str, + *, + ntype="decision", + status="stable", + stale_after=None, + date=None, + verified=None, + title="T", +) -> Path: fm = {"type": ntype, "title": title, "status": status} if date: fm["metadata"] = {"date": date} @@ -58,8 +65,7 @@ def _write_note(od: Path, name: str, *, ntype="decision", status="stable", fm["verified"] = verified p = od / "notes" / name p.write_text( - "---\n" + yaml.safe_dump(fm, allow_unicode=True, sort_keys=False) - + "---\n\nbody\n", + "---\n" + yaml.safe_dump(fm, allow_unicode=True, sort_keys=False) + "---\n\nbody\n", encoding="utf-8", ) return p @@ -72,6 +78,7 @@ def _stats_due(issues) -> list: def _touch(od: Path, rel_path: str, last_hit: str) -> None: """Seed one hit event (T2: telemetry jsonl replaces the stats table).""" from tests.telemetry_seed import seed_hits + seed_hits(od, {rel_path: (1, last_hit)}) @@ -84,15 +91,18 @@ def _touch(od: Path, rel_path: str, last_hit: str) -> None: # 1. Config fallback chain (F2 helper) # --------------------------------------------------------------------------- # def test_fallback_chain_by_type(): - schema = {"conventions": {"default_stale_days": 90, "freshness": { - "default_window_days": 180, "by_type": {"workaround": 45}}}} + schema = { + "conventions": { + "default_stale_days": 90, + "freshness": {"default_window_days": 180, "by_type": {"workaround": 45}}, + } + } assert freshness_window_days("workaround", schema) == 45 assert freshness_window_days("WORKAROUND", schema) == 45 # case-insensitive def test_fallback_chain_default_window(): - schema = {"conventions": {"default_stale_days": 90, "freshness": { - "default_window_days": 180}}} + schema = {"conventions": {"default_stale_days": 90, "freshness": {"default_window_days": 180}}} assert freshness_window_days("unknown_type", schema) == 180 @@ -139,12 +149,17 @@ def test_judge_retrieval_defer(): def test_judge_date_fallback_uses_type_window(): # No stale_after -> fall back to metadata.date + type window old = (TODAY - timedelta(days=100)).strftime("%Y-%m-%d") - schema_cfg = load_freshness_config({"conventions": {"freshness": { - "default_window_days": 180, "by_type": {"workaround": 45}}}}) + schema_cfg = load_freshness_config( + {"conventions": {"freshness": {"default_window_days": 180, "by_type": {"workaround": 45}}}} + ) # workaround, 100 days old, 45d window -> due - assert evaluate_note_freshness({"date": old, "type": "workaround"}, schema_cfg)["state"] == "due" + assert ( + evaluate_note_freshness({"date": old, "type": "workaround"}, schema_cfg)["state"] == "due" + ) # decision, 100 days old, 365d default -> fresh - assert evaluate_note_freshness({"date": old, "type": "decision"}, schema_cfg)["state"] == "fresh" + assert ( + evaluate_note_freshness({"date": old, "type": "decision"}, schema_cfg)["state"] == "fresh" + ) def test_judge_no_signal_is_fresh(): @@ -162,16 +177,27 @@ def _read_fm(od: Path, name: str) -> dict: def test_ingest_writes_type_window(tmp_path): - od = _mk_wiki(tmp_path, freshness={ - "default_window_days": 180, - "by_type": {"workaround": 45, "decision": 365}, - }) + od = _mk_wiki( + tmp_path, + freshness={ + "default_window_days": 180, + "by_type": {"workaround": 45, "decision": 365}, + }, + ) store = SessionStore() for ntype in ("workaround", "decision"): - r = json.loads(handle_ingest_note({ - "output_dir": str(od), "title": f"n-{ntype}", - "note_type": ntype, "content": "body", "status": "draft", - }, store)) + r = json.loads( + handle_ingest_note( + { + "output_dir": str(od), + "title": f"n-{ntype}", + "note_type": ntype, + "content": "body", + "status": "draft", + }, + store, + ) + ) assert r["status"] == "ingested", r name = Path(r["note_path"]).name fm = _read_fm(od, name) @@ -181,14 +207,11 @@ def test_ingest_writes_type_window(tmp_path): def test_confirm_renews_by_type_window(tmp_path): - od = _mk_wiki(tmp_path, freshness={ - "default_window_days": 180, "by_type": {"workaround": 45}}) + od = _mk_wiki(tmp_path, freshness={"default_window_days": 180, "by_type": {"workaround": 45}}) # Old workaround note whose stale_after has lapsed - _write_note(od, "w.md", ntype="workaround", status="draft", - stale_after=PAST) + _write_note(od, "w.md", ntype="workaround", status="draft", stale_after=PAST) store = SessionStore() - r = json.loads(handle_confirm_note( - {"output_dir": str(od), "note_file": "w.md"}, store)) + r = json.loads(handle_confirm_note({"output_dir": str(od), "note_file": "w.md"}, store)) assert "error" not in r, r fm = _read_fm(od, "w.md") assert fm["status"] == "stable" @@ -212,8 +235,7 @@ def test_lint_flags_lapsed_stale_after(tmp_path): def test_lint_retrieval_defer(tmp_path): - od = _mk_wiki(tmp_path, freshness={ - "default_window_days": 180, "retrieval_defer_days": 60}) + od = _mk_wiki(tmp_path, freshness={"default_window_days": 180, "retrieval_defer_days": 60}) _write_note(od, "lapsed.md", stale_after=PAST) _touch(od, "notes/lapsed.md", (TODAY - timedelta(days=2)).strftime("%Y-%m-%d")) assert _stats_due(_check_stale_notes(od)) == [] @@ -228,16 +250,19 @@ def test_lint_skips_draft_and_deprecated(tmp_path): def test_lint_dispatch_reads_schema_config(tmp_path): # Tight window configured -> dispatch (no hardcoded args) must honor it. - od = _mk_wiki(tmp_path, freshness={ - "default_window_days": 180, "by_type": {"workaround": 5}}) + od = _mk_wiki(tmp_path, freshness={"default_window_days": 180, "by_type": {"workaround": 5}}) # workaround confirmed 10 days ago (window 5) with a fresh date field: # old logic (date-age 90d) would NOT flag; new type-aware logic MUST. - _write_note(od, "wa.md", ntype="workaround", status="stable", - date=(TODAY - timedelta(days=10)).strftime("%Y-%m-%d"), - stale_after=(TODAY - timedelta(days=5)).strftime("%Y-%m-%d")) + _write_note( + od, + "wa.md", + ntype="workaround", + status="stable", + date=(TODAY - timedelta(days=10)).strftime("%Y-%m-%d"), + stale_after=(TODAY - timedelta(days=5)).strftime("%Y-%m-%d"), + ) store = SessionStore() - resp = json.loads(handle_lint_wiki( - {"output_dir": str(od), "checks": ["stale_notes"]}, store)) + resp = json.loads(handle_lint_wiki({"output_dir": str(od), "checks": ["stale_notes"]}, store)) files = [i["file"] for i in resp.get("issues", []) if i["check"] == "stale_notes"] assert files == ["notes/wa.md"] @@ -246,8 +271,11 @@ def test_lint_no_double_report_with_okf(tmp_path): od = _mk_wiki(tmp_path, freshness={"default_window_days": 180}) _write_note(od, "lapsed.md", stale_after=PAST) store = SessionStore() - resp = json.loads(handle_lint_wiki( - {"output_dir": str(od), "checks": ["stale_notes", "okf_conformance"]}, store)) + resp = json.loads( + handle_lint_wiki( + {"output_dir": str(od), "checks": ["stale_notes", "okf_conformance"]}, store + ) + ) lapsed_issues = [i for i in resp.get("issues", []) if i["file"] == "notes/lapsed.md"] checks = {i["check"] for i in lapsed_issues} assert checks == {"stale_notes"} # not double-reported by okf_conformance @@ -260,8 +288,12 @@ def test_okf_still_audits_notes_when_stale_check_absent(tmp_path): assert len(issues) == 1 # Without stale_notes in checks, okf_conformance still flags the note. from codewiki.mcp.tools.wiki_lint import _check_okf_conformance - okf = [i for i in _check_okf_conformance(od, skip_notes_staleness=False) - if i["file"] == "notes/lapsed.md"] + + okf = [ + i + for i in _check_okf_conformance(od, skip_notes_staleness=False) + if i["file"] == "notes/lapsed.md" + ] assert any("stale_after" in i["message"] for i in okf) @@ -270,6 +302,7 @@ def test_okf_still_audits_notes_when_stale_check_absent(tmp_path): # --------------------------------------------------------------------------- # def _run_migrate(od: Path, dry_run=False): import importlib.util + spec = importlib.util.spec_from_file_location( "migrate_freshness", Path(__file__).resolve().parents[1] / "scripts" / "migrate_freshness.py", @@ -280,13 +313,16 @@ def _run_migrate(od: Path, dry_run=False): def test_backfill_renews_from_verified_and_is_idempotent(tmp_path): - od = _mk_wiki(tmp_path, freshness={ - "default_window_days": 180, "by_type": {"decision": 365}}) + od = _mk_wiki(tmp_path, freshness={"default_window_days": 180, "by_type": {"decision": 365}}) verified_at = TODAY - timedelta(days=10) - _write_note(od, "v.md", ntype="decision", status="stable", - stale_after=PAST, - verified=[{"by": "human:alice", - "at": verified_at.strftime("%Y-%m-%dT00:00:00Z")}]) + _write_note( + od, + "v.md", + ntype="decision", + status="stable", + stale_after=PAST, + verified=[{"by": "human:alice", "at": verified_at.strftime("%Y-%m-%dT00:00:00Z")}], + ) _write_note(od, "nv.md", ntype="decision", status="stable", stale_after=PAST) stats = _run_migrate(od) @@ -305,8 +341,14 @@ def test_backfill_renews_from_verified_and_is_idempotent(tmp_path): def test_backfill_dry_run_writes_nothing(tmp_path): od = _mk_wiki(tmp_path, freshness={"by_type": {"decision": 365}}) verified_at = TODAY - timedelta(days=10) - _write_note(od, "v.md", ntype="decision", status="stable", stale_after=PAST, - verified=[{"by": "x", "at": verified_at.strftime("%Y-%m-%dT00:00:00Z")}]) + _write_note( + od, + "v.md", + ntype="decision", + status="stable", + stale_after=PAST, + verified=[{"by": "x", "at": verified_at.strftime("%Y-%m-%dT00:00:00Z")}], + ) before = (od / "notes" / "v.md").read_text(encoding="utf-8") stats = _run_migrate(od, dry_run=True) assert stats["updated"] == 1 diff --git a/tests/test_friction.py b/tests/test_friction.py index 08819f4..c0fc314 100644 --- a/tests/test_friction.py +++ b/tests/test_friction.py @@ -13,6 +13,7 @@ Design reference: docs/知识飞轮增强设计方案-P0三项.md §2. """ + import importlib.util import json import sys @@ -32,6 +33,7 @@ # K1: score_friction pure-function matrix # --------------------------------------------------------------------------- # + def _u(content: str) -> dict: return {"role": "user", "content": content} @@ -42,10 +44,14 @@ def _a(content: str = "ok, done") -> dict: def test_two_corrections_trigger_suggest_distill(): turns = [ - _u("帮我看看这个函数"), _a(), - _u("不对，这个逻辑有问题"), _a(), - _u("应该是先校验再处理"), _a(), - _u("现在对了，谢谢"), _a(), + _u("帮我看看这个函数"), + _a(), + _u("不对，这个逻辑有问题"), + _a(), + _u("应该是先校验再处理"), + _a(), + _u("现在对了，谢谢"), + _a(), ] r = score_friction(turns) assert r["signals"]["correction"] == 2 @@ -104,8 +110,14 @@ def test_repeat_detection_normalized(): # Whitespace-insensitive + case-insensitive comparison; a run of >2 # identical adjacent user turns counts as ONE group. turns = [ - _u("请重试 Scan"), _a(), _u("请重试scan"), _a(), _u(" 请 重 试 scan "), - _a(), _u("换个话题"), _a(), + _u("请重试 Scan"), + _a(), + _u("请重试scan"), + _a(), + _u(" 请 重 试 scan "), + _a(), + _u("换个话题"), + _a(), ] r = score_friction(turns) assert r["signals"]["repeat"] == 1 @@ -116,8 +128,14 @@ def test_repeat_detection_normalized(): def test_repeat_non_adjacent_not_counted(): turns = [ - _u("再来一次"), _a(), _u("别的请求"), _a(), _u("再来一次"), _a(), - _u("结尾"), _a(), + _u("再来一次"), + _a(), + _u("别的请求"), + _a(), + _u("再来一次"), + _a(), + _u("结尾"), + _a(), ] r = score_friction(turns) assert r["signals"]["repeat"] == 0 @@ -125,8 +143,14 @@ def test_repeat_non_adjacent_not_counted(): def test_interrupt_marker_counts(): turns = [ - _u("继续"), _a(), _u("[Request interrupted by user for tool use]"), _a(), - _u("[Request interrupted"), _a(), _u("好了"), _a(), + _u("继续"), + _a(), + _u("[Request interrupted by user for tool use]"), + _a(), + _u("[Request interrupted"), + _a(), + _u("好了"), + _a(), ] r = score_friction(turns) assert r["signals"]["interrupt"] == 2 @@ -138,8 +162,14 @@ def test_interrupt_marker_counts(): def test_correction_counted_once_per_turn(): # Multiple keywords in one user turn → still a single correction. turns = [ - _u("不对，你搞错了，不是这样的"), _a(), _u("应该是这样"), _a(), - _u("嗯"), _a(), _u("好"), _a(), + _u("不对，你搞错了，不是这样的"), + _a(), + _u("应该是这样"), + _a(), + _u("嗯"), + _a(), + _u("好"), + _a(), ] r = score_friction(turns) assert r["signals"]["correction"] == 2 @@ -147,7 +177,14 @@ def test_correction_counted_once_per_turn(): def test_config_override_threshold(): turns = [ - _u("不对"), _a(), _u("错了"), _a(), _u("继续"), _a(), _u("完成"), _a(), + _u("不对"), + _a(), + _u("错了"), + _a(), + _u("继续"), + _a(), + _u("完成"), + _a(), ] base = score_friction(turns) assert base["score"] == 40 @@ -171,7 +208,14 @@ def test_config_override_keywords_and_markers(): # "重来" is a DEFAULT correction keyword; use a phrase outside the default # vocabulary so custom-config behaviour can be observed in isolation. turns = [ - _u("再跑一次那个命令"), _a(), _u("xxx"), _a(), _u("yyy"), _a(), _u("zzz"), _a(), + _u("再跑一次那个命令"), + _a(), + _u("xxx"), + _a(), + _u("yyy"), + _a(), + _u("zzz"), + _a(), ] # Default vocabulary: neither a correction nor an interrupt. base = score_friction(turns) @@ -187,8 +231,7 @@ def test_config_override_keywords_and_markers(): def test_format_friction_signals_line(): - s = format_friction_signals({"correction": 2, "interrupt": 0, "repeat": 1, - "user_turns": 9}) + s = format_friction_signals({"correction": 2, "interrupt": 0, "repeat": 1, "user_turns": 9}) assert s == "correction=2,interrupt=0,repeat=1,user_turns=9" # No YAML-special characters (survives naive line scanners). assert ":" not in s @@ -198,6 +241,7 @@ def test_format_friction_signals_line(): # K2: capture_conversation integration # --------------------------------------------------------------------------- # + def _capture(repo: Path, conversation, **kwargs) -> dict: args = {"output_dir": str(repo / "repowiki"), "conversation": conversation} args.update(kwargs) @@ -231,10 +275,8 @@ def test_capture_writes_friction_frontmatter_and_returns_friction(tmp_path): raw_file = tmp_path / "repowiki" / "raw" / Path(result["stored_at"]).name text = raw_file.read_text(encoding="utf-8") fm_block = text.split("---")[1] if text.startswith("---") else text - score_lines = [ln for ln in fm_block.splitlines() - if ln.startswith("friction_score:")] - signal_lines = [ln for ln in fm_block.splitlines() - if ln.startswith("friction_signals:")] + score_lines = [ln for ln in fm_block.splitlines() if ln.startswith("friction_score:")] + signal_lines = [ln for ln in fm_block.splitlines() if ln.startswith("friction_signals:")] assert len(score_lines) == 1 assert len(signal_lines) == 1 assert score_lines[0] == f"friction_score: {fr['score']}" @@ -257,15 +299,16 @@ def test_capture_writes_zero_score_too(tmp_path): def test_supersede_refreshes_friction_score(tmp_path): # First capture: calm conversation, low score. - calm = [_u("开始任务"), _a("好的"), _u("继续"), _a("继续中"), - _u("还有一步"), _a("完成")] + calm = [_u("开始任务"), _a("好的"), _u("继续"), _a("继续中"), _u("还有一步"), _a("完成")] first = _capture(tmp_path, calm, source_session_id="sess-1") assert first["friction"]["score"] == 0 # Same IDE session re-captured with a growing, friction-heavy transcript. heated = calm + [ - _u("不对，这里逻辑反了"), _a("抱歉，我改一下"), - _u("应该是先做校验"), _a("已修复。"), + _u("不对，这里逻辑反了"), + _a("抱歉，我改一下"), + _u("应该是先做校验"), + _a("已修复。"), ] second = _capture(tmp_path, heated, source_session_id="sess-1") assert second["status"] == "captured" @@ -287,11 +330,12 @@ def test_supersede_refreshes_friction_score(tmp_path): # K3: distill prepare ordering + get_task_context friction payload # --------------------------------------------------------------------------- # + def _write_raw_with_friction(repo: Path, name: str, score: int, task_id: str = "") -> Path: raw_dir = repo / "repowiki" / "raw" raw_dir.mkdir(parents=True, exist_ok=True) p = raw_dir / name - extra = f"task_id: \"{task_id}\"\n" if task_id else "" + extra = f'task_id: "{task_id}"\n' if task_id else "" p.write_text( "---\n" "type: conversation\n" @@ -311,10 +355,15 @@ def test_prepare_lists_captures_by_friction_desc(tmp_path): _write_raw_with_friction(tmp_path, "conv-mid.md", 20) _write_raw_with_friction(tmp_path, "conv-legacy.md", 0) # pre-K-line: no key - out = json.loads(distill.handle_distill_conversation({ - "output_dir": str(tmp_path / "repowiki"), - "mode": "prepare", - }, SessionStore())) + out = json.loads( + distill.handle_distill_conversation( + { + "output_dir": str(tmp_path / "repowiki"), + "mode": "prepare", + }, + SessionStore(), + ) + ) assert out["status"] == "prepared" scores = [c["friction_score"] for c in out["captures"]] @@ -331,10 +380,15 @@ def test_prepare_lists_captures_by_friction_desc(tmp_path): def test_prepare_no_hint_when_all_calm(tmp_path): _write_raw_with_friction(tmp_path, "conv-calm.md", 5) - out = json.loads(distill.handle_distill_conversation({ - "output_dir": str(tmp_path / "repowiki"), - "mode": "prepare", - }, SessionStore())) + out = json.loads( + distill.handle_distill_conversation( + { + "output_dir": str(tmp_path / "repowiki"), + "mode": "prepare", + }, + SessionStore(), + ) + ) assert out["status"] == "prepared" assert "friction_hint" not in out assert out["captures"][0]["friction_score"] == 5 @@ -345,8 +399,7 @@ def test_get_task_context_pending_raws_carry_friction(tmp_path): od = str(repo / "repowiki") store = SessionStore() - r = json.loads(tm.handle_create_task( - {"output_dir": od, "title": "摩擦信号机制"}, store)) + r = json.loads(tm.handle_create_task({"output_dir": od, "title": "摩擦信号机制"}, store)) assert r["ok"] is True task_id = r["task"]["id"] @@ -355,8 +408,7 @@ def test_get_task_context_pending_raws_carry_friction(tmp_path): _capture(repo, calm, task_id=task_id) _capture(repo, _correction_conversation(), task_id=task_id) - ctx = json.loads(tm.handle_get_task_context( - {"output_dir": od, "task_id": task_id}, store)) + ctx = json.loads(tm.handle_get_task_context({"output_dir": od, "task_id": task_id}, store)) assert ctx["ok"] is True assert ctx["pending_raw_count"] == 2 entries = ctx["pending_raws"] @@ -394,7 +446,7 @@ def test_hook_friction_hint_for_high_score(tmp_path): (raw_dir / "conv-a.md").write_text( "---\n" "status: pending\n" - "task_id: \"task-x\"\n" + 'task_id: "task-x"\n' "friction_score: 35\n" "friction_signals: correction=2,interrupt=0,repeat=0,user_turns=9\n" "---\n\nuser: hi", @@ -418,8 +470,7 @@ def test_hook_no_hint_for_low_or_missing_score(tmp_path): assert hook._latest_friction_hint(str(tmp_path)) == "" # Pre-K-line file without the key: silent. - (raw_dir / "conv-b.md").write_text( - "---\nstatus: pending\n---\n\nuser: hi", encoding="utf-8") + (raw_dir / "conv-b.md").write_text("---\nstatus: pending\n---\n\nuser: hi", encoding="utf-8") assert hook._latest_friction_hint(str(tmp_path)) == "" @@ -451,14 +502,25 @@ def test_hook_message_embeds_friction_hint(tmp_path): "---\n\nuser: hi", encoding="utf-8", ) - event = json.dumps({"session_id": "s", "cwd": str(tmp_path), - "hook_event_name": "SessionStart", "source": "startup"}) + event = json.dumps( + { + "session_id": "s", + "cwd": str(tmp_path), + "hook_event_name": "SessionStart", + "source": "startup", + } + ) env = dict(os.environ) env["CODEBUDDY_PROJECT_DIR"] = str(tmp_path) env["PYTHONUTF8"] = "1" proc = subprocess.run( - [sys.executable, str(HOOK_PATH)], input=event, capture_output=True, - text=True, encoding="utf-8", env=env, timeout=30, + [sys.executable, str(HOOK_PATH)], + input=event, + capture_output=True, + text=True, + encoding="utf-8", + env=env, + timeout=30, ) assert proc.returncode == 0, proc.stderr out = json.loads(proc.stdout) diff --git a/tests/test_hook_registry.py b/tests/test_hook_registry.py index 9bb0e73..25bacd7 100644 --- a/tests/test_hook_registry.py +++ b/tests/test_hook_registry.py @@ -9,6 +9,7 @@ - CLI query: delimited block output, coverage/usage/matched fields present, --check mode lightweight, projection reuses the MCP handler """ + from __future__ import annotations from pathlib import Path @@ -105,16 +106,15 @@ def _mk_wiki(tmp_path) -> Path: od = tmp_path / "repowiki" (od / "notes").mkdir(parents=True) (od / "notes" / "pitfall-port.md").write_text( - "---\ntype: pitfall\ntitle: 端口冲突排查\nstatus: stable\n---\n\n" - "端口冲突用 lsof 排查。\n", + "---\ntype: pitfall\ntitle: 端口冲突排查\nstatus: stable\n---\n\n端口冲突用 lsof 排查。\n", encoding="utf-8", ) (od / "notes" / "unrelated.md").write_text( - "---\ntype: lesson\ntitle: 无关笔记\nstatus: stable\n---\n\n" - "数据库索引优化。\n", + "---\ntype: lesson\ntitle: 无关笔记\nstatus: stable\n---\n\n数据库索引优化。\n", encoding="utf-8", ) from codewiki.mcp.tools.wiki_search import build_full_index + build_full_index(od, session=None) return od @@ -122,6 +122,7 @@ def _mk_wiki(tmp_path) -> Path: class TestCliQuery: def test_delimited_block_output(self, tmp_path): from codewiki.cli.commands.query import query_command + od = _mk_wiki(tmp_path) res = CliRunner().invoke(query_command, ["端口冲突", "--output-dir", str(od)]) assert res.exit_code == 0, res.output @@ -135,47 +136,50 @@ def test_delimited_block_output(self, tmp_path): def test_missing_terms_noted(self, tmp_path): from codewiki.cli.commands.query import query_command + od = _mk_wiki(tmp_path) - res = CliRunner().invoke( - query_command, ["端口冲突 量子", "--output-dir", str(od)]) + res = CliRunner().invoke(query_command, ["端口冲突 量子", "--output-dir", str(od)]) assert res.exit_code == 0 assert "missing_terms: 量子" in res.output assert "topically adjacent" in res.output def test_check_mode_lightweight(self, tmp_path): from codewiki.cli.commands.query import query_command + od = _mk_wiki(tmp_path) - res = CliRunner().invoke( - query_command, ["端口冲突", "--check", "--output-dir", str(od)]) + res = CliRunner().invoke(query_command, ["端口冲突", "--check", "--output-dir", str(od)]) assert res.exit_code == 0 assert "relevant: true" in res.output assert "top_score:" in res.output assert "snippet" not in res.output # lightweight: no bodies # check must not record telemetry hits from codewiki.mcp.tools import telemetry + agg = telemetry.aggregate_usage(od) assert not agg.get("notes/pitfall-port.md", {}).get("hits") def test_missing_output_dir_errors(self, tmp_path): from codewiki.cli.commands.query import query_command - res = CliRunner().invoke( - query_command, ["x", "--output-dir", str(tmp_path / "nope")]) + + res = CliRunner().invoke(query_command, ["x", "--output-dir", str(tmp_path / "nope")]) assert res.exit_code == 2 def test_full_search_records_telemetry(self, tmp_path): # the projection reuses the MCP handler → hit telemetry recorded from codewiki.cli.commands.query import query_command + od = _mk_wiki(tmp_path) CliRunner().invoke(query_command, ["端口冲突", "--output-dir", str(od)]) from codewiki.mcp.tools import telemetry + agg = telemetry.aggregate_usage(od) assert agg.get("notes/pitfall-port.md", {}).get("hits", 0) >= 1 def test_expand_flag(self, tmp_path): from codewiki.cli.commands.query import query_command + od = _mk_wiki(tmp_path) - res = CliRunner().invoke( - query_command, ["端口冲突", "--output-dir", str(od), "--expand"]) + res = CliRunner().invoke(query_command, ["端口冲突", "--output-dir", str(od), "--expand"]) assert res.exit_code == 0 assert "lsof" in res.output # full page content included @@ -187,16 +191,18 @@ class TestPromptRegistryDriven: def test_prompt_contains_tiers_and_detection(self, tmp_path): (tmp_path / ".codebuddy").mkdir() from codewiki.mcp.prompts import _prompt_team_memory_hook + s = _prompt_team_memory_hook({"repo_path": str(tmp_path)}) assert "hooks.yaml" in s assert "已验证支持" in s and "理论支持" in s - assert "`codebuddy`" in s # detected in this fake repo - assert "cursor 家族采集降级" in s # downgrade disclosed + assert "`codebuddy`" in s # detected in this fake repo + assert "cursor 家族采集降级" in s # downgrade disclosed assert "只为探测到的智能体接线" in s def test_prompt_equivalence_for_verified(self): # regression: existing wiring steps must survive the rewrite from codewiki.mcp.prompts import _prompt_team_memory_hook + s = _prompt_team_memory_hook({"repo_path": "."}) assert "install-hooks" in s assert "settings.json" in s @@ -209,6 +215,7 @@ def test_prompt_no_ambiguous_installed_agents_wording(self, tmp_path): # IDE agent to wire Qoder/Claude Code in a repo that only had # .codebuddy — wording must say "已探测" (detected), never "已安装". from codewiki.mcp.prompts import _prompt_team_memory_hook + s = _prompt_team_memory_hook({"repo_path": str(tmp_path), "action": "enable"}) assert "覆盖全部已探测" in s assert "覆盖全部已安装" not in s @@ -221,8 +228,7 @@ def test_init_wiki_prompt_carries_wiring_guardrail(self): # three IDEs without any guardrail, letting agents wire (and create) # .qoder/.claude dirs in repos that never used those tools. from codewiki.mcp.prompts import _prompt_init_wiki - s = _prompt_init_wiki( - {"repo_path": ".", "enable_task_management": "true"} - ) + + s = _prompt_init_wiki({"repo_path": ".", "enable_task_management": "true"}) assert "只为项目根目录已存在配置目录的智能体接线" in s assert "绝不主动新建" in s diff --git a/tests/test_ide_hook_capture.py b/tests/test_ide_hook_capture.py index 58df58a..bf88853 100644 --- a/tests/test_ide_hook_capture.py +++ b/tests/test_ide_hook_capture.py @@ -61,14 +61,18 @@ def test_envelope_does_not_supersede_full_transcript(enable_hook, monkeypatch, t sid = "ide-session-abc" # 1) Stop fires with a full inline transcript -> captured with source_session_id - rc1 = _run_hook_stdin(monkeypatch, { - "hook_event_name": "Stop", - "session_id": sid, - "conversation": [ - {"role": "user", "content": "real question with substance"}, - {"role": "assistant", "content": "real detailed answer"}, - ], - }, repo) + rc1 = _run_hook_stdin( + monkeypatch, + { + "hook_event_name": "Stop", + "session_id": sid, + "conversation": [ + {"role": "user", "content": "real question with substance"}, + {"role": "assistant", "content": "real detailed answer"}, + ], + }, + repo, + ) assert rc1 == 0 files = _raw_files(repo) @@ -80,11 +84,15 @@ def test_envelope_does_not_supersede_full_transcript(enable_hook, monkeypatch, t # 2) SessionEnd fires for the SAME session but WITHOUT any transcript. # Before the fix, the synthesized envelope carried source_session_id and # supersede-replaced the full transcript -> data loss. - rc2 = _run_hook_stdin(monkeypatch, { - "hook_event_name": "SessionEnd", - "session_id": sid, - "cwd": str(repo), - }, repo) + rc2 = _run_hook_stdin( + monkeypatch, + { + "hook_event_name": "SessionEnd", + "session_id": sid, + "cwd": str(repo), + }, + repo, + ) assert rc2 == 0 files = _raw_files(repo) @@ -106,10 +114,14 @@ def test_envelope_only_fires_for_lifecycle_events(enable_hook, monkeypatch, tmp_ repo = tmp_path / "repo" repo.mkdir() - rc = _run_hook_stdin(monkeypatch, { - "hook_event_name": "UserPromptSubmit", - "session_id": "s1", - }, repo) + rc = _run_hook_stdin( + monkeypatch, + { + "hook_event_name": "UserPromptSubmit", + "session_id": "s1", + }, + repo, + ) assert rc == 0 assert not (repo / "repowiki" / "raw").exists() or not _raw_files(repo) @@ -117,20 +129,23 @@ def test_envelope_only_fires_for_lifecycle_events(enable_hook, monkeypatch, tmp_ # --------------------------------------------------------------------------- # # Issue #2: inline turns under several common keys # --------------------------------------------------------------------------- # -@pytest.mark.parametrize("key", ["conversation", "messages", "turns", - "transcript_turns", "chat"]) +@pytest.mark.parametrize("key", ["conversation", "messages", "turns", "transcript_turns", "chat"]) def test_inline_turns_keys_are_captured(enable_hook, monkeypatch, tmp_path, key): repo = tmp_path / "repo" repo.mkdir() - rc = _run_hook_stdin(monkeypatch, { - "hook_event_name": "SessionEnd", - "session_id": "s2", - key: [ - {"role": "user", "content": f"inline via {key}"}, - {"role": "assistant", "content": "reply"}, - ], - }, repo) + rc = _run_hook_stdin( + monkeypatch, + { + "hook_event_name": "SessionEnd", + "session_id": "s2", + key: [ + {"role": "user", "content": f"inline via {key}"}, + {"role": "assistant", "content": "reply"}, + ], + }, + repo, + ) assert rc == 0 files = _raw_files(repo) assert len(files) == 1 @@ -149,8 +164,7 @@ def test_load_transcript_supports_wrapper_keys(tmp_path): """Transcript files wrapped in transcript_turns/chat keys must be readable.""" for key in ("conversation", "messages", "turns", "transcript_turns", "chat"): f = tmp_path / f"t-{key}.json" - f.write_text(json.dumps({key: [{"role": "user", "content": "hi"}]}), - encoding="utf-8") + f.write_text(json.dumps({key: [{"role": "user", "content": "hi"}]}), encoding="utf-8") turns = _ide_hook._load_transcript(str(f)) assert turns == [{"role": "user", "content": "hi"}], f"key={key} failed" @@ -168,12 +182,14 @@ def _make_codebuddy_transcript(tmp_path: Path, messages: list[dict]) -> Path: index_messages = [] for msg in messages: msg_id = msg["id"] - index_messages.append({ - "id": msg_id, - "type": "text", - "role": msg["role"], - "isComplete": True, - }) + index_messages.append( + { + "id": msg_id, + "type": "text", + "role": msg["role"], + "isComplete": True, + } + ) msg_file = msgs_dir / f"{msg_id}.json" msg_file.write_text(json.dumps(msg["file_data"]), encoding="utf-8") @@ -184,42 +200,54 @@ def _make_codebuddy_transcript(tmp_path: Path, messages: list[dict]) -> Path: def test_expand_codebuddy_index(tmp_path): """CodeBuddy index.json + messages/ dir must expand into full turns.""" - index_file = _make_codebuddy_transcript(tmp_path, [ - { - "id": "m1", "role": "user", - "file_data": { + index_file = _make_codebuddy_transcript( + tmp_path, + [ + { + "id": "m1", "role": "user", - "message": json.dumps({ + "file_data": { "role": "user", - "content": [{"type": "text", "text": "hello world"}], - }), + "message": json.dumps( + { + "role": "user", + "content": [{"type": "text", "text": "hello world"}], + } + ), + }, }, - }, - { - "id": "m2", "role": "assistant", - "file_data": { + { + "id": "m2", "role": "assistant", - "message": json.dumps({ + "file_data": { "role": "assistant", - "content": [ - {"type": "reasoning", "text": "thinking..."}, - {"type": "tool-call", "toolName": "list_dir", "args": {}}, - {"type": "text", "text": "here is the answer"}, - ], - }), + "message": json.dumps( + { + "role": "assistant", + "content": [ + {"type": "reasoning", "text": "thinking..."}, + {"type": "tool-call", "toolName": "list_dir", "args": {}}, + {"type": "text", "text": "here is the answer"}, + ], + } + ), + }, }, - }, - { - "id": "m3", "role": "tool", - "file_data": { + { + "id": "m3", "role": "tool", - "message": json.dumps({ + "file_data": { "role": "tool", - "content": [{"type": "tool-result", "text": "file listing"}], - }), + "message": json.dumps( + { + "role": "tool", + "content": [{"type": "tool-result", "text": "file listing"}], + } + ), + }, }, - }, - ]) + ], + ) turns = _ide_hook._load_transcript(str(index_file)) assert turns is not None @@ -231,58 +259,76 @@ def test_expand_codebuddy_index(tmp_path): def test_expand_codebuddy_index_only_user_assistant(tmp_path): """Only user/assistant dialogue is kept; system/thinking/other roles dropped.""" - index_file = _make_codebuddy_transcript(tmp_path, [ - { - "id": "m0", "role": "system", - "file_data": { + index_file = _make_codebuddy_transcript( + tmp_path, + [ + { + "id": "m0", "role": "system", - "message": json.dumps({ + "file_data": { "role": "system", - "content": [{"type": "text", "text": "You are a helpful assistant."}], - }), + "message": json.dumps( + { + "role": "system", + "content": [{"type": "text", "text": "You are a helpful assistant."}], + } + ), + }, }, - }, - { - "id": "m1", "role": "user", - "file_data": { + { + "id": "m1", "role": "user", - "message": json.dumps({ + "file_data": { "role": "user", - "content": [{"type": "text", "text": "user question"}], - }), + "message": json.dumps( + { + "role": "user", + "content": [{"type": "text", "text": "user question"}], + } + ), + }, }, - }, - { - "id": "m2", "role": "assistant", - "file_data": { + { + "id": "m2", "role": "assistant", - "message": json.dumps({ + "file_data": { "role": "assistant", - "content": [{"type": "text", "text": "assistant answer"}], - }), + "message": json.dumps( + { + "role": "assistant", + "content": [{"type": "text", "text": "assistant answer"}], + } + ), + }, }, - }, - { - "id": "m3", "role": "thinking", - "file_data": { + { + "id": "m3", "role": "thinking", - "message": json.dumps({ + "file_data": { "role": "thinking", - "content": [{"type": "text", "text": "internal thought"}], - }), + "message": json.dumps( + { + "role": "thinking", + "content": [{"type": "text", "text": "internal thought"}], + } + ), + }, }, - }, - { - "id": "m4", "role": "tool", - "file_data": { + { + "id": "m4", "role": "tool", - "message": json.dumps({ + "file_data": { "role": "tool", - "content": [{"type": "text", "text": "tool output"}], - }), + "message": json.dumps( + { + "role": "tool", + "content": [{"type": "text", "text": "tool output"}], + } + ), + }, }, - }, - ]) + ], + ) turns = _ide_hook._load_transcript(str(index_file)) assert turns == [ @@ -299,16 +345,30 @@ def test_expand_codebuddy_index_skips_missing_files(tmp_path): msgs_dir.mkdir() # Only create file for m1, not m2 - (msgs_dir / "m1.json").write_text(json.dumps({ - "role": "user", - "message": json.dumps({"role": "user", "content": [{"type": "text", "text": "hi"}]}), - }), encoding="utf-8") + (msgs_dir / "m1.json").write_text( + json.dumps( + { + "role": "user", + "message": json.dumps( + {"role": "user", "content": [{"type": "text", "text": "hi"}]} + ), + } + ), + encoding="utf-8", + ) index_file = session_dir / "index.json" - index_file.write_text(json.dumps({"messages": [ - {"id": "m1", "type": "text", "role": "user", "isComplete": True}, - {"id": "m2", "type": "text", "role": "assistant", "isComplete": True}, - ]}), encoding="utf-8") + index_file.write_text( + json.dumps( + { + "messages": [ + {"id": "m1", "type": "text", "role": "user", "isComplete": True}, + {"id": "m2", "type": "text", "role": "assistant", "isComplete": True}, + ] + } + ), + encoding="utf-8", + ) turns = _ide_hook._load_transcript(str(index_file)) assert turns == [{"role": "user", "content": "hi"}] @@ -317,11 +377,16 @@ def test_expand_codebuddy_index_skips_missing_files(tmp_path): def test_expand_codebuddy_index_not_an_index(tmp_path): """A messages list with inline content is NOT treated as an index.""" f = tmp_path / "t.json" - f.write_text(json.dumps({ - "messages": [ - {"id": "m1", "role": "user", "content": "inline content"}, - ], - }), encoding="utf-8") + f.write_text( + json.dumps( + { + "messages": [ + {"id": "m1", "role": "user", "content": "inline content"}, + ], + } + ), + encoding="utf-8", + ) turns = _ide_hook._load_transcript(str(f)) assert turns == [{"id": "m1", "role": "user", "content": "inline content"}] @@ -331,34 +396,47 @@ def test_codebuddy_transcript_e2e(enable_hook, monkeypatch, tmp_path): repo = tmp_path / "repo" repo.mkdir() - index_file = _make_codebuddy_transcript(tmp_path, [ - { - "id": "m1", "role": "user", - "file_data": { + index_file = _make_codebuddy_transcript( + tmp_path, + [ + { + "id": "m1", "role": "user", - "message": json.dumps({ + "file_data": { "role": "user", - "content": [{"type": "text", "text": "what is CodeWiki?"}], - }), + "message": json.dumps( + { + "role": "user", + "content": [{"type": "text", "text": "what is CodeWiki?"}], + } + ), + }, }, - }, - { - "id": "m2", "role": "assistant", - "file_data": { + { + "id": "m2", "role": "assistant", - "message": json.dumps({ + "file_data": { "role": "assistant", - "content": [{"type": "text", "text": "CodeWiki is an LLM wiki."}], - }), + "message": json.dumps( + { + "role": "assistant", + "content": [{"type": "text", "text": "CodeWiki is an LLM wiki."}], + } + ), + }, }, - }, - ]) + ], + ) - rc = _run_hook_stdin(monkeypatch, { - "hook_event_name": "SessionEnd", - "session_id": "cb-session-001", - "transcript_path": str(index_file), - }, repo) + rc = _run_hook_stdin( + monkeypatch, + { + "hook_event_name": "SessionEnd", + "session_id": "cb-session-001", + "transcript_path": str(index_file), + }, + repo, + ) assert rc == 0 files = _raw_files(repo) @@ -372,37 +450,71 @@ def test_codebuddy_transcript_e2e(enable_hook, monkeypatch, tmp_path): def test_extract_codebuddy_message_text_variants(): """Unit tests for _extract_codebuddy_message_text edge cases.""" # JSON string message with content blocks - assert _ide_hook._extract_codebuddy_message_text({ - "message": json.dumps({"content": [{"type": "text", "text": "abc"}]}), - }) == "abc" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "message": json.dumps({"content": [{"type": "text", "text": "abc"}]}), + } + ) + == "abc" + ) # Plain string message (not JSON) - assert _ide_hook._extract_codebuddy_message_text({ - "message": "plain text", - }) == "plain text" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "message": "plain text", + } + ) + == "plain text" + ) # Dict message - assert _ide_hook._extract_codebuddy_message_text({ - "message": {"content": [{"type": "text", "text": "xyz"}]}, - }) == "xyz" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "message": {"content": [{"type": "text", "text": "xyz"}]}, + } + ) + == "xyz" + ) # Fallback to top-level content - assert _ide_hook._extract_codebuddy_message_text({ - "content": [{"type": "text", "text": "fallback"}], - }) == "fallback" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "content": [{"type": "text", "text": "fallback"}], + } + ) + == "fallback" + ) # String content - assert _ide_hook._extract_codebuddy_message_text({ - "content": "direct string", - }) == "direct string" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "content": "direct string", + } + ) + == "direct string" + ) # All noise -> empty - assert _ide_hook._extract_codebuddy_message_text({ - "message": json.dumps({"content": [ - {"type": "reasoning", "text": "thinking"}, - {"type": "tool-call", "toolName": "x"}, - ]}), - }) == "" + assert ( + _ide_hook._extract_codebuddy_message_text( + { + "message": json.dumps( + { + "content": [ + {"type": "reasoning", "text": "thinking"}, + {"type": "tool-call", "toolName": "x"}, + ] + } + ), + } + ) + == "" + ) # --------------------------------------------------------------------------- # @@ -412,9 +524,16 @@ def test_hook_disabled_by_default(monkeypatch, tmp_path): monkeypatch.delenv("CODEWIKI_TEAM_MEMORY_HOOK", raising=False) repo = tmp_path / "repo" repo.mkdir() - monkeypatch.setattr("sys.stdin", _FakeStdin(json.dumps({ - "conversation": [{"role": "user", "content": "x"}], - }))) + monkeypatch.setattr( + "sys.stdin", + _FakeStdin( + json.dumps( + { + "conversation": [{"role": "user", "content": "x"}], + } + ) + ), + ) rc = _ide_hook.main(["--repo-path", str(repo)]) assert rc == 0 assert not _raw_files(repo) @@ -454,13 +573,19 @@ def get(self, sid): return None import json as _json - result = _json.loads(_cap.handle_capture_conversation({ - "output_dir": str(out), - "conversation": [ - {"role": "user", "content": "review 最近一次提交"}, - {"role": "assistant", "content": "好的，我来审查"}, - ], - }, _Store())) + + result = _json.loads( + _cap.handle_capture_conversation( + { + "output_dir": str(out), + "conversation": [ + {"role": "user", "content": "review 最近一次提交"}, + {"role": "assistant", "content": "好的，我来审查"}, + ], + }, + _Store(), + ) + ) assert result["status"] == "captured" # conversation_id is the filename stem without the conv- prefix assert result["conversation_id"].startswith("conv-") @@ -482,12 +607,18 @@ def get(self, sid): return None import json as _json - result = _json.loads(_cap.handle_capture_conversation({ - "output_dir": str(out), - "conversation": [ - {"role": "assistant", "content": "only assistant text"}, - ], - }, _Store())) + + result = _json.loads( + _cap.handle_capture_conversation( + { + "output_dir": str(out), + "conversation": [ + {"role": "assistant", "content": "only assistant text"}, + ], + }, + _Store(), + ) + ) assert result["status"] == "captured" # Falls back to a timestamp-like stem with no CJK slug prefix issues assert result["conversation_id"].startswith("conv-") @@ -507,12 +638,14 @@ def get(self, sid): return None import json as _json - conv = [{"role": "user", "content": "重复的开场白"}, - {"role": "assistant", "content": "回答"}] - r1 = _json.loads(_cap.handle_capture_conversation( - {"output_dir": str(out), "conversation": conv}, _Store())) - r2 = _json.loads(_cap.handle_capture_conversation( - {"output_dir": str(out), "conversation": conv}, _Store())) + + conv = [{"role": "user", "content": "重复的开场白"}, {"role": "assistant", "content": "回答"}] + _json.loads( + _cap.handle_capture_conversation({"output_dir": str(out), "conversation": conv}, _Store()) + ) + _json.loads( + _cap.handle_capture_conversation({"output_dir": str(out), "conversation": conv}, _Store()) + ) # Second capture supersedes the first (same source_session empty) — but here # neither has source_session_id, so they are both written. Ensure distinct. files = list(raw.glob("conv-*.md")) diff --git a/tests/test_index_freshness.py b/tests/test_index_freshness.py index 5125a29..7f16fb2 100644 --- a/tests/test_index_freshness.py +++ b/tests/test_index_freshness.py @@ -12,6 +12,7 @@ _resolve_db_path resolves relative cache_db against repo root; legacy absolute entries still honoured; missing absolute falls back to layout """ + from __future__ import annotations import json @@ -56,8 +57,9 @@ def test_new_note_found_after_pull(self, tmp_path): encoding="utf-8", ) res = search(od, "端口冲突", session=None) - assert any(r["file"] == "notes/fresh-note.md" for r in res), \ + assert any(r["file"] == "notes/fresh-note.md" for r in res), ( "pulled note must be findable after self-heal rebuild" + ) def test_deleted_note_gone_after_heal(self, tmp_path): od = _mk_wiki(tmp_path) @@ -68,7 +70,7 @@ def test_deleted_note_gone_after_heal(self, tmp_path): ) _reset_throttle() search(od, "临时经验", session=None) # indexed - note.unlink() # (simulated) removed upstream + note.unlink() # (simulated) removed upstream _reset_throttle() res = search(od, "临时经验", session=None) assert not any(r["file"] == "notes/temp-note.md" for r in res) @@ -92,8 +94,9 @@ def test_content_only_change_rebuilds(self, tmp_path): ) _reset_throttle() res = search(od, "zonecheck", session=None) - assert any("confirmable" in r["file"] for r in res), \ + assert any("confirmable" in r["file"] for r in res), ( "content-only change must trigger rebuild via mtime sampling" + ) def test_fresh_index_no_rebuild(self, tmp_path): od = _mk_wiki(tmp_path) @@ -108,9 +111,11 @@ def test_throttle_single_scan_per_minute(self, tmp_path): _reset_throttle() calls = [] orig = fr.scan_disk_inventory + def counting(od_): calls.append(1) return orig(od_) + fr.scan_disk_inventory = counting try: fr.ensure_fresh(od) @@ -130,12 +135,12 @@ def test_handle_query_wiki_end_to_end_pull(self, tmp_path): od = _mk_wiki(tmp_path) _reset_throttle() (od / "wiki" / "modules" / "newmod.md").write_text( - "---\ntype: Module\ntitle: 新模块文档\n---\n\n" - "新模块处理供应链供应链追溯。\n", + "---\ntype: Module\ntitle: 新模块文档\n---\n\n新模块处理供应链供应链追溯。\n", encoding="utf-8", ) - out = json.loads(handle_query_wiki( - {"output_dir": str(od), "query": "供应链追溯"}, SessionStore())) + out = json.loads( + handle_query_wiki({"output_dir": str(od), "query": "供应链追溯"}, SessionStore()) + ) assert any("newmod" in r["file"] for r in out["results"]) @@ -146,12 +151,19 @@ def test_relative_cache_db_resolves(self, tmp_path): (repo / ".codewiki").mkdir() (repo / ".codewiki" / "analysis_cache.db").write_text("x", encoding="utf-8") (repo / "repowiki" / ".meta" / "project.json").write_text( - json.dumps({"repo_name": "repo", "output_dir": "repowiki", - "cache_db": ".codewiki/analysis_cache.db"}), + json.dumps( + { + "repo_name": "repo", + "output_dir": "repowiki", + "cache_db": ".codewiki/analysis_cache.db", + } + ), encoding="utf-8", ) - assert _resolve_db_path(repo / "repowiki") == \ - (repo / ".codewiki" / "analysis_cache.db").resolve() + assert ( + _resolve_db_path(repo / "repowiki") + == (repo / ".codewiki" / "analysis_cache.db").resolve() + ) def test_legacy_absolute_still_works(self, tmp_path): repo = tmp_path / "repo" @@ -160,7 +172,8 @@ def test_legacy_absolute_still_works(self, tmp_path): db.parent.mkdir() db.write_text("x", encoding="utf-8") (repo / "repowiki" / ".meta" / "project.json").write_text( - json.dumps({"cache_db": str(db)}), encoding="utf-8") + json.dumps({"cache_db": str(db)}), encoding="utf-8" + ) assert _resolve_db_path(repo / "repowiki") == db.resolve() def test_missing_absolute_falls_back(self, tmp_path): @@ -170,14 +183,13 @@ def test_missing_absolute_falls_back(self, tmp_path): fallback = repo / ".codewiki" / "analysis_cache.db" fallback.write_text("x", encoding="utf-8") (repo / "repowiki" / ".meta" / "project.json").write_text( - json.dumps({"cache_db": "D:\\gone\\machine\\analysis_cache.db"}), - encoding="utf-8") + json.dumps({"cache_db": "D:\\gone\\machine\\analysis_cache.db"}), encoding="utf-8" + ) assert _resolve_db_path(repo / "repowiki") == fallback.resolve() def test_json_index_stores_built_at(self, tmp_path): od = _mk_wiki(tmp_path) - data = json.loads( - (od / ".meta" / "search_index.json").read_text(encoding="utf-8")) + data = json.loads((od / ".meta" / "search_index.json").read_text(encoding="utf-8")) assert float(data.get("built_at") or 0) > 0 @@ -188,8 +200,8 @@ def test_index_md_rebuilt_on_stale(self, tmp_path): _reset_throttle() idx_md = od / "wiki" / "index.md" if idx_md.exists(): - idx_md.unlink() # simulate a lost/corrupted catalog - (od / "notes" / "another.md").write_text( # trigger staleness + idx_md.unlink() # simulate a lost/corrupted catalog + (od / "notes" / "another.md").write_text( # trigger staleness "---\ntype: pitfall\ntitle: 另一踩坑\n---\n\n内容正文 bodytext\n", encoding="utf-8", ) @@ -199,9 +211,11 @@ def test_index_md_rebuilt_on_stale(self, tmp_path): def test_gitignore_excludes_search_index(self): """T3: search_index.json stays out of version control.""" import subprocess + out = subprocess.run( ["git", "check-ignore", "repowiki/.meta/search_index.json"], cwd=str(Path(__file__).resolve().parents[1]), - capture_output=True, text=True, + capture_output=True, + text=True, ) assert out.returncode == 0, "search_index.json must be gitignored" diff --git a/tests/test_l0_archive.py b/tests/test_l0_archive.py index cc7a047..b26fb39 100644 --- a/tests/test_l0_archive.py +++ b/tests/test_l0_archive.py @@ -9,6 +9,7 @@ - archived conversations are NOT indexed (link-only discovery); - query_wiki note results expose source_ref for on-demand tracing. """ + import json from pathlib import Path @@ -20,19 +21,18 @@ # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello", - extra_fm: str = "") -> str: +def _write_raw( + repo: str, cid: str, body: str = "user: hi\nassistant: hello", extra_fm: str = "" +) -> str: raw_dir = Path(repo) / "repowiki" / RAW_DIR raw_dir.mkdir(parents=True, exist_ok=True) p = raw_dir / f"conv-{cid}.md" p.write_text( "---\n" "type: conversation\n" - f"conversation_id: \"{cid}\"\n" + f'conversation_id: "{cid}"\n' "status: pending\n" - "origin: conversation\n" - + extra_fm + - "---\n\n" + body, + "origin: conversation\n" + extra_fm + "---\n\n" + body, encoding="utf-8", ) return str(p) @@ -40,12 +40,15 @@ def _write_raw(repo: str, cid: str, body: str = "user: hi\nassistant: hello", def _submit(repo: str, distilled: dict, **extra_args): store = SessionStore() - out = distill.handle_distill_conversation({ - "output_dir": f"{repo}/repowiki", - "mode": "submit", - "distilled": distilled, - **extra_args, - }, store) + out = distill.handle_distill_conversation( + { + "output_dir": f"{repo}/repowiki", + "mode": "submit", + "distilled": distilled, + **extra_args, + }, + store, + ) data = json.loads(out) by_cid = {r["conversation_id"]: r for r in data.get("distilled", [])} return data, by_cid @@ -54,6 +57,7 @@ def _submit(repo: str, distilled: dict, **extra_args): def _note_source_ref(repo: str, note_file: str) -> str: """Read metadata.source_ref from a note file (yaml-parsed).""" import yaml + text = (Path(repo) / "repowiki" / "notes" / note_file).read_text(encoding="utf-8") end = text.find("---", 3) fm = yaml.safe_load(text[3:end]) @@ -71,9 +75,21 @@ def test_drop_raw_argument_deletes(tmp_path): repo = str(tmp_path) cid = "privacy-arg" raw_path = _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "privacy note", "note_type": "general", "content": "x", - }]}}, drop_raw=True) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "privacy note", + "note_type": "general", + "content": "x", + } + ] + } + }, + drop_raw=True, + ) r = by_cid[f"conv-{cid}"] assert r["deleted_raw"] is True assert r["archived_raw"] is None @@ -85,9 +101,20 @@ def test_drop_raw_frontmatter_deletes(tmp_path): repo = str(tmp_path) cid = "privacy-fm" raw_path = _write_raw(repo, cid, extra_fm="drop_raw: true\n") - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "privacy fm note", "note_type": "general", "content": "x", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "privacy fm note", + "note_type": "general", + "content": "x", + } + ] + } + }, + ) r = by_cid[f"conv-{cid}"] assert r["deleted_raw"] is True assert r["archived_raw"] is None @@ -101,10 +128,20 @@ def test_source_ref_repointed_to_archive(tmp_path): repo = str(tmp_path) cid = "repoint-001" _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "traceable note", "note_type": "decision", - "content": "## Background\ntrace me", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "traceable note", + "note_type": "decision", + "content": "## Background\ntrace me", + } + ] + } + }, + ) r = by_cid[f"conv-{cid}"] assert r["archived_raw"] == f"conversations/conv-{cid}.md" note_file = _notes(repo)[0] @@ -120,23 +157,40 @@ def test_source_ref_repointed_across_conflict_rounds(tmp_path): # Pre-existing note creating a weak title-similarity band for one draft store = SessionStore() from codewiki.mcp.tools.knowledge_loop import handle_ingest_note, handle_confirm_note - r0 = json.loads(handle_ingest_note({ - "output_dir": f"{repo}/repowiki", - "title": "alpha beta gamma delta epsilon", - "note_type": "pitfall", "content": "existing", "status": "draft", - }, store)) - handle_confirm_note({ - "output_dir": f"{repo}/repowiki", - "note_file": Path(r0["note_path"]).name, - }, store) + + r0 = json.loads( + handle_ingest_note( + { + "output_dir": f"{repo}/repowiki", + "title": "alpha beta gamma delta epsilon", + "note_type": "pitfall", + "content": "existing", + "status": "draft", + }, + store, + ) + ) + handle_confirm_note( + { + "output_dir": f"{repo}/repowiki", + "note_file": Path(r0["note_path"]).name, + }, + store, + ) _write_raw(repo, cid) notes_json = { "notes": [ - {"title": "alpha beta gamma zeta eta", "note_type": "lesson", - "content": "weak duplicate candidate"}, - {"title": "completely unrelated knowledge", "note_type": "decision", - "content": "fresh knowledge"}, + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "weak duplicate candidate", + }, + { + "title": "completely unrelated knowledge", + "note_type": "decision", + "content": "fresh knowledge", + }, ] } # Round 1 @@ -148,8 +202,12 @@ def test_source_ref_repointed_across_conflict_rounds(tmp_path): # Round 2: resolve the conflict with store resolve_json = { "notes": [ - {"title": "alpha beta gamma zeta eta", "note_type": "lesson", - "content": "weak duplicate candidate", "dedup_action": "store"}, + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "weak duplicate candidate", + "dedup_action": "store", + }, ] } _data, by_cid = _submit(repo, {cid: resolve_json}) @@ -158,8 +216,7 @@ def test_source_ref_repointed_across_conflict_rounds(tmp_path): assert r2["archived_raw"] == f"conversations/conv-{cid}.md" # Both distilled notes (round-1 fresh + round-2 stored) repointed - refs = {_note_source_ref(repo, n) for n in _notes(repo) - if n != Path(r0["note_path"]).name} + refs = {_note_source_ref(repo, n) for n in _notes(repo) if n != Path(r0["note_path"]).name} assert refs == {f"conversations/conv-{cid}.md"}, refs @@ -171,13 +228,24 @@ def test_archived_conversations_not_searchable(tmp_path): cid = "noindex-001" marker = "独特标记词鲲鹏协议" _write_raw(repo, cid, body=f"user: 讨论{marker}\nassistant: 好的") - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "noindex note", "note_type": "general", - "content": f"## Background\n提到{marker}", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "noindex note", + "note_type": "general", + "content": f"## Background\n提到{marker}", + } + ] + } + }, + ) assert by_cid[f"conv-{cid}"]["archived_raw"] from codewiki.mcp.tools.wiki_search import build_full_index, search + od = Path(repo) / "repowiki" build_full_index(od) hits = search(od, marker, max_results=10, score_threshold=0.0) @@ -191,20 +259,35 @@ def test_query_wiki_surfaces_source_ref(tmp_path): repo = str(tmp_path) cid = "surface-ref" _write_raw(repo, cid, body="user: 讨论缓存击穿\nassistant: 结论") - _data, by_cid = _submit(repo, {cid: {"notes": [{ - "title": "缓存击穿防护决策", "note_type": "decision", - "content": "## Background\n缓存击穿需要互斥锁", - }]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "缓存击穿防护决策", + "note_type": "decision", + "content": "## Background\n缓存击穿需要互斥锁", + } + ] + } + }, + ) assert by_cid[f"conv-{cid}"]["archived_raw"] from codewiki.mcp.tools.knowledge_loop import handle_query_wiki + store = SessionStore() - resp = json.loads(handle_query_wiki({ - "output_dir": f"{repo}/repowiki", - "query": "缓存击穿", - }, store)) - note_hits = [r for r in resp.get("results", []) - if r.get("file", "").startswith("notes/")] + resp = json.loads( + handle_query_wiki( + { + "output_dir": f"{repo}/repowiki", + "query": "缓存击穿", + }, + store, + ) + ) + note_hits = [r for r in resp.get("results", []) if r.get("file", "").startswith("notes/")] assert note_hits, resp assert note_hits[0].get("source_ref") == f"conversations/conv-{cid}.md" @@ -214,16 +297,32 @@ def test_conflict_pending_keeps_raw_unarchived(tmp_path): cid = "pending-noarchive" store = SessionStore() from codewiki.mcp.tools.knowledge_loop import handle_ingest_note - handle_ingest_note({ - "output_dir": f"{repo}/repowiki", - "title": "alpha beta gamma delta epsilon", - "note_type": "pitfall", "content": "existing", "status": "draft", - }, store) + + handle_ingest_note( + { + "output_dir": f"{repo}/repowiki", + "title": "alpha beta gamma delta epsilon", + "note_type": "pitfall", + "content": "existing", + "status": "draft", + }, + store, + ) _write_raw(repo, cid) - _data, by_cid = _submit(repo, {cid: {"notes": [ - {"title": "alpha beta gamma zeta eta", "note_type": "lesson", - "content": "conflict"}, - ]}}) + _data, by_cid = _submit( + repo, + { + cid: { + "notes": [ + { + "title": "alpha beta gamma zeta eta", + "note_type": "lesson", + "content": "conflict", + }, + ] + } + }, + ) r = by_cid[f"conv-{cid}"] assert r["status"] == "conflicts_pending" assert r["archived_raw"] is None diff --git a/tests/test_latest_compat.py b/tests/test_latest_compat.py index 73d148b..b89133a 100644 --- a/tests/test_latest_compat.py +++ b/tests/test_latest_compat.py @@ -57,6 +57,7 @@ def test_core_modules_import() -> None: import codewiki.src.be.agent_tools.generate_sub_module_documentations import codewiki.src.be.agent_tools.read_code_components import codewiki.src.be.agent_tools.str_replace_editor + try: import codewiki.src.be.caw_toolkit # noqa: F401 except ImportError as exc: diff --git a/tests/test_lint_fix.py b/tests/test_lint_fix.py index 69bc541..aef9cc7 100644 --- a/tests/test_lint_fix.py +++ b/tests/test_lint_fix.py @@ -116,20 +116,12 @@ def test_fix_clears_broken_links_computed_on_old_index(tmp_path): """ output_dir = _make_wiki(tmp_path) - result_before = _run_lint( - output_dir, checks=["stale_refs", "broken_links"] - ) + result_before = _run_lint(output_dir, checks=["stale_refs", "broken_links"]) assert any( - "2026-08-01-deleted-note" in str(i.get("message", "")) - for i in result_before["issues"] + "2026-08-01-deleted-note" in str(i.get("message", "")) for i in result_before["issues"] ) - result = _run_lint( - output_dir, fix=True, checks=["stale_refs", "broken_links"] - ) + result = _run_lint(output_dir, fix=True, checks=["stale_refs", "broken_links"]) assert [i for i in result["issues"] if i["check"] == "stale_refs"] == [] assert [i for i in result["issues"] if i["check"] == "broken_links"] == [] - assert not any( - "2026-08-01-deleted-note" in str(i.get("message", "")) - for i in result["issues"] - ) + assert not any("2026-08-01-deleted-note" in str(i.get("message", "")) for i in result["issues"]) diff --git a/tests/test_low_adoption.py b/tests/test_low_adoption.py index d0cca6b..11bb14b 100644 --- a/tests/test_low_adoption.py +++ b/tests/test_low_adoption.py @@ -12,6 +12,7 @@ - draft notes are out of scope - full dispatch (checks=['all']) surfaces low_adoption in the output """ + from __future__ import annotations import importlib @@ -35,19 +36,15 @@ def _mk_wiki(tmp_path, low_adoption=None) -> Path: conv: dict = {} if low_adoption is not None: conv["usage_ranking"] = {"low_adoption": low_adoption} - (od / "schema.yaml").write_text( - yaml.safe_dump({"conventions": conv}), encoding="utf-8" - ) + (od / "schema.yaml").write_text(yaml.safe_dump({"conventions": conv}), encoding="utf-8") return od -def _write_note(od: Path, name: str, *, status="stable", title="T", - ntype="pitfall") -> Path: +def _write_note(od: Path, name: str, *, status="stable", title="T", ntype="pitfall") -> Path: fm = {"type": ntype, "title": title, "status": status} p = od / "notes" / name p.write_text( - "---\n" + yaml.safe_dump(fm, allow_unicode=True, sort_keys=False) - + "---\n\nbody\n", + "---\n" + yaml.safe_dump(fm, allow_unicode=True, sort_keys=False) + "---\n\nbody\n", encoding="utf-8", ) return p @@ -58,32 +55,42 @@ def _append_events(od: Path, events: list) -> None: each other's writes — write_telemetry rewrites the whole file).""" from tests.telemetry_seed import write_telemetry from codewiki.mcp.tools import telemetry as _tel + existing = [] p = Path(od) / ".meta" / _tel.TELEMETRY_DIRNAME / "tester.jsonl" if p.exists(): import json as _json + existing = [ - _json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() - if line.strip() + _json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() if line.strip() ] write_telemetry(od, "tester", existing + events) def _touch(od: Path, rel_path: str, hit_count: int, last_hit: str) -> None: """Seed a hit event (T2: telemetry jsonl replaces the stats table).""" - _append_events(od, [ - {"t": "hit", "doc": rel_path, "at": last_hit, "n": hit_count}, - ]) + _append_events( + od, + [ + {"t": "hit", "doc": rel_path, "at": last_hit, "n": hit_count}, + ], + ) def _adopt(od: Path, doc_path: str, count: int = 1) -> None: """Record *count* adoption events (distinct capture keys; T2 jsonl).""" - _append_events(od, [ - {"t": "adopted", "doc": doc_path, - "at": datetime.now().isoformat(timespec="seconds"), - "key": f"tester/sess-{i}"} - for i in range(count) - ]) + _append_events( + od, + [ + { + "t": "adopted", + "doc": doc_path, + "at": datetime.now().isoformat(timespec="seconds"), + "key": f"tester/sess-{i}", + } + for i in range(count) + ], + ) def _adopted(issues) -> list: @@ -163,6 +170,7 @@ def test_empty_adoption_table_skips(tmp_path): _touch(od, "notes/hot.md", hit_count=20, last_hit=RECENT) # an empty telemetry file exists (bundle-wide zero adoption) from tests.telemetry_seed import write_telemetry + write_telemetry(od, "tester", []) assert _check_low_adoption(od) == [] @@ -219,8 +227,7 @@ def test_dispatch_all_surfaces_low_adoption(tmp_path): _adopt(od, "notes/other.md") store = SessionStore() - resp = json.loads(handle_lint_wiki( - {"output_dir": str(od), "checks": ["all"]}, store)) + resp = json.loads(handle_lint_wiki({"output_dir": str(od), "checks": ["all"]}, store)) issues = _adopted(resp.get("issues", [])) assert len(issues) == 1 assert issues[0]["severity"] == "warning" diff --git a/tests/test_module_tree_validation.py b/tests/test_module_tree_validation.py index f829a5e..bd27ad6 100644 --- a/tests/test_module_tree_validation.py +++ b/tests/test_module_tree_validation.py @@ -178,14 +178,16 @@ def test_multiple_unmatched_and_leftover(tmp_path): def test_standalone_no_session_skips_validation(tmp_path): """Without an analysis session there is no id index; save must still work.""" store = SessionStore() - result = json.loads(handle_save_module_tree( - { - "repo_path": str(tmp_path), - "output_dir": str(tmp_path), - "module_tree": {"core": {"components": ["src/a.py::A"]}}, - }, - store, - )) + result = json.loads( + handle_save_module_tree( + { + "repo_path": str(tmp_path), + "output_dir": str(tmp_path), + "module_tree": {"core": {"components": ["src/a.py::A"]}}, + }, + store, + ) + ) assert result["status"] == "saved" assert "validation" not in result assert "warning" not in result diff --git a/tests/test_ontology_graph.py b/tests/test_ontology_graph.py index 99cfd63..e635f3b 100644 --- a/tests/test_ontology_graph.py +++ b/tests/test_ontology_graph.py @@ -7,6 +7,7 @@ repo. These tests are skipped until the feature lands; once the modules exist they will run automatically. """ + from __future__ import annotations import textwrap @@ -23,6 +24,7 @@ # --- fixtures -------------------------------------------------------------- + def _write_page(d: Path, rel: str, body: str) -> None: p = d / rel p.parent.mkdir(parents=True, exist_ok=True) @@ -36,7 +38,10 @@ def wiki_output(tmp_path: Path) -> Path: od.mkdir() (od / "wiki").mkdir() # OrderService entity with a relations table - _write_page(od, "wiki/entities/OrderService.md", textwrap.dedent(""" + _write_page( + od, + "wiki/entities/OrderService.md", + textwrap.dedent(""" --- title: OrderService page_type: entity @@ -51,9 +56,13 @@ def wiki_output(tmp_path: Path) -> Path: | calls | PaymentGateway | 调用支付 | Body text with [[Account]] wikilink. - """).lstrip()) + """).lstrip(), + ) # Account entity (no relations section) - _write_page(od, "wiki/entities/Account.md", textwrap.dedent(""" + _write_page( + od, + "wiki/entities/Account.md", + textwrap.dedent(""" --- title: Account page_type: entity @@ -61,21 +70,27 @@ def wiki_output(tmp_path: Path) -> Path: --- # Account Account body. - """).lstrip()) + """).lstrip(), + ) # A relation target that does NOT exist yet -> placeholder - _write_page(od, "wiki/entities/PaymentGateway.md", textwrap.dedent(""" + _write_page( + od, + "wiki/entities/PaymentGateway.md", + textwrap.dedent(""" --- title: PaymentGateway page_type: entity --- # PaymentGateway Exists. - """).lstrip()) + """).lstrip(), + ) return od # --- P0.1 extraction ------------------------------------------------------- + def test_extract_types_and_relations(wiki_output: Path): all_docs = { "wiki/entities/OrderService.md": "OrderService", @@ -107,19 +122,23 @@ def test_extract_and_write_merges_idempotently(wiki_output: Path): } # Pre-seed ontology.yaml with a hand-authored relation (must be preserved) onto = wiki_output / "ontology.yaml" - onto.write_text(textwrap.dedent(""" + onto.write_text( + textwrap.dedent(""" types: [] relations: - from: ManualService relation: depends_on to: Account note: hand-authored - """).lstrip(), encoding="utf-8") + """).lstrip(), + encoding="utf-8", + ) info = extract_and_write(wiki_output, all_docs, ontology_path=onto, write_stubs=True) assert info["relations"] >= 4 # 3 extracted + 1 hand # hand-authored preserved import yaml + data = yaml.safe_load(onto.read_text(encoding="utf-8")) hand = [r for r in data["relations"] if r.get("from") == "ManualService"] assert hand and hand[0].get("note") == "hand-authored" @@ -130,8 +149,10 @@ def test_extract_and_write_merges_idempotently(wiki_output: Path): # --- P0.2 consumption via graph_expand ------------------------------------ + def test_graph_expand_consumes_ontology_relations(wiki_output: Path): from codewiki.mcp.cache import AnalysisCache + all_docs = { "wiki/entities/OrderService.md": "OrderService", "wiki/entities/Account.md": "Account", @@ -145,7 +166,8 @@ def test_graph_expand_consumes_ontology_relations(wiki_output: Path): cache.build_search_index(wiki_output) # seed from Account, hop 1 should reach OrderService via ontology edge expanded = cache.graph_expand( - [("wiki/entities/Account.md", 1.0)], hop=1, output_dir=wiki_output) + [("wiki/entities/Account.md", 1.0)], hop=1, output_dir=wiki_output + ) files = {e["file"] for e in expanded} # OrderService depends_on Account -> edge Account->OrderService exists assert "wiki/entities/OrderService.md" in files @@ -155,15 +177,18 @@ def test_graph_expand_consumes_ontology_relations(wiki_output: Path): # --- P1 lint --------------------------------------------------------------- + def test_lint_ontology_stale_flags_broken_relation(wiki_output: Path): from codewiki.mcp.tools.wiki_lint import _check_ontology_stale + all_docs = { "wiki/entities/OrderService.md": "OrderService", "wiki/entities/Account.md": "Account", "wiki/entities/PaymentGateway.md": "PaymentGateway", } - extract_and_write(wiki_output, all_docs, ontology_path=wiki_output / "ontology.yaml", - write_stubs=False) + extract_and_write( + wiki_output, all_docs, ontology_path=wiki_output / "ontology.yaml", write_stubs=False + ) issues = _check_ontology_stale(wiki_output) # OrderItem is referenced but has no page and no stub -> error error_msgs = [i["message"] for i in issues if i["severity"] == "error"] @@ -172,8 +197,10 @@ def test_lint_ontology_stale_flags_broken_relation(wiki_output: Path): # --- P2 view --------------------------------------------------------------- + def test_generate_ontology_view_full(wiki_output: Path): from codewiki.mcp.tools.ontology_view import generate_ontology_view + all_docs = { "wiki/entities/OrderService.md": "OrderService", "wiki/entities/Account.md": "Account", @@ -192,5 +219,6 @@ def test_generate_ontology_view_full(wiki_output: Path): def test_generate_ontology_view_impact_requires_root(wiki_output: Path): from codewiki.mcp.tools.ontology_view import generate_ontology_view + res = generate_ontology_view(output_dir=str(wiki_output), view_type="impact") assert "error" in res diff --git a/tests/test_openviking_borrowings.py b/tests/test_openviking_borrowings.py index c03c8a3..18989df 100644 --- a/tests/test_openviking_borrowings.py +++ b/tests/test_openviking_borrowings.py @@ -17,10 +17,14 @@ # ── V4: note_types ────────────────────────────────────────────────────────── + def test_v4_table_single_source(): from codewiki.mcp.tools.note_types import ( - DEFAULT_NOTE_TYPES, promotion_targets, - freshness_windows, merge_fields_for, validate_note_types, + DEFAULT_NOTE_TYPES, + promotion_targets, + freshness_windows, + merge_fields_for, + validate_note_types, ) from codewiki.mcp.tools.distill_conversation import _VALID_NOTE_TYPES import codewiki.mcp.tools.knowledge_loop as kl @@ -39,10 +43,15 @@ def test_v4_table_single_source(): def test_v4_schema_overrides(): from codewiki.mcp.tools.note_types import load_note_types, freshness_windows - schema = {"conventions": {"note_types": { - "workaround": {"freshness_days": 20}, - "custom_type": {"freshness_days": 90, "promote_to": "concept"}, - }}} + + schema = { + "conventions": { + "note_types": { + "workaround": {"freshness_days": 20}, + "custom_type": {"freshness_days": 90, "promote_to": "concept"}, + } + } + } table = load_note_types(schema) assert table["workaround"]["freshness_days"] == 20 assert table["workaround"]["promote_to"] == "query" # unlisted sub-keys inherit @@ -52,14 +61,17 @@ def test_v4_schema_overrides(): def test_v4_load_freshness_config_precedence(): import codewiki.mcp.tools.knowledge_loop as kl + # legacy schema without note_types: custom by_type preserved (no override) legacy = {"conventions": {"freshness": {"by_type": {"workaround": 30}}}} assert kl.load_freshness_config(legacy)["by_type"] == {"workaround": 30} # schema WITH note_types: derived windows win; defaults fill the rest - mod = {"conventions": { - "note_types": {"workaround": {"freshness_days": 20}}, - "freshness": {"by_type": {"workaround": 45}}, - }} + mod = { + "conventions": { + "note_types": {"workaround": {"freshness_days": 20}}, + "freshness": {"by_type": {"workaround": 45}}, + } + } cfg = kl.load_freshness_config(mod) assert cfg["by_type"]["workaround"] == 20 assert cfg["by_type"]["pitfall"] == 180 @@ -68,6 +80,7 @@ def test_v4_load_freshness_config_precedence(): def test_v4_registry_enum_from_table(): import codewiki.mcp.registry as reg from codewiki.mcp.tools.note_types import DEFAULT_NOTE_TYPES + # ingest_note 的 note_type 枚举是 note_type 域，从权威表生成 ing = reg.REGISTRY["ingest_note"] ing_blob = json.dumps(ing.schema.inputSchema, ensure_ascii=False) @@ -82,13 +95,14 @@ def test_v4_registry_enum_from_table(): # ── V3: note_merge ────────────────────────────────────────────────────────── + def _mk_note(tmp_path, name, title, body, *, date, related, note_type="pitfall"): notes = tmp_path / "notes" notes.mkdir(exist_ok=True) related_line = "related_modules: [" + ", ".join(f'"{r}"' for r in related) + "]" (notes / name).write_text( - f"---\ntitle: \"{title}\"\ntype: {note_type}\nstatus: draft\n" - f"date: {date}\n{related_line}\ntags: [\"{note_type}\", \"alpha\"]\n" + f'---\ntitle: "{title}"\ntype: {note_type}\nstatus: draft\n' + f'date: {date}\n{related_line}\ntags: ["{note_type}", "alpha"]\n' f"metadata:\n date: {date}\n---\n\n{body}\n", encoding="utf-8", ) @@ -97,10 +111,9 @@ def _mk_note(tmp_path, name, title, body, *, date, related, note_type="pitfall") def test_v3_merge_notes_field_strategies(tmp_path): from codewiki.mcp.tools.note_merge import merge_notes - a = _mk_note(tmp_path, "a.md", "旧标题", "旧结论正文", - date="2026-01-01", related=["mod_a"]) - b = _mk_note(tmp_path, "b.md", "新标题", "新结论正文", - date="2026-06-01", related=["mod_b"]) + + a = _mk_note(tmp_path, "a.md", "旧标题", "旧结论正文", date="2026-01-01", related=["mod_a"]) + b = _mk_note(tmp_path, "b.md", "新标题", "新结论正文", date="2026-06-01", related=["mod_b"]) res = merge_notes(tmp_path, [a, b]) assert "error" not in res # oldest first (chronological story), newest title/type (replace) @@ -120,17 +133,23 @@ def test_v3_merge_notes_field_strategies(tmp_path): def test_v3_merge_union_strategy_override(tmp_path): from codewiki.mcp.tools.note_merge import merge_notes + a = _mk_note(tmp_path, "a.md", "A", "正文A", date="2026-01-01", related=["m1"]) b = _mk_note(tmp_path, "b.md", "B", "正文B", date="2026-06-01", related=["m2"]) - schema = {"conventions": {"note_types": { - "pitfall": {"merge_fields": {"related_modules": "union"}}, - }}} + schema = { + "conventions": { + "note_types": { + "pitfall": {"merge_fields": {"related_modules": "union"}}, + } + } + } res = merge_notes(tmp_path, [a, b], schema) assert '"m1"' in res["content"] and '"m2"' in res["content"] def test_v3_merge_write_and_idempotence_guards(tmp_path): from codewiki.mcp.tools.note_merge import merge_notes + a = _mk_note(tmp_path, "a.md", "A", "x", date="2026-01-01", related=[]) b = _mk_note(tmp_path, "b.md", "B", "y", date="2026-06-01", related=[]) res = merge_notes(tmp_path, [a, b], write=True) @@ -143,23 +162,29 @@ def test_v3_merge_write_and_idempotence_guards(tmp_path): # ── V2: injection_budget + doc_description ───────────────────────────────── + def test_v2_budget_config_and_defaults(): from codewiki.mcp.tools.injection_budget import load_budget + assert load_budget(None)["search_result_chars"] == 1200 off = {"conventions": {"injection_budget": {"search_result_chars": 0}}} assert load_budget(off)["search_result_chars"] == 0 - custom = {"conventions": {"injection_budget": {"search_result_chars": 50, - "agents_md_module_lines": 2}}} + custom = { + "conventions": { + "injection_budget": {"search_result_chars": 50, "agents_md_module_lines": 2} + } + } cfg = load_budget(custom) assert cfg["search_result_chars"] == 50 and cfg["agents_md_module_lines"] == 2 def test_v2_snippet_degradation(tmp_path): from codewiki.mcp.tools.injection_budget import apply_snippet_budget - results = [{"file": "wiki/modules/A.md", "snippet": "x" * 400, - "relevance_score": 1.5}, - {"file": "wiki/modules/B.md", "snippet": "y" * 400, - "relevance_score": 1.2}] + + results = [ + {"file": "wiki/modules/A.md", "snippet": "x" * 400, "relevance_score": 1.5}, + {"file": "wiki/modules/B.md", "snippet": "y" * 400, "relevance_score": 1.2}, + ] # budget 500: first entry consumes it, second degrades schema = {"conventions": {"injection_budget": {"search_result_chars": 500}}} n = apply_snippet_budget(results, tmp_path, schema) @@ -172,21 +197,24 @@ def test_v2_snippet_degradation(tmp_path): r2 = [{"file": "a.md", "snippet": "z" * 900, "relevance_score": 1}] assert apply_snippet_budget(r2, tmp_path, {"conventions": {}}) == 0 r3 = [{"file": "a.md", "snippet": "z" * 900, "relevance_score": 1}] - assert apply_snippet_budget( - r3, tmp_path, {"conventions": {"injection_budget": {"search_result_chars": 0}}} - ) == 0 + assert ( + apply_snippet_budget( + r3, tmp_path, {"conventions": {"injection_budget": {"search_result_chars": 0}}} + ) + == 0 + ) def test_v2_degraded_line_uses_description(tmp_path): from codewiki.mcp.tools.injection_budget import apply_snippet_budget + doc = tmp_path / "wiki/modules/A.md" doc.parent.mkdir(parents=True) doc.write_text( '---\ntitle: A\ndescription: "认证模块，负责 JWT 签发与校验"\n---\n\n正文\n', encoding="utf-8", ) - results = [{"file": "wiki/modules/A.md", "snippet": "s" * 2000, - "relevance_score": 2.0}] + results = [{"file": "wiki/modules/A.md", "snippet": "s" * 2000, "relevance_score": 2.0}] schema = {"conventions": {"injection_budget": {"search_result_chars": 100}}} apply_snippet_budget(results, tmp_path, schema) assert "认证模块" in results[0]["snippet"] @@ -194,6 +222,7 @@ def test_v2_degraded_line_uses_description(tmp_path): def test_v2_cap_module_lines(tmp_path): from codewiki.mcp.tools.injection_budget import cap_module_lines + mods = [f"Mod{i}" for i in range(10)] schema = {"conventions": {"injection_budget": {"agents_md_module_lines": 3}}} res = cap_module_lines(mods, tmp_path, schema) @@ -205,6 +234,7 @@ def test_v2_cap_module_lines(tmp_path): def test_v2_doc_description_extract_and_backfill(tmp_path): from codewiki.mcp.tools.doc_description import extract_lede, ensure_description, backfill_dir + body = "\n\n这是第一句话。这是第二句话！这是第三句会被截掉。\n\n## 架构\n\n后文。\n" lede = extract_lede(body) assert lede.startswith("这是第一句话") and "第三句" not in lede @@ -213,8 +243,7 @@ def test_v2_doc_description_extract_and_backfill(tmp_path): doc = tmp_path / "wiki/modules/A.md" doc.parent.mkdir(parents=True) - doc.write_text("---\ntitle: A\n---\n\n首段摘要句一。句二。\n\n## X\n\nb\n", - encoding="utf-8") + doc.write_text("---\ntitle: A\n---\n\n首段摘要句一。句二。\n\n## X\n\nb\n", encoding="utf-8") assert ensure_description(doc) is True text = doc.read_text(encoding="utf-8") assert 'description: "首段摘要句一。句二。"' in text @@ -230,29 +259,31 @@ def test_v2_doc_description_extract_and_backfill(tmp_path): def test_v2_crlf_preserved_on_backfill(tmp_path): from codewiki.mcp.tools.doc_description import ensure_description + doc = tmp_path / "crlf.md" raw = b"---\r\ntitle: T\r\n---\r\n\r\nCRLF first para.\r\n\r\n## H\r\n" doc.write_bytes(raw) assert ensure_description(doc) is True out = doc.read_bytes() - assert b"description: \"CRLF first para.\"" in out + assert b'description: "CRLF first para."' in out assert out.count(b"\r\n") >= 4 # original CRLFs intact (byte-level rewrite) # ── V7': doc_update_notify ────────────────────────────────────────────────── + def test_v7_affected_notes_and_payload(tmp_path): from codewiki.mcp.tools.doc_update_notify import affected_notes, reminder_payload + notes = tmp_path / "notes" notes.mkdir(parents=True) (notes / "n1.md").write_text( "---\ntitle: N1\ntype: pitfall\nstatus: stable\n" - "related_modules: [\"AnalysisPipeline\", \"CLI\"]\n---\n\nbody\n", + 'related_modules: ["AnalysisPipeline", "CLI"]\n---\n\nbody\n', encoding="utf-8", ) (notes / "n2.md").write_text( - "---\ntitle: N2\ntype: lesson\nstatus: draft\n" - "related_modules: [\"cache\"]\n---\n\nbody\n", + '---\ntitle: N2\ntype: lesson\nstatus: draft\nrelated_modules: ["cache"]\n---\n\nbody\n', encoding="utf-8", ) hits = affected_notes(tmp_path, ["analysis_pipeline"]) # norm: _/-/case @@ -266,9 +297,11 @@ def test_v7_affected_notes_and_payload(tmp_path): # ── V6: distill merge strategies + union helper ───────────────────────────── + def test_v6_union_fm_list(): from codewiki.mcp.tools.distill_conversation import _union_fm_list - head = "---\ntitle: T\nrelated_modules: [\"a\", \"b\"]\ntags: [\"x\"]\n---\n" + + head = '---\ntitle: T\nrelated_modules: ["a", "b"]\ntags: ["x"]\n---\n' out = _union_fm_list(head, "related_modules", from_text="see `c` and `a`") assert '"a", "b", "c"' in out # no extras → unchanged (same object value) @@ -279,27 +312,33 @@ def test_v6_union_fm_list(): def test_v6_merge_action_applies_field_strategies(tmp_path): from codewiki.mcp.tools.distill_conversation import _apply_dedup_action + note = tmp_path / "notes" / "target.md" note.parent.mkdir(parents=True) note.write_text( "---\ntitle: 目标笔记\ntype: pitfall\nstatus: draft\n" - "related_modules: [\"m1\"]\n---\n\n原有正文。\n", + 'related_modules: ["m1"]\n---\n\n原有正文。\n', encoding="utf-8", ) res = _apply_dedup_action( - "merge", "notes/target.md", "新知识标题", - "补充内容，涉及模块 `m2`。", "", tmp_path, + "merge", + "notes/target.md", + "新知识标题", + "补充内容，涉及模块 `m2`。", + "", + tmp_path, ) assert res["status"] == "merged" text = note.read_text(encoding="utf-8") assert "## 新知识标题" in text - assert "合并自蒸馏候选：新知识标题" in text # provenance marker (V6) - assert '"m1", "m2"' in text or '"m1","m2"' in text # related union (V6) - assert "原有正文。" in text # append keeps old body + assert "合并自蒸馏候选：新知识标题" in text # provenance marker (V6) + assert '"m1", "m2"' in text or '"m1","m2"' in text # related union (V6) + assert "原有正文。" in text # append keeps old body def test_v6_update_action_unchanged(tmp_path): from codewiki.mcp.tools.distill_conversation import _apply_dedup_action + note = tmp_path / "notes" / "t2.md" note.parent.mkdir(parents=True) note.write_text( diff --git a/tests/test_promotion.py b/tests/test_promotion.py index 417dacc..0790a6d 100644 --- a/tests/test_promotion.py +++ b/tests/test_promotion.py @@ -9,6 +9,7 @@ - suggested page_type routing (pitfall → query, lesson → concept) - promote-note workflow prompt: handler content + registration """ + from __future__ import annotations import asyncio @@ -54,7 +55,7 @@ def _write_note( "---", f"type: {note_type}", f"title: {name.removesuffix('.md')}", - "tags: [\"test\"]", + 'tags: ["test"]', "metadata:", ] if date: @@ -67,9 +68,7 @@ def _write_note( fm.append(" - by: codewiki/1.0") fm.append(f" at: {verified_at}") fm.append("---") - (od / "notes" / name).write_text( - "\n".join(fm) + "\n\nbody content\n", encoding="utf-8" - ) + (od / "notes" / name).write_text("\n".join(fm) + "\n\nbody content\n", encoding="utf-8") def _seed_adoption(od: Path, doc_path: str, count: int) -> None: @@ -81,6 +80,7 @@ def _seed_adoption(od: Path, doc_path: str, count: int) -> None: def _seed_stats_table(od: Path, doc_path: str) -> None: """Seed hit events so wiki_stats has usage rows to report (T2 jsonl).""" from tests.telemetry_seed import seed_hits + seed_hits(od, {doc_path: (5, _days_ago(1))}) @@ -117,7 +117,9 @@ def test_draft_status_not_candidate(self, tmp_path): def test_promoted_to_marker_excludes(self, tmp_path): od = _mk_wiki(tmp_path) _write_note( - od, "note-a.md", date=_days_ago(15), + od, + "note-a.md", + date=_days_ago(15), promoted_to="wiki/queries/note-a.md", ) _seed_adoption(od, "notes/note-a.md", 5) @@ -154,9 +156,9 @@ class TestAgeFallback: def test_verified_at_used_when_no_date(self, tmp_path): od = _mk_wiki(tmp_path) _write_note( - od, "note-a.md", - verified_at=(datetime.now() - timedelta(days=20)).strftime( - "%Y-%m-%dT%H:%M:%SZ"), + od, + "note-a.md", + verified_at=(datetime.now() - timedelta(days=20)).strftime("%Y-%m-%dT%H:%M:%SZ"), ) _seed_adoption(od, "notes/note-a.md", 3) cands = _promotion_candidates(od) @@ -165,10 +167,10 @@ def test_verified_at_used_when_no_date(self, tmp_path): def test_date_preferred_over_verified(self, tmp_path): od = _mk_wiki(tmp_path) _write_note( - od, "note-a.md", + od, + "note-a.md", date=_days_ago(30), - verified_at=(datetime.now() - timedelta(days=1)).strftime( - "%Y-%m-%dT%H:%M:%SZ"), + verified_at=(datetime.now() - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), ) _seed_adoption(od, "notes/note-a.md", 3) cands = _promotion_candidates(od) @@ -211,10 +213,10 @@ def test_general_leaves_blank(self, tmp_path): class TestSchemaOverride: def test_min_adopted_override(self, tmp_path): import yaml + od = _mk_wiki(tmp_path) (od / "schema.yaml").write_text( - yaml.safe_dump({"conventions": {"promotion": { - "min_adopted": 1, "min_age_days": 14}}}), + yaml.safe_dump({"conventions": {"promotion": {"min_adopted": 1, "min_age_days": 14}}}), encoding="utf-8", ) _write_note(od, "note-a.md", date=_days_ago(15)) @@ -224,10 +226,10 @@ def test_min_adopted_override(self, tmp_path): def test_min_age_days_override(self, tmp_path): import yaml + od = _mk_wiki(tmp_path) (od / "schema.yaml").write_text( - yaml.safe_dump({"conventions": {"promotion": { - "min_adopted": 3, "min_age_days": 30}}}), + yaml.safe_dump({"conventions": {"promotion": {"min_adopted": 3, "min_age_days": 30}}}), encoding="utf-8", ) _write_note(od, "note-a.md", date=_days_ago(20)) @@ -237,6 +239,7 @@ def test_min_age_days_override(self, tmp_path): def test_config_from_repo_schema_files(self): # The shipped schema templates must carry the promotion thresholds. import yaml + root = Path(__file__).resolve().parent.parent for rel in ("schema.yaml", "codewiki/templates/schema.yaml"): data = yaml.safe_load((root / rel).read_text(encoding="utf-8")) @@ -254,9 +257,7 @@ def test_wiki_stats_surfaces_promotion_candidates(self, tmp_path): _write_note(od, "note-a.md", date=_days_ago(15)) _seed_stats_table(od, "notes/note-a.md") _seed_adoption(od, "notes/note-a.md", 3) - out = json.loads( - handle_wiki_stats({"output_dir": str(od)}, SessionStore()) - ) + out = json.loads(handle_wiki_stats({"output_dir": str(od)}, SessionStore())) assert "promotion_candidates" in out cands = out["promotion_candidates"] assert len(cands) == 1 @@ -269,9 +270,7 @@ def test_wiki_stats_omits_when_no_candidates(self, tmp_path): _write_note(od, "note-a.md", date=_days_ago(15)) _seed_stats_table(od, "notes/note-a.md") _seed_adoption(od, "notes/note-a.md", 1) - out = json.loads( - handle_wiki_stats({"output_dir": str(od)}, SessionStore()) - ) + out = json.loads(handle_wiki_stats({"output_dir": str(od)}, SessionStore())) assert "promotion_candidates" not in out def test_ranked_by_adopted_count(self, tmp_path): @@ -282,7 +281,8 @@ def test_ranked_by_adopted_count(self, tmp_path): _seed_adoption(od, "notes/note-high.md", 7) cands = _promotion_candidates(od) assert [c["file"] for c in cands] == [ - "notes/note-high.md", "notes/note-low.md", + "notes/note-high.md", + "notes/note-low.md", ] @@ -296,12 +296,14 @@ def list_prompts(self): def deco(fn): self._list = fn return fn + return deco def get_prompt(self): def deco(fn): self._get = fn return fn + return deco @@ -332,10 +334,12 @@ def test_handler_contains_key_sections(self): assert "不删除" in text def test_handler_interpolates_arguments(self): - text = _prompt_promote_note({ - "note_file": "notes/2026-08-01-port-conflict.md", - "output_dir": "D:/repo/repowiki", - }) + text = _prompt_promote_note( + { + "note_file": "notes/2026-08-01-port-conflict.md", + "output_dir": "D:/repo/repowiki", + } + ) assert "notes/2026-08-01-port-conflict.md" in text assert "D:/repo/repowiki" in text diff --git a/tests/test_query_transparency.py b/tests/test_query_transparency.py index c10222e..6b70b10 100644 --- a/tests/test_query_transparency.py +++ b/tests/test_query_transparency.py @@ -9,6 +9,7 @@ - wiki_stats cold_candidates: once-hot-now-cold docs surfaced as a retrieval health signal (mirrors usage_ranking cold definition) """ + from __future__ import annotations import json @@ -55,6 +56,7 @@ def _stats_rows(od: Path) -> list: retrieval_stats read — check-mode must record NOTHING, so any hit event present means stats leaked).""" from codewiki.mcp.tools import telemetry + return [ (doc, e["hits"]) for doc, e in (telemetry.aggregate_usage(od) or {}).items() @@ -65,6 +67,7 @@ def _stats_rows(od: Path) -> list: def _seed_stats(od: Path, rows: list) -> None: """Seed telemetry hit events {rel_path: (hits, last_hit)} (T2 migration).""" from tests.telemetry_seed import seed_hits + seed_hits(od, {fp: (hits, last_hit) for fp, hits, last_hit in rows}) @@ -150,11 +153,14 @@ def test_cold_detection(self, tmp_path): od = _mk_wiki(tmp_path) old = (datetime.now() - timedelta(days=200)).strftime("%Y-%m-%d") recent = datetime.now().strftime("%Y-%m-%d") - _seed_stats(od, [ - ("wiki/modules/auth.md", 10, old), # hot then cold -> listed - ("notes/pitfall-port-conflict.md", 8, recent), # hot, still warm - ("wiki/modules/other.md", 1, old), # never hot -> skipped - ]) + _seed_stats( + od, + [ + ("wiki/modules/auth.md", 10, old), # hot then cold -> listed + ("notes/pitfall-port-conflict.md", 8, recent), # hot, still warm + ("wiki/modules/other.md", 1, old), # never hot -> skipped + ], + ) cold = _cold_candidates(od) assert cold is not None assert [c["file_path"] for c in cold] == ["wiki/modules/auth.md"] @@ -165,18 +171,18 @@ def test_wiki_stats_surfaces_cold(self, tmp_path): od = _mk_wiki(tmp_path) old = (datetime.now() - timedelta(days=220)).strftime("%Y-%m-%d") _seed_stats(od, [("wiki/modules/auth.md", 5, old)]) - out = json.loads( - handle_wiki_stats({"output_dir": str(od)}, SessionStore()) - ) + out = json.loads(handle_wiki_stats({"output_dir": str(od)}, SessionStore())) assert "cold_candidates" in out assert out["cold_candidates"][0]["file_path"] == "wiki/modules/auth.md" def test_schema_overrides_thresholds(self, tmp_path): import yaml + od = _mk_wiki(tmp_path) (od / "schema.yaml").write_text( - yaml.safe_dump({"conventions": {"usage_ranking": { - "cold_days": 30, "cold_min_hits": 2}}}), + yaml.safe_dump( + {"conventions": {"usage_ranking": {"cold_days": 30, "cold_min_hits": 2}}} + ), encoding="utf-8", ) sixty = (datetime.now() - timedelta(days=60)).strftime("%Y-%m-%d") @@ -212,10 +218,7 @@ def test_handle_query_wiki_returns_coverage(self, tmp_path): def test_matched_tokens_per_result(self, tmp_path): od = _mk_wiki(tmp_path) out = _query(od, query="端口 冲突") - entry = next( - r for r in out["results"] - if r["file"].endswith("pitfall-port-conflict.md") - ) + entry = next(r for r in out["results"] if r["file"].endswith("pitfall-port-conflict.md")) assert "端口" in entry.get("matched_tokens", []) assert "冲突" in entry.get("matched_tokens", []) diff --git a/tests/test_review_changes.py b/tests/test_review_changes.py index 27f88fd..bcb77ac 100644 --- a/tests/test_review_changes.py +++ b/tests/test_review_changes.py @@ -201,8 +201,11 @@ def test_prepare_end_to_end(): check("changed_sources annotated", bool(pkg["target"].get("changed_sources"))) # focus restrict - out_focus = json.loads(handle_review_changes( - {"repo_path": REPO_PATH, "mode": "prepare", "focus": "general"}, store)) + out_focus = json.loads( + handle_review_changes( + {"repo_path": REPO_PATH, "mode": "prepare", "focus": "general"}, store + ) + ) pkg_focus = json.loads(Path(out_focus["file"]).read_text(encoding="utf-8")) ev = pkg_focus.get("evidence", {}) check("focus=general restricts axes", set(ev.keys()) == {"general"}, detail=str(ev.keys())) @@ -246,10 +249,16 @@ def fake_query(store, session, arguments): conv = rc._collect_convention_evidence(None, fake_session) check("convention hits non-empty", bool(conv["hits"]), str(conv["hits"])[:200]) if conv["hits"]: - check("convention path from file key", - conv["hits"][0]["path"] == "notes/note-1.md", str(conv["hits"][0])) - check("convention score from relevance_score", - conv["hits"][0]["score"] == 2.5, str(conv["hits"][0])) + check( + "convention path from file key", + conv["hits"][0]["path"] == "notes/note-1.md", + str(conv["hits"][0]), + ) + check( + "convention score from relevance_score", + conv["hits"][0]["score"] == 2.5, + str(conv["hits"][0]), + ) check("doctrine extracted", conv.get("doctrine") == "DOCTRINE", str(conv)[:120]) changes = [FileChange(path="codewiki/mcp/tools/foo.py", added_lines=[1])] @@ -259,8 +268,11 @@ def fake_query(store, session, arguments): h = mod["hits"][0] check("module path from file key", h["path"] == "notes/note-1.md", str(h)) check("module type from note frontmatter", h["type"] == "pitfall", str(h)) - check("module related_modules from note frontmatter", - h["related_modules"] == ["codewiki/mcp/tools"], str(h)) + check( + "module related_modules from note frontmatter", + h["related_modules"] == ["codewiki/mcp/tools"], + str(h), + ) check("module score from relevance_score", h["score"] == 2.5, str(h)) finally: rc._query_wiki = orig diff --git a/tests/test_strip_system_injection.py b/tests/test_strip_system_injection.py index d890754..2f77e79 100644 --- a/tests/test_strip_system_injection.py +++ b/tests/test_strip_system_injection.py @@ -6,6 +6,7 @@ transcript so only the human-AI dialogue survives. is real user input and must keep its inner text (shell removed). """ + from __future__ import annotations from pathlib import Path @@ -65,8 +66,15 @@ def test_real_world_sample(): out = _strip_system_injection(raw) assert "你好" in out assert "有什么我可以帮你的吗" in out - for noise in ("", "", "", "", - "", "OS Version", "AGENTS.md"): + for noise in ( + "", + "", + "", + "", + "", + "OS Version", + "AGENTS.md", + ): assert noise not in out diff --git a/tests/test_task_manager.py b/tests/test_task_manager.py index 0ed49ec..34ad8b1 100644 --- a/tests/test_task_manager.py +++ b/tests/test_task_manager.py @@ -758,9 +758,9 @@ def test_compact_submit_rewrites_and_archives(tmp_path, monkeypatch): from pathlib import Path # File-domain compaction: the caller's own per-user file is rewritten. - text = ( - Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md" - ).read_text(encoding="utf-8") + text = (Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md").read_text( + encoding="utf-8" + ) assert text.startswith(tm._SUMMARY_HEADING) assert "早期记忆摘要" in text assert "memories-archive/alice.md，截至" in text and "共 25 条" in text @@ -808,9 +808,9 @@ def test_compact_submit_legacy_entries_get_synthetic_heading(tmp_path, monkeypat # Legacy converges into the caller's own file: kept entries land there, # and the legacy single file is REMOVED (attribution now explicit). - text = ( - Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md" - ).read_text(encoding="utf-8") + text = (Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md").read_text( + encoding="utf-8" + ) assert "旧条目44" in text and "旧条目25" in text assert not (Path(repo) / "repowiki" / "tasks" / task_id / "memories.md").exists() @@ -854,9 +854,9 @@ def test_compact_second_round_appends_archive_and_carries_summary(tmp_path, monk # Archive is append-only: round-1 originals (记忆0..24) AND round-2 (记忆25..44) present. assert "记忆0" in archive and "记忆44" in archive # Old summary replaced by the new one in the caller's own file. - text = ( - Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md" - ).read_text(encoding="utf-8") + text = (Path(repo) / "repowiki" / "tasks" / task_id / "memories" / "alice.md").read_text( + encoding="utf-8" + ) assert "第二轮摘要" in text and "第一轮摘要" not in text assert "新记忆124" in text # latest entries kept @@ -931,7 +931,9 @@ def test_layered_loading_own_full_others_summary_plus_two(tmp_path, monkeypatch) task_id = r["task"]["id"] for i in range(3): - _call(tm.handle_add_task_memory, output_dir=_od(repo), task_id=task_id, content=f"我的记忆{i}") + _call( + tm.handle_add_task_memory, output_dir=_od(repo), task_id=task_id, content=f"我的记忆{i}" + ) bob_text = ( f"{tm._SUMMARY_HEADING}\n\nbob 的早期工作摘要。\n\n> 指针行。\n\n" @@ -1043,7 +1045,9 @@ def test_user_id_change_old_file_becomes_warm_layer(tmp_path, monkeypatch): r = _call(tm.handle_create_task, output_dir=_od(repo), title="身份变更") task_id = r["task"]["id"] - _call(tm.handle_add_task_memory, output_dir=_od(repo), task_id=task_id, content="alice 时的记忆") + _call( + tm.handle_add_task_memory, output_dir=_od(repo), task_id=task_id, content="alice 时的记忆" + ) _write_user_mem( repo, task_id, @@ -1146,7 +1150,9 @@ def test_index_drops_entries_whose_dir_is_gone(tmp_path): lst = _call(tm.handle_list_tasks, output_dir=_od(repo)) ids = [t["id"] for t in lst["tasks"]] assert keep_id in ids and gone_id not in ids - data = json.loads((Path(repo) / "repowiki" / "tasks" / ".index.json").read_text(encoding="utf-8")) + data = json.loads( + (Path(repo) / "repowiki" / "tasks" / ".index.json").read_text(encoding="utf-8") + ) assert gone_id not in [t["id"] for t in data["tasks"]] diff --git a/tests/test_task_session_start.py b/tests/test_task_session_start.py index 7e54cb3..711cdce 100644 --- a/tests/test_task_session_start.py +++ b/tests/test_task_session_start.py @@ -145,8 +145,7 @@ def test_doctrine_injected_when_present(tmp_path): wiki = tmp_path / "repowiki" / "wiki" wiki.mkdir(parents=True, exist_ok=True) (wiki / "doctrine.md").write_text( - "---\ntype: Doctrine\nstatus: stable\n---\n\n" - "## Operating Thesis\n\nWrite deep modules.\n", + "---\ntype: Doctrine\nstatus: stable\n---\n\n## Operating Thesis\n\nWrite deep modules.\n", encoding="utf-8", ) diff --git a/tests/test_team_telemetry.py b/tests/test_team_telemetry.py index 41037d4..204d0fe 100644 --- a/tests/test_team_telemetry.py +++ b/tests/test_team_telemetry.py @@ -9,6 +9,7 @@ - telemetry.enabled=false switches writes to the gitignored local dir - capture integration: adoption declarations land in the user's stream """ + from __future__ import annotations import json @@ -33,8 +34,7 @@ def _all_events(od: Path) -> list: out = [] for p in sorted(d.glob("*.jsonl")): out.extend( - json.loads(line) - for line in p.read_text(encoding="utf-8").splitlines() if line.strip() + json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() if line.strip() ) return out @@ -46,6 +46,7 @@ def _days_ago(n: int) -> str: class TestUserId: def test_env_override_first(self, monkeypatch): from codewiki.src.config import user_id + monkeypatch.setenv("CODEWIKI_USER", "pseudonym-x") assert user_id() == "pseudonym-x" @@ -91,8 +92,7 @@ def test_multi_user_fold(self, tmp_path): p = tmp_path / ".meta" / "telemetry" p.mkdir(parents=True, exist_ok=True) (p / f"{user}.jsonl").write_text( - json.dumps({"t": "hit", "doc": "notes/a.md", - "at": _days_ago(1), "n": n}) + "\n", + json.dumps({"t": "hit", "doc": "notes/a.md", "at": _days_ago(1), "n": n}) + "\n", encoding="utf-8", ) agg = aggregate_usage(tmp_path) @@ -106,8 +106,7 @@ def test_adoption_key_dedup(self, tmp_path): {"t": "adopted", "doc": "notes/a.md", "at": "y", "key": "u/s1"}, # dup {"t": "adopted", "doc": "notes/a.md", "at": "z", "key": "u/s2"}, ] - (p / "u.jsonl").write_text( - "\n".join(json.dumps(e) for e in lines) + "\n", encoding="utf-8") + (p / "u.jsonl").write_text("\n".join(json.dumps(e) for e in lines) + "\n", encoding="utf-8") agg = aggregate_usage(tmp_path) assert agg["notes/a.md"]["adopted"] == 2 # distinct keys only @@ -116,16 +115,17 @@ def test_bad_line_skipped(self, tmp_path): p.mkdir(parents=True) (p / "u.jsonl").write_text( "not json at all\n" - + json.dumps({"t": "hit", "doc": "notes/a.md", - "at": _days_ago(0), "n": 4}) + "\n", - encoding="utf-8") + + json.dumps({"t": "hit", "doc": "notes/a.md", "at": _days_ago(0), "n": 4}) + + "\n", + encoding="utf-8", + ) agg = aggregate_usage(tmp_path) assert agg["notes/a.md"]["hits"] == 4 def test_mtime_cache_invalidates(self, tmp_path): record_hit(tmp_path, "notes/a.md", 1) assert aggregate_usage(tmp_path)["notes/a.md"]["hits"] == 1 - record_hit(tmp_path, "notes/a.md", 2) # mtime changes + record_hit(tmp_path, "notes/a.md", 2) # mtime changes assert aggregate_usage(tmp_path)["notes/a.md"]["hits"] == 3 def test_missing_dirs_empty(self, tmp_path): @@ -137,14 +137,23 @@ def test_events_flow_to_other_checkout(self, tmp_path): # user A's checkout — alice's events written under her own name # (write_telemetry pins the user file; record_* uses the machine user) from tests.telemetry_seed import write_telemetry + a = tmp_path / "a" / "repowiki" (a / "notes").mkdir(parents=True) (a / "notes" / "x.md").write_text("---\ntitle: x\n---\nbody", encoding="utf-8") - write_telemetry(a, "alice", [ - {"t": "hit", "doc": "notes/x.md", "at": _days_ago(1), "n": 6}, - {"t": "adopted", "doc": "notes/x.md", "at": "2026-08-22T10:00:00", - "key": "alice/sess-1"}, - ]) + write_telemetry( + a, + "alice", + [ + {"t": "hit", "doc": "notes/x.md", "at": _days_ago(1), "n": 6}, + { + "t": "adopted", + "doc": "notes/x.md", + "at": "2026-08-22T10:00:00", + "key": "alice/sess-1", + }, + ], + ) # simulate pull: md + telemetry land in user B's checkout b = tmp_path / "b" / "repowiki" @@ -152,8 +161,8 @@ def test_events_flow_to_other_checkout(self, tmp_path): (b / "notes").mkdir() (b / "notes" / "x.md").write_text("---\ntitle: x\n---\nbody", encoding="utf-8") import shutil - shutil.copytree( - a / ".meta" / "telemetry", b / ".meta" / "telemetry", dirs_exist_ok=True) + + shutil.copytree(a / ".meta" / "telemetry", b / ".meta" / "telemetry", dirs_exist_ok=True) agg = aggregate_usage(b) assert agg["notes/x.md"]["hits"] == 6 @@ -165,7 +174,8 @@ def test_disabled_writes_to_local_dir(self, tmp_path): od = tmp_path / "repowiki" od.mkdir() (od / "schema.yaml").write_text( - "conventions:\n telemetry:\n enabled: false\n", encoding="utf-8") + "conventions:\n telemetry:\n enabled: false\n", encoding="utf-8" + ) assert telemetry_enabled(od) is False record_hit(od, "notes/a.md", 2) assert not (od / ".meta" / "telemetry").exists() @@ -179,20 +189,32 @@ def test_adoption_declared_in_conversation_recorded(self, tmp_path): od = tmp_path / "repowiki" od.mkdir() (od / "notes").mkdir() - (od / "notes" / "pit.md").write_text( - "---\ntitle: t\n---\nbody", encoding="utf-8") + (od / "notes" / "pit.md").write_text("---\ntitle: t\n---\nbody", encoding="utf-8") turns = [ {"role": "user", "content": "q"}, - {"role": "assistant", "content": - 'a\n'}, + { + "role": "assistant", + "content": 'a\n', + }, ] - res = json.loads(handle_capture_conversation({ - "output_dir": str(od), "repo_path": str(tmp_path), - "conversation": turns, "source_session_id": "sess-42", - }, SessionStore())) + res = json.loads( + handle_capture_conversation( + { + "output_dir": str(od), + "repo_path": str(tmp_path), + "conversation": turns, + "source_session_id": "sess-42", + }, + SessionStore(), + ) + ) assert res.get("adopted_docs") == ["notes/pit.md"] # event landed in the CURRENT user's stream (user-dependent name) files = list((od / ".meta" / "telemetry").glob("*.jsonl")) assert len(files) == 1 - events = [json.loads(line) for line in files[0].read_text(encoding="utf-8").splitlines() if line.strip()] + events = [ + json.loads(line) + for line in files[0].read_text(encoding="utf-8").splitlines() + if line.strip() + ] assert any(e["t"] == "adopted" and e["doc"] == "notes/pit.md" for e in events) diff --git a/tests/test_transcript_filters.py b/tests/test_transcript_filters.py index cfd8114..07182c8 100644 --- a/tests/test_transcript_filters.py +++ b/tests/test_transcript_filters.py @@ -7,6 +7,7 @@ - ``distill_conversation._should_extract_l1`` / ``_filter_transcript_lines`` -- strict gate applied before the LLM call; drops pure-symbol/question rows. """ + from __future__ import annotations from pathlib import Path diff --git a/tests/test_usage_ranking.py b/tests/test_usage_ranking.py index 4c8167b..88e7c5a 100644 --- a/tests/test_usage_ranking.py +++ b/tests/test_usage_ranking.py @@ -12,6 +12,7 @@ - U2 lint linkage: stale_notes output sorted by (overdue_days desc, last_hit asc) with hit_count surfaced in the message (judgment untouched). """ + from __future__ import annotations import inspect @@ -45,8 +46,14 @@ def _days_ago(n: int) -> str: # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _mk_note(notes_dir: Path, name: str, title: str, body: str, - ntype: str = "general", status: str = "stable") -> Path: +def _mk_note( + notes_dir: Path, + name: str, + title: str, + body: str, + ntype: str = "general", + status: str = "stable", +) -> Path: notes_dir.mkdir(parents=True, exist_ok=True) p = notes_dir / name p.write_text( @@ -65,11 +72,11 @@ def _write_stats(od: Path, rows: dict) -> None: carries the whole count. """ from tests.telemetry_seed import seed_hits + seed_hits(od, rows) -def _write_stale_note(od: Path, name: str, *, stale_after: str, - title: str | None = None) -> Path: +def _write_stale_note(od: Path, name: str, *, stale_after: str, title: str | None = None) -> Path: notes = od / "notes" notes.mkdir(parents=True, exist_ok=True) fm = { @@ -114,8 +121,7 @@ def test_heat_below_cold_min_hits_not_penalized(): # hit=1 never reaches cold_min_hits: an ancient last_hit changes nothing. boosted = 1.0 + 0.03 * math.log(2) # ≈ 1.0208 — boost still applies assert compute_usage_heat(1, _days_ago(300), CFG) == pytest.approx(boosted) - assert compute_usage_heat(1, _days_ago(300), CFG) == \ - compute_usage_heat(1, _days_ago(1), CFG) + assert compute_usage_heat(1, _days_ago(300), CFG) == compute_usage_heat(1, _days_ago(1), CFG) def test_heat_cold_boundary_strictly_greater(): @@ -131,8 +137,7 @@ def test_heat_floor_at_08(): def test_heat_bad_last_hit_is_not_cold(): - assert compute_usage_heat(5, "not-a-date", CFG) == \ - compute_usage_heat(5, _days_ago(1), CFG) + assert compute_usage_heat(5, "not-a-date", CFG) == compute_usage_heat(5, _days_ago(1), CFG) assert compute_usage_heat(5, None, CFG) == compute_usage_heat(5, _days_ago(1), CFG) @@ -146,9 +151,18 @@ def test_config_defaults_when_schema_missing(): def test_config_overrides_from_schema(): - cfg = load_usage_ranking_config({"conventions": {"usage_ranking": { - "enabled": False, "boost_cap": 0.2, "cold_days": 90, "cold_min_hits": 5, - }}}) + cfg = load_usage_ranking_config( + { + "conventions": { + "usage_ranking": { + "enabled": False, + "boost_cap": 0.2, + "cold_days": 90, + "cold_min_hits": 5, + } + } + } + ) assert cfg["enabled"] is False assert cfg["boost_cap"] == 0.2 assert cfg["cold_days"] == 90 @@ -157,9 +171,17 @@ def test_config_overrides_from_schema(): def test_config_malformed_values_fall_back_per_key(): - cfg = load_usage_ranking_config({"conventions": {"usage_ranking": { - "boost_cap": "not-a-float", "cold_days": "x", "enabled": "yes", - }}}) + cfg = load_usage_ranking_config( + { + "conventions": { + "usage_ranking": { + "boost_cap": "not-a-float", + "cold_days": "x", + "enabled": "yes", + } + } + } + ) assert cfg["boost_cap"] == 0.15 assert cfg["cold_days"] == 180 assert cfg["enabled"] is True # non-bool keeps the default @@ -185,10 +207,13 @@ def test_json_path_cold_doc_ranked_after_warm(tmp_path): od = tmp_path / "repowiki" _mk_note(od / "notes", "n-cold.md", "gateway timeout alpha", SHARED_BODY) _mk_note(od / "notes", "n-warm.md", "gateway timeout bravo", SHARED_BODY) - _write_stats(od, { - "notes/n-cold.md": (5, _days_ago(200)), # hot then cold -> penalised - "notes/n-warm.md": (5, _days_ago(1)), # same hits, still warm - }) + _write_stats( + od, + { + "notes/n-cold.md": (5, _days_ago(200)), # hot then cold -> penalised + "notes/n-warm.md": (5, _days_ago(1)), # same hits, still warm + }, + ) wiki_search.build_full_index(od) res = wiki_search.search(od, QUERY) @@ -241,8 +266,9 @@ def test_sqlite_path_orders_hot_doc_first(tmp_path): # exemption: identical raw scores when usage weighting is off raw = cache.search(QUERY, output_dir=od, apply_usage=False) by_file = {r["file"]: r for r in raw} - assert by_file["notes/n-a.md"]["relevance_score"] == \ - by_file["notes/n-b.md"]["relevance_score"] + assert ( + by_file["notes/n-a.md"]["relevance_score"] == by_file["notes/n-b.md"]["relevance_score"] + ) finally: cache.close() @@ -251,10 +277,13 @@ def test_sqlite_path_cold_doc_ranked_after_warm(tmp_path): od = tmp_path / "repowiki" _mk_note(od / "notes", "n-cold.md", "gateway timeout alpha", SHARED_BODY) _mk_note(od / "notes", "n-warm.md", "gateway timeout bravo", SHARED_BODY) - _write_stats(od, { - "notes/n-cold.md": (5, _days_ago(200)), - "notes/n-warm.md": (5, _days_ago(1)), - }) + _write_stats( + od, + { + "notes/n-cold.md": (5, _days_ago(200)), + "notes/n-warm.md": (5, _days_ago(1)), + }, + ) cache = AnalysisCache(tmp_path, db_path=tmp_path / ".codewiki" / "analysis_cache.db") try: @@ -275,8 +304,9 @@ def _score_by_file(res): def test_enabled_false_matches_no_heat_ordering(tmp_path): od = tmp_path / "repowiki" (od / "notes").mkdir(parents=True) - (od / "schema.yaml").write_text(yaml.safe_dump( - {"conventions": {"usage_ranking": {"enabled": False}}}), encoding="utf-8") + (od / "schema.yaml").write_text( + yaml.safe_dump({"conventions": {"usage_ranking": {"enabled": False}}}), encoding="utf-8" + ) _mk_note(od / "notes", "n-a.md", "gateway timeout alpha", SHARED_BODY) _mk_note(od / "notes", "n-b.md", "gateway timeout bravo", SHARED_BODY) _write_stats(od, {"notes/n-a.md": (10, _days_ago(1))}) @@ -306,12 +336,18 @@ def test_apply_usage_false_keeps_usage_field_but_no_heat(tmp_path): assert len(res) == 2 by_file = {r["file"]: r for r in res} # no heat: identical-BM25 docs tie again - assert by_file["notes/n-a.md"]["relevance_score"] == \ - by_file["notes/n-b.md"]["relevance_score"] + assert by_file["notes/n-a.md"]["relevance_score"] == by_file["notes/n-b.md"]["relevance_score"] # usage field still present and populated - assert by_file["notes/n-a.md"]["usage"] == \ - {"hit_count": 10, "last_hit": _days_ago(1), "adopted_count": 0} - assert by_file["notes/n-b.md"]["usage"] == {"hit_count": 0, "last_hit": None, "adopted_count": 0} + assert by_file["notes/n-a.md"]["usage"] == { + "hit_count": 10, + "last_hit": _days_ago(1), + "adopted_count": 0, + } + assert by_file["notes/n-b.md"]["usage"] == { + "hit_count": 0, + "last_hit": None, + "adopted_count": 0, + } # --------------------------------------------------------------------------- # @@ -326,10 +362,22 @@ def test_usage_field_present_on_all_entries(tmp_path): wiki_search.build_full_index(od) res = wiki_search.search(od, QUERY) by_file = {r["file"]: r for r in res} - assert set(by_file["notes/n-a.md"]["usage"].keys()) == {"hit_count", "last_hit", "adopted_count"} - assert by_file["notes/n-a.md"]["usage"] == {"hit_count": 10, "last_hit": _days_ago(3), "adopted_count": 0} + assert set(by_file["notes/n-a.md"]["usage"].keys()) == { + "hit_count", + "last_hit", + "adopted_count", + } + assert by_file["notes/n-a.md"]["usage"] == { + "hit_count": 10, + "last_hit": _days_ago(3), + "adopted_count": 0, + } # never-retrieved docs carry a zero usage record - assert by_file["notes/n-b.md"]["usage"] == {"hit_count": 0, "last_hit": None, "adopted_count": 0} + assert by_file["notes/n-b.md"]["usage"] == { + "hit_count": 0, + "last_hit": None, + "adopted_count": 0, + } def test_sqlite_path_usage_field_present(tmp_path): @@ -341,7 +389,11 @@ def test_sqlite_path_usage_field_present(tmp_path): try: cache.build_search_index(od) res = cache.search(QUERY, output_dir=od) - assert res and res[0]["usage"] == {"hit_count": 7, "last_hit": _days_ago(2), "adopted_count": 0} + assert res and res[0]["usage"] == { + "hit_count": 7, + "last_hit": _days_ago(2), + "adopted_count": 0, + } finally: cache.close() @@ -364,19 +416,23 @@ def _populate(od: Path) -> None: _mk_note(od / "notes", "n-hot.md", "gateway timeout alpha", SHARED_BODY) _mk_note(od / "notes", "n-cold.md", "gateway timeout bravo", SHARED_BODY) _mk_note(od / "notes", "n-new.md", "gateway timeout charlie", SHARED_BODY) - _write_stats(od, { - "notes/n-hot.md": (10, _days_ago(1)), - "notes/n-cold.md": (5, _days_ago(200)), - }) - - od_json = tmp_path / "alpha" / "repowiki" # parent has no .codewiki -> JSON + _write_stats( + od, + { + "notes/n-hot.md": (10, _days_ago(1)), + "notes/n-cold.md": (5, _days_ago(200)), + }, + ) + + od_json = tmp_path / "alpha" / "repowiki" # parent has no .codewiki -> JSON od_sql = tmp_path / "beta" / "repowiki" _populate(od_json) _populate(od_sql) - wiki_search.build_full_index(od_json) # legacy JSON index - cache = AnalysisCache(tmp_path / "beta", - db_path=tmp_path / "beta" / ".codewiki" / "analysis_cache.db") + wiki_search.build_full_index(od_json) # legacy JSON index + cache = AnalysisCache( + tmp_path / "beta", db_path=tmp_path / "beta" / ".codewiki" / "analysis_cache.db" + ) try: cache.build_search_index(od_sql) @@ -384,8 +440,11 @@ def _populate(od: Path) -> None: res_sql = cache.search(QUERY, output_dir=od_sql) # same file set, same order (heat decides: hot > new > cold) - assert [r["file"] for r in res_json] == \ - ["notes/n-hot.md", "notes/n-new.md", "notes/n-cold.md"] + assert [r["file"] for r in res_json] == [ + "notes/n-hot.md", + "notes/n-new.md", + "notes/n-cold.md", + ] assert [r["file"] for r in res_sql] == [r["file"] for r in res_json] # per-file scores agree (JSON index rounds avg_doc_len; allow slack) scores_json = _score_by_file(res_json) @@ -414,20 +473,23 @@ def test_stale_notes_sorted_by_overdue_then_last_hit(tmp_path): # overdue 5 days, last hit 100 / 70 days ago (both beyond the defer window) _write_stale_note(od, "c.md", stale_after=_days_ago(5)) _write_stale_note(od, "d.md", stale_after=_days_ago(5)) - _write_stats(od, { - "notes/old.md": (5, _days_ago(90)), - "notes/new.md": (2, _days_ago(70)), - "notes/c.md": (1, _days_ago(100)), - "notes/d.md": (1, _days_ago(70)), - }) + _write_stats( + od, + { + "notes/old.md": (5, _days_ago(90)), + "notes/new.md": (2, _days_ago(70)), + "notes/c.md": (1, _days_ago(100)), + "notes/d.md": (1, _days_ago(70)), + }, + ) issues = [i for i in _check_stale_notes(od) if i["check"] == "stale_notes"] assert [i["file"] for i in issues] == [ - "notes/old.md", # overdue 40 (primary key: overdue_days desc) - "notes/new.md", # overdue 10 - "notes/nv.md", # overdue 5, never retrieved ("" sorts first) - "notes/c.md", # overdue 5, last hit 100d ago - "notes/d.md", # overdue 5, last hit 70d ago + "notes/old.md", # overdue 40 (primary key: overdue_days desc) + "notes/new.md", # overdue 10 + "notes/nv.md", # overdue 5, never retrieved ("" sorts first) + "notes/c.md", # overdue 5, last hit 100d ago + "notes/d.md", # overdue 5, last hit 70d ago ] # hit_count surfaced in the message by_file = {i["file"]: i for i in issues} @@ -443,9 +505,12 @@ def test_stale_notes_judgment_unchanged(tmp_path): od.mkdir(parents=True) _write_stale_note(od, "deferred.md", stale_after=_days_ago(10)) _write_stale_note(od, "due.md", stale_after=_days_ago(10)) - _write_stats(od, { - "notes/deferred.md": (3, _days_ago(2)), # recent hit -> deferred - "notes/due.md": (3, _days_ago(90)), # stale hit -> due - }) + _write_stats( + od, + { + "notes/deferred.md": (3, _days_ago(2)), # recent hit -> deferred + "notes/due.md": (3, _days_ago(90)), # stale hit -> due + }, + ) issues = [i for i in _check_stale_notes(od) if i["check"] == "stale_notes"] assert [i["file"] for i in issues] == ["notes/due.md"] diff --git a/tests/test_watch.py b/tests/test_watch.py index 728a38e..0e324d6 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -30,25 +30,24 @@ def _session(store, tmp_path): # Incremental refresh: modified file # ------------------------------------------------------------------ + def test_refresh_modified_file_updates_graph(analyzed_repo) -> None: """Editing b.py (adding a function) shows up in the session store; untouched files (a.py, c.py) keep their components.""" tmp_path, store = analyzed_repo session = _session(store, tmp_path) - (tmp_path / "b.py").write_text( - PY_B + "\ndef func_new():\n return 1\n", encoding="utf-8" - ) + (tmp_path / "b.py").write_text(PY_B + "\ndef func_new():\n return 1\n", encoding="utf-8") watcher = RepoWatcher(session, store, interval=1.0) changed = watcher.refresh_once() assert "b.py" in changed, changed ids = set(session.components.keys()) - assert "b.py::func_new" in ids # new function parsed - assert "b.py::func_b" in ids # old function still cached + assert "b.py::func_new" in ids # new function parsed + assert "b.py::func_b" in ids # old function still cached assert "b.py::func_other" in ids - assert "a.py::func_a" in ids # untouched file untouched + assert "a.py::func_a" in ids # untouched file untouched assert "c.py::func_c" in ids assert watcher.batches == 1 assert watcher.last_sync is not None @@ -59,9 +58,7 @@ def test_refresh_is_idempotent(analyzed_repo) -> None: updated — a git-based detector would loop forever on uncommitted edits).""" tmp_path, store = analyzed_repo session = _session(store, tmp_path) - (tmp_path / "b.py").write_text( - PY_B.replace("return 42", "return 43"), encoding="utf-8" - ) + (tmp_path / "b.py").write_text(PY_B.replace("return 42", "return 43"), encoding="utf-8") watcher = RepoWatcher(session, store, interval=1.0) assert "b.py" in watcher.refresh_once() @@ -73,6 +70,7 @@ def test_refresh_is_idempotent(analyzed_repo) -> None: # Incremental refresh: new / deleted files # ------------------------------------------------------------------ + def test_refresh_new_file_adds_components(analyzed_repo) -> None: tmp_path, store = analyzed_repo session = _session(store, tmp_path) @@ -102,6 +100,7 @@ def test_refresh_deleted_file_removes_components(analyzed_repo) -> None: # Degradation # ------------------------------------------------------------------ + def test_watcher_degrades_gracefully(analyzed_repo, monkeypatch) -> None: """A failing poll stops the loop and marks the watcher degraded — the session falls back to manual mode instead of crashing.""" @@ -127,6 +126,7 @@ def boom(): # MCP entry point # ------------------------------------------------------------------ + def test_handle_watch_repo_lifecycle(analyzed_repo) -> None: tmp_path, store = analyzed_repo session = _session(store, tmp_path) @@ -137,9 +137,7 @@ def test_handle_watch_repo_lifecycle(analyzed_repo) -> None: assert parsed["watch"] is None # start - raw = handle_watch_repo( - {"repo_path": str(tmp_path), "action": "start", "interval": 1.0}, store - ) + raw = handle_watch_repo({"repo_path": str(tmp_path), "action": "start", "interval": 1.0}, store) parsed = json.loads(raw) assert parsed["ok"] is True assert parsed["watch"]["running"] is True @@ -156,9 +154,7 @@ def test_handle_watch_repo_lifecycle(analyzed_repo) -> None: assert session.watcher is None # start again after stop (restart works) - raw = handle_watch_repo( - {"repo_path": str(tmp_path), "action": "start", "interval": 1.0}, store - ) + raw = handle_watch_repo({"repo_path": str(tmp_path), "action": "start", "interval": 1.0}, store) assert json.loads(raw)["watch"]["running"] is True handle_watch_repo({"repo_path": str(tmp_path), "action": "stop"}, store) @@ -176,6 +172,7 @@ def test_handle_watch_repo_requires_session(tmp_path) -> None: # graph_stale attachment on query tools # ------------------------------------------------------------------ + def test_attach_graph_stale_noop_without_watcher(analyzed_repo) -> None: tmp_path, store = analyzed_repo session = _session(store, tmp_path) diff --git a/uv.lock b/uv.lock index 2083f71..26a1bca 100644 --- a/uv.lock +++ b/uv.lock @@ -298,38 +298,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, ] -[[package]] -name = "black" -version = "26.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "mypy-extensions" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs" }, - { name = "pytokens" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/37/5628dd55bf2b34257fc7603f0fe97c40e3aaf24265f416a9c85c95ca1436/black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73", size = 679439, upload-time = "2026-05-18T16:53:36.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/24/99/7744b906703228264ef73bdd534df88ec1ef3de45c4e78f6d31b9e32d0c9/black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8", size = 2012518, upload-time = "2026-05-18T17:05:20.108Z" }, - { url = "https://files.pythonhosted.org/packages/b7/c0/c5a3b1636dfd09c42534f2b3cf33506814f6d3e066fb0879ffa16c1ae860/black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217", size = 1816016, upload-time = "2026-05-18T17:05:21.84Z" }, - { url = "https://files.pythonhosted.org/packages/1f/0e/36044316b65ca471d3bb6d3703fd06fb50c6b727c3562f6a5a3153634f88/black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d", size = 1884150, upload-time = "2026-05-18T17:05:23.546Z" }, - { url = "https://files.pythonhosted.org/packages/b3/33/dafc5808c2af43672912111d7c3354af1615f7e2be3bed7a878461abbe4d/black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264", size = 1486825, upload-time = "2026-05-18T17:05:25.004Z" }, - { url = "https://files.pythonhosted.org/packages/82/14/b965ee6ad2a311f28bdbf692def3ee9848d2ae289dab28b27657fcee3e78/black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418", size = 1288646, upload-time = "2026-05-18T17:05:26.477Z" }, - { url = "https://files.pythonhosted.org/packages/3f/5c/c384363980e11e25ca6b93205949bb331fbf35f4e0dbec376dfa6326cec8/black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3", size = 2009020, upload-time = "2026-05-18T17:05:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/0b/df/9f31c5e0babbfed77d505fc5d120beb98b21b33feaeded3924ea941fe360/black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0", size = 1813335, upload-time = "2026-05-18T17:05:31.266Z" }, - { url = "https://files.pythonhosted.org/packages/fb/24/8e7b9a2fa61b0afd82209efe937557d180a1fa055bd7f6161eb9defc3719/black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294", size = 1881614, upload-time = "2026-05-18T17:05:32.718Z" }, - { url = "https://files.pythonhosted.org/packages/49/ad/b4e0d9365ba8ac34f6bbab62a4b1b2dd5d618fac3fa1b8db968c844201b5/black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a", size = 1488925, upload-time = "2026-05-18T17:05:34.259Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/652b859bf5df88a751c30451b09338f7fd26a77d1271c666992f836b7711/black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52", size = 1289883, upload-time = "2026-05-18T17:05:36.019Z" }, - { url = "https://files.pythonhosted.org/packages/a6/16/a8da8eb208c51c7f4ce74609a45d0dcc6d8a2141e45e81ee5289d1bb0d59/black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168", size = 2004800, upload-time = "2026-05-18T17:05:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/11/8a/a479296a19e383b70a725882a6cf3d786540601ff03cabbaaf1cce864c5a/black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3", size = 1815576, upload-time = "2026-05-18T17:05:40.309Z" }, - { url = "https://files.pythonhosted.org/packages/81/6b/cfaf3d39f25132c156a068f6b805576c9103a84086019507c70e1911ee7d/black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18", size = 1877927, upload-time = "2026-05-18T17:05:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/66/76/302e313964bcff7e28df329d39f84f5270095730d85ff0acc260610a0d82/black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50", size = 1511860, upload-time = "2026-05-18T17:05:43.943Z" }, - { url = "https://files.pythonhosted.org/packages/27/4e/a3827e35e0e567f9f9ee59e2a0ab979267dca98718f25547ca8c6733afd4/black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae", size = 1316632, upload-time = "2026-05-18T17:05:45.521Z" }, - { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, -] - [[package]] name = "cachetools" version = "7.1.7" @@ -463,6 +431,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.5.1" @@ -608,7 +585,7 @@ wheels = [ [[package]] name = "codewiki-plus" -version = "5.4.3" +version = "5.4.4" source = { editable = "." } dependencies = [ { name = "click" }, @@ -654,8 +631,8 @@ dependencies = [ [package.optional-dependencies] dev = [ - { name = "black" }, { name = "mypy" }, + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -664,8 +641,8 @@ dev = [ [package.dev-dependencies] dev = [ - { name = "black" }, { name = "mypy" }, + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -674,7 +651,6 @@ dev = [ [package.metadata] requires-dist = [ - { name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" }, { name = "click", specifier = ">=8.1.0" }, { name = "coding-agent-wrapper", specifier = ">=0.1.2" }, { name = "colorama", specifier = ">=0.4.6" }, @@ -692,6 +668,7 @@ requires-dist = [ { name = "networkx", specifier = ">=3.5" }, { name = "openai", specifier = ">=1.107.0" }, { name = "pathspec", specifier = ">=0.12.1" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0.0" }, { name = "psutil", specifier = ">=7.0.0" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-ai", specifier = ">=1.0.6" }, @@ -705,7 +682,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.4" }, { name = "rich", specifier = ">=14.1.0" }, { name = "ruamel-yaml", specifier = ">=0.18.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.3" }, { name = "tree-sitter", specifier = ">=0.23.2" }, { name = "tree-sitter-c", specifier = ">=0.21.4" }, { name = "tree-sitter-c-sharp", specifier = ">=0.23.1" }, @@ -724,12 +701,12 @@ provides-extras = ["dev"] [package.metadata.requires-dev] dev = [ - { name = "black", specifier = ">=23.0.0" }, { name = "mypy", specifier = ">=1.5.0" }, + { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pytest", specifier = ">=7.4.0" }, { name = "pytest-asyncio", specifier = ">=0.21.0" }, { name = "pytest-cov", specifier = ">=4.1.0" }, - { name = "ruff", specifier = ">=0.1.0" }, + { name = "ruff", specifier = "==0.16.3" }, ] [[package]] @@ -905,6 +882,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, ] +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1384,6 +1370,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + [[package]] name = "idna" version = "3.19" @@ -2091,6 +2086,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "openai" version = "2.54.0" @@ -2273,6 +2277,22 @@ version = "1.3.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/15/fb/ea010a1fa5773b4fa185506b6eecabfa898338a124a515329ec7d4ae0d98/pminit-1.3.2.tar.gz", hash = "sha256:2c5bcfdbc6df07c640f9d51bd0c9766615911d4294e2ec43fb7db2becab54c2b", size = 8266, upload-time = "2026-06-16T21:26:19.542Z" } +[[package]] +name = "pre-commit" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/89/1f3e8e1fc3e97de0fa963495832f581f025f29471602a309e48808244292/pre_commit-4.6.2.tar.gz", hash = "sha256:8f5d7bfb021ecdbcd9d49d89847082dd24172ccde534390081a679ad046e2441", size = 198670, upload-time = "2026-08-10T22:07:18.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, +] + [[package]] name = "prompt-toolkit" version = "3.0.53" @@ -2770,6 +2790,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "python-discovery" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/8f/3c92c45737f654f2488ab3662b7604a55d3d35146d37c9ce80f5c95b95a6/python_discovery-1.5.3.tar.gz", hash = "sha256:e500eb24025fb7c4876c1fdcfbafd9028a10c71b661aee38cb6fb0de594518c1", size = 82477, upload-time = "2026-08-24T14:48:46.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/12/823d9a321904ccfd2969a24b84fdfd1e6614c707ec569c62879bf1dbc6c5/python_discovery-1.5.3-py3-none-any.whl", hash = "sha256:8305296358f1aa2ed302a25b84be7df84fef8ca47c7dce2da63cb7325333044e", size = 38290, upload-time = "2026-08-24T14:48:45.305Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.3" @@ -2816,35 +2848,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/26/9b74985db1afb99a11fad9f15189426d6ba7c7fb8e42cf8b13c4a445ef5f/pythonmonkey-1.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:dd807356c16b9b457c7e56e67184e6cae8b2a6f101ab8657510bd0559c55e5a1", size = 13357901, upload-time = "2026-06-16T21:25:47.682Z" }, ] -[[package]] -name = "pytokens" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, - { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, - { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, - { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, - { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, - { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, - { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, - { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, - { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, - { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, - { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, - { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, - { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, - { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, - { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, - { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, - { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, - { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, - { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, - { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, -] - [[package]] name = "pywin32" version = "312" @@ -3617,6 +3620,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/2b/ebd108734a8204c6b4b93c681c9a38c5273b3ccd5d129fee4ffc1d97772c/uvicorn-0.52.3-py3-none-any.whl", hash = "sha256:116af2710dbf47c80f463cd20ee4884b6662f4c9f227d797ddc7279d2fcc2c7c", size = 79859, upload-time = "2026-08-13T16:50:01.323Z" }, ] +[[package]] +name = "virtualenv" +version = "21.7.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/60/fc54e876e34f94dd0cf0185aaecfd4bfa906653f003d9b2fb21428642fca/virtualenv-21.7.5.tar.gz", hash = "sha256:a73c4246fba3c8901ff9717399f466e00eeca5a3834981f1a6ebb4f1e94de2f8", size = 5346743, upload-time = "2026-08-25T05:39:16.14Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/d8/401141bf45637be916c86d325bd821c5838c7eff83294b934cd94e774e4f/virtualenv-21.7.5-py3-none-any.whl", hash = "sha256:e36ca889510ab6cb0b1dca93c59e5431dd4422a3c88f487358d470c90af8c07a", size = 5324697, upload-time = "2026-08-25T05:39:14.229Z" }, +] + [[package]] name = "wcwidth" version = "0.8.2" From a5be55f7765f8c8687b83a30ebacc8876df3d4e3 Mon Sep 17 00:00:00 2001 From: LiberiFatali Date: Thu, 27 Aug 2026 19:58:24 +0700 Subject: [PATCH 2/4] docs: note fork-before-clone in CONTRIBUTING MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contributors must fork mambo-wang/CodeWiki-Plus first, clone their fork, and add upstream remote — direct clone of upstream is read-only. --- CONTRIBUTING.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc609ec..3b732dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,15 @@ Prerequisites: Python 3.12+, [uv](https://docs.astral.sh/uv/) (or `pip`). +1. Fork https://github.com/mambo-wang/CodeWiki-Plus on GitHub, then: + ```bash -git clone https://github.com/mambo-wang/CodeWiki-Plus.git +git clone https://github.com//CodeWiki-Plus.git cd CodeWiki-Plus +git remote add upstream https://github.com/mambo-wang/CodeWiki-Plus.git uv sync --frozen # or `pip install -e .[dev]` uv run pre-commit install # enables ruff check + format on commit uv run pytest -q # verify setup ``` + +Create a feature branch, push to your fork, and open a PR against `mambo-wang:develop`. From 965c13b3ee407e542ad6f966d1e9dc6d3a539a50 Mon Sep 17 00:00:00 2001 From: LiberiFatali Date: Thu, 27 Aug 2026 20:00:53 +0700 Subject: [PATCH 3/4] docs: clarify installation and PR instructions in CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b732dc..edb25af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,9 +8,9 @@ Prerequisites: Python 3.12+, [uv](https://docs.astral.sh/uv/) (or `pip`). git clone https://github.com//CodeWiki-Plus.git cd CodeWiki-Plus git remote add upstream https://github.com/mambo-wang/CodeWiki-Plus.git -uv sync --frozen # or `pip install -e .[dev]` +uv sync --frozen # installs project dependencies uv run pre-commit install # enables ruff check + format on commit uv run pytest -q # verify setup ``` -Create a feature branch, push to your fork, and open a PR against `mambo-wang:develop`. +1. Create a feature branch, push to your fork, and open a PR against `mambo-wang:develop`. From 5d42e10f37d559ba0deac6cb5349a32777e01b6d Mon Sep 17 00:00:00 2001 From: LiberiFatali Date: Thu, 27 Aug 2026 20:01:56 +0700 Subject: [PATCH 4/4] docs: minor numbering --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index edb25af..1ca63f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,4 +13,4 @@ uv run pre-commit install # enables ruff check + format on commit uv run pytest -q # verify setup ``` -1. Create a feature branch, push to your fork, and open a PR against `mambo-wang:develop`. +2. Create a feature branch, push to your fork, and open a PR against `mambo-wang:develop`.