diff --git a/Cargo.lock b/Cargo.lock index a94537b..7db761a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1759,7 +1759,7 @@ dependencies = [ [[package]] name = "neuromesh-api" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "futures-util", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "neuromesh-cache" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "neuromesh-core", @@ -1800,7 +1800,7 @@ dependencies = [ [[package]] name = "neuromesh-cli" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "dirs", @@ -1830,7 +1830,7 @@ dependencies = [ [[package]] name = "neuromesh-context" -version = "1.1.0" +version = "1.2.0" dependencies = [ "blake3", "chrono", @@ -1849,7 +1849,7 @@ dependencies = [ [[package]] name = "neuromesh-core" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "dirs", @@ -1863,7 +1863,7 @@ dependencies = [ [[package]] name = "neuromesh-embed" -version = "1.1.0" +version = "1.2.0" dependencies = [ "dirs", "fastembed", @@ -1878,7 +1878,7 @@ dependencies = [ [[package]] name = "neuromesh-graph" -version = "1.1.0" +version = "1.2.0" dependencies = [ "bincode", "chrono", @@ -1900,7 +1900,7 @@ dependencies = [ [[package]] name = "neuromesh-graph-proxy" -version = "1.1.0" +version = "1.2.0" dependencies = [ "dirs", "neuromesh-core", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "neuromesh-index" -version = "1.1.0" +version = "1.2.0" dependencies = [ "blake3", "chrono", @@ -1931,7 +1931,7 @@ dependencies = [ [[package]] name = "neuromesh-local-ai" -version = "1.1.0" +version = "1.2.0" dependencies = [ "neuromesh-core", "parking_lot", @@ -1942,7 +1942,7 @@ dependencies = [ [[package]] name = "neuromesh-mcp" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "neuromesh-cache", @@ -1967,7 +1967,7 @@ dependencies = [ [[package]] name = "neuromesh-memory" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "dirs", @@ -1981,7 +1981,7 @@ dependencies = [ [[package]] name = "neuromesh-observability" -version = "1.1.0" +version = "1.2.0" dependencies = [ "chrono", "dirs", @@ -1996,7 +1996,7 @@ dependencies = [ [[package]] name = "neuromesh-parser" -version = "1.1.0" +version = "1.2.0" dependencies = [ "neuromesh-core", "neuromesh-index", @@ -2027,7 +2027,7 @@ dependencies = [ [[package]] name = "neuromesh-provider" -version = "1.1.0" +version = "1.2.0" dependencies = [ "futures-util", "neuromesh-core", @@ -2040,7 +2040,7 @@ dependencies = [ [[package]] name = "neuromesh-router" -version = "1.1.0" +version = "1.2.0" dependencies = [ "neuromesh-context", "neuromesh-core", @@ -2053,7 +2053,7 @@ dependencies = [ [[package]] name = "neuromesh-task" -version = "1.1.0" +version = "1.2.0" dependencies = [ "neuromesh-core", "neuromesh-parser", diff --git a/Cargo.toml b/Cargo.toml index 295baa0..d225b22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ exclude = ["tests/fixtures/mini-axum"] [workspace.package] -version = "1.1.0" +version = "1.2.0" edition = "2021" rust-version = "1.80" authors = ["NeuroMesh Team "] diff --git a/crates/neuromesh-mcp/src/protocol.rs b/crates/neuromesh-mcp/src/protocol.rs index 83cb2d9..10d0535 100644 --- a/crates/neuromesh-mcp/src/protocol.rs +++ b/crates/neuromesh-mcp/src/protocol.rs @@ -114,7 +114,7 @@ pub fn canonical_tool_name(name: &str) -> String { } pub fn initialize_instructions() -> &'static str { - "NeuroMesh MCP v1.1.0 (code-context-engine) — agent loop: (1) get_context_packet with the user task as written (default engine: fast — graph + query-side lexical expansion; prompt only; no client keywords; opt in to hybrid/deep for MiniLM embed-primary); (2) if coverage is partial/no_seed_resolved/no_confident_match, neuromesh_search_symbols or neuromesh_expand_gap; (3) neuromesh_expand_fold only when you need a folded body; (4) neuromesh_trace / neuromesh_get_dependencies for callers and blast radius; (5) after a successful edit, neuromesh_record_feedback with touched nodes. Check retrieval.resolution_tier and retrieval.cache_hit. Prefer these tools over reading whole files. neuromesh_get_context is deprecated — use get_context_packet." + "NeuroMesh MCP v1.2.0 (code-context-engine) — agent loop: (1) get_context_packet with the user task as written (default engine: fast — graph + query-side lexical expansion; prompt only; no client keywords; opt in to hybrid/deep for MiniLM embed-primary); for a pasted issue or bug report the response also lists where_to_look — the files to open next, best first; (2) if coverage is partial/no_seed_resolved/no_confident_match, neuromesh_search_symbols or neuromesh_expand_gap; (3) neuromesh_expand_fold only when you need a folded body; (4) neuromesh_trace / neuromesh_get_dependencies for callers and blast radius; (5) after a successful edit, neuromesh_record_feedback with touched nodes. Check retrieval.resolution_tier and retrieval.cache_hit. Prefer these tools over reading whole files. neuromesh_get_context is deprecated — use get_context_packet." } pub fn tool_success(id: Option, val: &Value) -> Value { diff --git a/crates/neuromesh-mcp/src/tools.rs b/crates/neuromesh-mcp/src/tools.rs index 9f65070..331b80b 100644 --- a/crates/neuromesh-mcp/src/tools.rs +++ b/crates/neuromesh-mcp/src/tools.rs @@ -500,12 +500,31 @@ impl McpToolHandler { neuromesh_graph::timing("mcp: telemetry", t_s); let t_s = std::time::Instant::now(); Ok({ - let value = cache_and_build( + let mut value = cache_and_build( &self.packet_cache, &self.graph.project_id().0, &build, detail, ); + // A long report (an issue, a bug template) gets the ranked + // files to open next: the packet ships 1-5, the file to + // change is in the top 10 far more often (SWE-bench dev: + // 0.72 at 10 vs 0.48 for the packet). + if task_desc.split_whitespace().count() + >= neuromesh_parser::text_normalize::REPORT_WORDS + { + if let Some(obj) = value.as_object_mut() { + obj.insert( + "where_to_look".into(), + json!(neuromesh_context::gold::localization_order( + &self.graph, + &task_desc, + &view, + 10 + )), + ); + } + } neuromesh_graph::timing("mcp: cache_and_build", t_s); let t_s = std::time::Instant::now(); #[cfg(feature = "embeddings")] diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8305836..8422642 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,7 +2,36 @@ All notable user-facing changes live here. The README stays a product guide, not a version diary. -## Unreleased +## 1.2.0 — 2026-10-09 + +### Bug reports and issues + +- **Where to look next** — for a pasted issue or bug report (60+ words) `get_context_packet` adds + `where_to_look`: the packet's files, then the next source files, best first (tests, docs and + fixtures last). `neuromesh packet --json` has the same list as `localization`. A packet holds 1–5 + files; the file to change is in the top 10 far more often. +- **Definition-level ranking for reports** — a report is also read definition by definition (file + head + every function/class span the index recorded, title counted three times) and fused with the + whole-file ranking. +- **Report hygiene** — issue-template scaffolding (HTML comments, links, checklists, section headings) + no longer counts as words of the report; code-like tokens (`L031`, `E501`, `utf8`) are kept whole. +- Measured on SWE-bench (file-level Acc@1/3/5/10, all gold files in the top k), tuned on the + SWE-bench *dev* split only: dev (225 issues) 0.160/0.347/0.427/0.538 with plain BM25 → + **0.249/0.484/0.600/0.680**; SWE-bench Lite test, run once (276 strict holdout issues): plain BM25 + 0.301/0.507/0.587/0.721 → **0.486/0.710/0.750/0.804** (`docs/measured.md`). + +### Speed + +- **Cold index of a large repository ~8× faster** — django (3.5k files): first packet 3m20s → 26 s. + Link resolution no longer rescans every path per import hint or every neighbour per call (147 s → + 10 s); the file walk checks and reads files in parallel (16 s → 3 s). Same graph. + +### Housekeeping + +- Debug builds keep line tables only (target/debug had reached 43 GB on Windows); the benchmark + index cache deletes superseded snapshots. + +## 1.1.x follow-ups (released with 1.2.0) ### Retrieval diff --git a/docs/measured.md b/docs/measured.md index 14d8d32..ce3248d 100644 --- a/docs/measured.md +++ b/docs/measured.md @@ -47,6 +47,36 @@ tuning sets, whose golds *do* want the neighbours — see `docs/planning/stage5- (F36′, F46, F47). Per-language, the gap is widest on flat package namespaces (R: 0.24) and narrowest where a question names one object (Scala: 0.80). +## SWE-bench Lite file localisation (2026-10-09, v1.2.0, single holdout run) + +Question = the issue text as filed; gold = the files the reference patch edits; metric = file-level +Acc@k (every gold file in the top k), as LocAgent reports it. The engine's list is `localization` in +`neuromesh packet --json` (`where_to_look` over MCP). No LLM, no embeddings, CPU only; p50 2.8 s per +issue including a cold index of the checkout. + +Tuning used only the SWE-bench **dev** split (225 issues, six other repositories). Lite test was run +once with the release binary. The 24 Lite instances from flask/requests/seaborn/xarray/pylint were +looked at in session 16 and are excluded from the strict row. + +| Lite test | Acc@1 | Acc@3 | Acc@5 | Acc@10 | +|---|---|---|---|---| +| **this engine, strict holdout (276)** | **0.486** | **0.710** | **0.750** | **0.804** | +| this engine, all 300 | 0.473 | 0.703 | 0.747 | 0.810 | +| plain BM25 over the checkout (276) | 0.301 | 0.507 | 0.587 | 0.721 | +| 1.1.0 + issue mode, packet order (258, session 16) | 0.337 | 0.504 | 0.535 | – | + +95% bootstrap intervals for the strict row: @1 [0.43, 0.55], @3 [0.65, 0.76], @5 [0.70, 0.80]. + +Published reference points (LocAgent, arXiv 2503.09089, Table 4; their filtered Lite subset, not the +same instances, so read as context): BM25 0.387 / 0.518 / 0.617; Jina-Code-v2 embeddings 0.434 / +0.712 / 0.803; CodeRankEmbed 0.526 / 0.777 / 0.847; Agentless with GPT-4o 0.672 / 0.745 / 0.745, +with Claude-3.5 0.726 / 0.792 / 0.796; LocAgent with Claude-3.5 0.777 / 0.920 / 0.942. LLM agents +remain ahead at Acc@1; the engine is an LLM-free first stage they can start from. + +SWE-bench dev (tuning set, 225): plain BM25 0.160 / 0.347 / 0.427 / 0.538 → 0.249 / 0.484 / 0.600 / +0.680. Raw results: `swebench/results-final-test.jsonl` (outside the repository); harness +`scripts/swebench_localize.py`, scoring `scripts/research/eval_loc.py`. + ## Speed (same machine, same hour, release binary without embeddings) ultralytics, 931 files: index ≈3.6 s; one question end-to-end p50 ≈0.70 s (n=10, cold process each diff --git a/docs/planning/handoff-2026-10-09-session17.fa.md b/docs/planning/handoff-2026-10-09-session17.fa.md new file mode 100644 index 0000000..2877ac4 --- /dev/null +++ b/docs/planning/handoff-2026-10-09-session17.fa.md @@ -0,0 +1,59 @@ +# Handoff — session 17 (۲۰۲۶-۱۰-۰۸ تا ۰۹) + +نقشه: `12-roadmap-2026-10-08-session17.fa.md`. لاگ پژوهشی: `docs/research/contributions-log.md` §8.1–8.4. + +## merge شده + +| PR | محتوا | عدد | +|---|---|---| +| #131 | issue mode، اصلاح escalation، cap روی file_rank | — | +| #132 | سرعت ایندکس | django سرد 3m20s → 26s (links 147s → 10s، scan 16s → 3s)، گراف یکسان | +| #133 | دیسک | target/ به 55GB رسیده بود (debug 43GB، کش ایندکس 6.7GB)؛ مرزدار شد | +| #134 | issue: حذف قالب، توکن‌های `L031`، `chunk_rank` (BM25 سطح definition، عنوان ×3)، لیست `localization` | dev-fast Acc@5 0.456 → 0.596 | + +## branch باز: `phase-o/release-1.2.0` + +نسخه 1.2.0، `where_to_look` در پاسخ MCP برای گزارش‌های بلند (smoke test شد)، CHANGELOG، +harness سریع‌تر (worktree reuse + sharding با hash پایدار). بعد از عدد holdout: measured.md، PR، tag +`v1.2.0` (workflow release سه باینری را می‌سازد)، به‌روزرسانی MCP دسکتاپ. + +## روش (مقاله) + +- dev = SWE-bench dev split (۲۲۵، ۶ ریپو؛ `swebench/devset/`). dev-fast = ۱۰ از هر ریپو (۵۹). +- holdout = Lite test (۳۰۰) — یک بار با باینری main `d6854b6` → `swebench/results-final-test.jsonl`. +- ارزیابی: `scripts/research/eval_loc.py` (Acc@k، `--ci` بوت‌استرپ، `--fuse`). + +## اعداد dev (اندازه‌گیری‌شده) + +| | @1 | @3 | @5 | @10 | +|---|---|---|---|---| +| BM25 ساده (۲۲۵) | 0.160 | 0.347 | 0.427 | 0.538 | +| ما، localization (۲۲۵) | **0.249** | **0.484** | **0.600** | **0.680** | +| ما، ۱۶۶ issue تنظیم‌نشده | 0.247 | 0.482 | 0.608 | 0.675 | +| تک‌فایلی (۱۲۴) BM25 → ما | 0.290 → 0.452 | 0.532 → 0.710 | 0.613 → **0.863** | 0.774 → 0.911 | + +## رد شده با عدد (تکرار نکن) + +tf بدنه در BM25F؛ fusion برای seedها (بی‌اثر)؛ کنار گذاشتن فایل‌های داده (بی‌اثر)؛ رأی همسایه‌های گراف +(@1 0.254 → 0.136)؛ reranker v2 روی issue (@3 0.492 → 0.373، ۸ ثانیه)؛ embedding chunkها روی CPU +(۲.۶ chunk/s). + +## پذیرفته ولی هنوز در موتور نیست + +reranker v2 برای پرسش‌های کوتاه زبان ساده: ripgrep R@3 0.208 → 0.583، click 0.750 → 0.958، self +0.643 → 0.679 (فقط ترتیب را عوض می‌کند، نه محتوا). گام بعدی: پیاده‌سازی اختیاری (فقط وقتی مدل نصب +است) + یک ست تازه‌ی holdout برای تأیید. + +## گام‌های بعدی + +1. انتشار 1.2.0 (بالا). +2. reranker v2 برای پرسش‌های ساده در موتور. +3. مقاله §5: holdout‌های بزرگ‌تر (≥۵۰ سؤال، دو نفر)، task success per token با مدل، ablation جدول. +4. مرحله‌ی LLM (Agentless-style، انتخاب از `where_to_look`) — نیاز به کلید API. + +## تله‌ها + +- `df -h /c` قبل از کار؛ target/debug > 10GB → پاک کن. +- شاردهای harness را با hash پایدار تقسیم کن (حالا پیش‌فرض)؛ هرگز دو اجرا روی یک فایل خروجی. +- clone‌ها blobless‌اند: قطع proxy = خطای checkout (۴۶ pydicom یک بار). +- `sed` با `\\` در MSYS خراب می‌کند؛ ویرایش Python با نرمال‌سازی CRLF یا Edit. diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index 244a703..ca4d6de 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -222,3 +222,43 @@ than lexical ranking on all three sets, and hurts on *long issue reports* (trunc loses the report's specifics; the first stage already reads the whole report). The earlier v1 rejection (§4) was a different experiment — it changed packet *content* (recall collapsed); this one only reorders. Paper: the query-length split is a finding in itself. + +Further rejected on dev-fast (session 17, numbers vs d = 0.254/0.492/0.576/0.695, 59 issues): + +| idea | Acc@1 | Acc@3 | Acc@5 | Acc@10 | verdict | +|---|---|---|---|---|---| +| data/CI files (`.yml`, `.json`, `.toml`…) moved behind code in the list | = | = | = | = | no effect, not kept | +| graph neighbours of the top three files as a third RRF vote, equal weight (LocAgent-style expansion) | 0.136 | 0.373 | 0.525 | 0.661 | rejected | +| same, weight 0.25 | 0.136 | 0.475 | 0.576 | 0.678 | rejected | + +Graph expansion pulls in hubs the top files call (base classes, utils) ahead of the edited module; +LocAgent gets value from the graph through an LLM choosing which edges to follow, not from a +blind neighbour vote. + +### 8.5 Holdout: SWE-bench Lite test, one run (2026-10-09, binary main d6854b6 = v1.2.0 engine) + +| | Acc@1 | Acc@3 | Acc@5 | Acc@10 | +|---|---|---|---|---| +| **ours, strict holdout (276; excludes the 24 session-16 dev-class instances)** | **0.486** [0.43,0.55] | **0.710** [0.65,0.76] | **0.750** [0.70,0.80] | **0.804** | +| ours, all 300 | 0.473 | 0.703 | 0.747 | 0.810 | +| ours, packet order only (276) | 0.366 | 0.536 | 0.558 | – | +| plain BM25 (276) | 0.301 | 0.507 | 0.587 | 0.721 | +| ours before session 17 (v1.1 + #131, packet, 258) | 0.337 | 0.504 | 0.535 | – | + +Cost: CPU only, no LLM, no embeddings; p50 2.8 s / p90 11.4 s per issue *including a cold index of +the checkout*; packet p50 11.1k tokens. + +Reading against LocAgent Table 4 (different filtered subset — context, not a head-to-head): above +Jina-Code-v2 at @1 (0.486 vs 0.434), equal at @3 (0.710 vs 0.712), below at @5 (0.750 vs 0.803); +below CodeRankEmbed everywhere; at @5 level with Agentless+GPT-4o (0.750 vs 0.745) and below every +LLM method at @1. Dev → holdout transfer: the gain over BM25 on dev (+0.09/+0.14/+0.17 @1/3/5) held +on holdout (+0.19/+0.20/+0.16) — no sign of overfitting to the dev split. + +Harness lessons (for the paper's reproducibility section): blobless clones need network at checkout +time; Windows MAX_PATH broke 14 scikit-learn checkouts under a deep temp path (fixed with a short +work dir); worktree re-creation per instance made the run IO-bound (now reused); positional +sharding overlapped when a resumed run started (now hash sharding). + +Paper next: (1) head-to-head on LocAgent's exact 274-instance subset; (2) an LLM stage over +`where_to_look` (Agentless-style file choice) to measure Acc@1 at a fraction of agent tokens; +(3) ablation table from §8.3 on the holdout (one run per component removed). diff --git a/scripts/swebench_localize.py b/scripts/swebench_localize.py index ee1ef96..12d3027 100644 --- a/scripts/swebench_localize.py +++ b/scripts/swebench_localize.py @@ -22,6 +22,7 @@ import sys import tempfile import time +import zlib sys.path.insert(0, os.path.dirname(__file__)) from compare_baselines import Bm25, repo_files # noqa: E402 @@ -36,6 +37,16 @@ def checkout(clone, commit, dest): git's worktree bookkeeping takes locks, so a failed add is retried (39 of the first 276 test-split instances were lost to that, not to the engine).""" clone, dest = os.path.abspath(clone), os.path.abspath(dest) + # An existing worktree is moved to the commit in place: only files that + # differ are rewritten. Re-creating a django worktree per instance (6k + # files through the virus scanner) made a run IO-bound at minutes each. + if os.path.isdir(os.path.join(dest, ".git")) or os.path.isfile(os.path.join(dest, ".git")): + r = subprocess.run(["git", "-C", dest, "checkout", "--force", "--detach", commit], + capture_output=True, text=True) + subprocess.run(["git", "-C", dest, "clean", "-ffdxq"], capture_output=True) + ok = subprocess.run(["git", "-C", dest, "rev-parse", "HEAD"], capture_output=True, text=True) + if r.returncode == 0 and ok.stdout.strip().startswith(commit[:12]): + return last = "" for attempt in range(5): if os.path.isdir(dest): @@ -125,7 +136,10 @@ def _main(): todo = [r for r in rows if r["instance_id"] not in done] if args.shard: i, n = (int(x) for x in args.shard.split("/")) - todo = todo[i::n] + # By a stable hash of the id, not by position in what is left: shards + # that start while another run is still writing the same files would + # otherwise see different leftovers and overlap. + todo = [r for r in todo if zlib.crc32(r["instance_id"].encode()) % n == i] if args.limit: todo = todo[: args.limit] os.makedirs(args.work, exist_ok=True)