From 33e06ace60e2ebfae5150141700232df569c7b6c Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Fri, 9 Oct 2026 01:30:43 +0330 Subject: [PATCH 1/2] issue mode: definition-level ranking + report hygiene; SWE-bench dev-fast Acc@5 0.456 -> 0.596 Measured on SWE-bench *dev* (tuning set; no repository shared with Lite test), dev-fast = 10 issues per dev repository, paired runs, file-level Acc@k: Acc@1 Acc@3 Acc@5 Acc@10 plain BM25 0.175 0.386 0.474 0.526 before 0.246 0.421 0.456 - this 0.263 0.509 0.596 0.719 - Issue-template scaffolding is stripped from reports (>= 60 words): HTML comments, links, checklists, section headings (not `#` lines inside code fences). It ranked CODE_OF_CONDUCT.md and the CLI module on sqlfluff. - Code-like tokens (`L031`, `LT02`, `E501`, `utf8`) are kept whole: every word tokenizer dropped the one term a rule report shares with `rules/L031.py`. - Definition-level BM25 (`chunk_rank`): file head + every definition span the graph recorded, path words on each, a file scores its best definition, the report's title counts 3x. Built lazily from the sources, cached like file_rank. Prototype in scripts/research/lex_variants.py first. - `localization` in `packet --json`: the packet's files, then the ranking's runners-up, tests/docs/fixtures last, and for a long report RRF with the definition ranking. A packet holds 1-5 files; Acc@5 was capped by it. Rejected with numbers (contributions-log 8.3/8.4): real body tf in BM25F (0.544 -> 0.509 @5), the same fusion choosing packet seeds (no change), a cross-encoder (jina-reranker-v2) on issues (0.492 -> 0.373 @3; it helps short plain-language questions: ripgrep 0.208 -> 0.583, click 0.750 -> 0.958). Twelve sets unchanged (dev 0.938 ... holdout-web 0.917/0.643, concept 0.679/0.398, concept-holdout 0.500/0.156, concept-holdout2 0.958/0.342). Co-Authored-By: Claude Opus 5.5 --- crates/neuromesh-cli/src/commands/packet.rs | 5 + crates/neuromesh-context/src/gold.rs | 61 +++++ crates/neuromesh-graph/src/chunk_rank.rs | 218 ++++++++++++++++ crates/neuromesh-graph/src/file_rank.rs | 21 +- crates/neuromesh-graph/src/graph.rs | 69 +++++ crates/neuromesh-graph/src/lib.rs | 1 + crates/neuromesh-parser/src/lib.rs | 4 +- crates/neuromesh-parser/src/text_normalize.rs | 64 +++++ crates/neuromesh-task/src/signature.rs | 3 +- docs/research/contributions-log.md | 104 ++++++++ scripts/research/dense_loc.py | 247 ++++++++++++++++++ scripts/research/eval_loc.py | 108 ++++++++ scripts/research/lex_variants.py | 119 +++++++++ scripts/research/lexical_ablation.py | 115 ++++++++ scripts/research/rerank_swe.py | 90 +++++++ scripts/research/rerank_test.py | 114 ++++++++ 16 files changed, 1339 insertions(+), 4 deletions(-) create mode 100644 crates/neuromesh-graph/src/chunk_rank.rs create mode 100644 scripts/research/dense_loc.py create mode 100644 scripts/research/eval_loc.py create mode 100644 scripts/research/lex_variants.py create mode 100644 scripts/research/lexical_ablation.py create mode 100644 scripts/research/rerank_swe.py create mode 100644 scripts/research/rerank_test.py diff --git a/crates/neuromesh-cli/src/commands/packet.rs b/crates/neuromesh-cli/src/commands/packet.rs index a0dc860..a74d417 100644 --- a/crates/neuromesh-cli/src/commands/packet.rs +++ b/crates/neuromesh-cli/src/commands/packet.rs @@ -41,6 +41,9 @@ struct PacketJsonOut { /// Repository-relative paths, best first (highest activation of any /// node in the file): what an evaluation needs for hit@k. selected_paths: Vec, + /// Packet files, then the next source files of the whole-question ranking + /// (20 deep): where to look, best first. + localization: Vec, /// The whole-question file ranking (BM25F), top 10, for evaluation. ranked_paths: Vec, identifiers: Vec, @@ -96,6 +99,7 @@ pub fn execute(args: &[String]) -> Result<()> { neuromesh_graph::timing("activate_tiered", started); let selected_paths = neuromesh_context::gold::packet_file_order(&graph, &prompt, &view); + let localization = neuromesh_context::gold::localization_order(&graph, &prompt, &view, 20); let mut files: Vec = packet_file_names(&view).into_iter().collect(); files.sort(); let reduction = if workspace_tokens > 0 { @@ -125,6 +129,7 @@ pub fn execute(args: &[String]) -> Result<()> { selected_files_count: files.len(), selected_files: files.clone(), selected_paths, + localization, ranked_paths: graph .file_rank(&prompt, 10) .into_iter() diff --git a/crates/neuromesh-context/src/gold.rs b/crates/neuromesh-context/src/gold.rs index 3bdd97c..f835bbb 100644 --- a/crates/neuromesh-context/src/gold.rs +++ b/crates/neuromesh-context/src/gold.rs @@ -1045,6 +1045,67 @@ pub fn packet_file_order( fused.into_iter().map(|(p, _)| p).collect() } +/// Where to look, best first, `k` deep: the packet's files in +/// [`packet_file_order`], then the whole-question ranking's next source files +/// (tests, docs and examples left out). A packet is 1–5 files; a localisation +/// list ("which file does this issue need changed?", file-level Acc@k as +/// SWE-bench localisation papers report it) needs more candidates than the +/// packet ships, and the runners-up are the files an agent should open next. +pub fn localization_order( + graph: &neuromesh_graph::NeuralProjectGraph, + prompt: &str, + view: &ContextView, + k: usize, +) -> Vec { + // A packet may carry a test, a fixture or a doc page as context; where to + // change code is a source file, so those go last. + let (mut out, aside): (Vec, Vec) = packet_file_order(graph, prompt, view) + .into_iter() + .partition(|p| !crate::selector::is_noise_path(std::path::Path::new(p))); + let mut seen: std::collections::HashSet = + out.iter().chain(aside.iter()).cloned().collect(); + for r in graph.file_rank(prompt, 4 * k.max(10)) { + if out.len() >= k { + break; + } + if crate::selector::is_noise_path(&r.path) { + continue; + } + let p = r.path.to_string_lossy().replace('\\', "/"); + if seen.insert(p.clone()) { + out.push(p); + } + } + // A long report also votes definition by definition: reciprocal-rank + // fusion (k = 60, equal weight) of this list with the definition-level + // ranking (SWE-bench dev-fast, 57 issues: Acc@1 0.211 → 0.298, @5 0.544 + // → 0.614 in the prototype). + if prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS { + const K: f32 = 60.0; + let by_def: Vec = graph + .chunk_rank(prompt, 4 * k.max(10)) + .into_iter() + .filter(|r| !crate::selector::is_noise_path(&r.path)) + .map(|r| r.path.to_string_lossy().replace('\\', "/")) + .collect(); + let mut fused: std::collections::HashMap = std::collections::HashMap::new(); + for list in [&out, &by_def] { + for (i, p) in list.iter().enumerate() { + *fused.entry(p.clone()).or_insert(0.0) += 1.0 / (K + i as f32 + 1.0); + } + } + let mut order: Vec<(String, f32)> = fused.into_iter().collect(); + order.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + out = order + .into_iter() + .map(|(p, _)| p) + .take(k.max(out.len())) + .collect(); + } + out.extend(aside); + out +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/neuromesh-graph/src/chunk_rank.rs b/crates/neuromesh-graph/src/chunk_rank.rs new file mode 100644 index 0000000..4445b95 --- /dev/null +++ b/crates/neuromesh-graph/src/chunk_rank.rs @@ -0,0 +1,218 @@ +//! Definition-level BM25 for long reports (issues, bug templates). +//! +//! A whole-file ranker dilutes a report's words over every definition of a +//! big module; read definition by definition, the function the report is +//! about scores on its own (LocAgent indexes entity contents the same way). +//! Chunks are the file head (first 40 lines) and every definition the graph +//! recorded with a line span, each carrying its file's path words; a file +//! scores its best chunk. The report's first line — an issue's title — counts +//! three times. +//! +//! SWE-bench dev (`dev-fast`, 57 issues): fused by reciprocal rank with the +//! packet's localisation list, Acc@1 0.211 → 0.298, @5 0.544 → 0.614, @10 +//! 0.667 → 0.702 (prototype, `scripts/research/lex_variants.py`). +//! +//! Built lazily from the sources on disk on the first long report, cached +//! with the file ranker's key. + +use crate::file_rank::{stemmer, terms_of, RankedFile}; +use neuromesh_core::NodeId; +use rayon::prelude::*; +use std::collections::HashMap; +use std::path::PathBuf; + +const K1: f32 = 1.2; +const B: f32 = 0.75; +const HEAD_LINES: usize = 40; +const MAX_CHUNK_LINES: usize = 400; +const TITLE_EXTRA: f32 = 2.0; + +/// One file as the index needs it: id, path, source, definition spans +/// (1-based lines, end inclusive). +pub(crate) struct ChunkSource { + pub id: NodeId, + pub path: PathBuf, + pub source: String, + pub spans: Vec>, +} + +/// A file's chunks as term counts, before they are numbered. +type FileChunks = (NodeId, PathBuf, Vec>); + +#[derive(Default)] +pub(crate) struct ChunkRankIndex { + files: Vec<(NodeId, PathBuf)>, + file_of: Vec, + len: Vec, + avg_len: f32, + post: HashMap>, +} + +impl ChunkRankIndex { + pub(crate) fn build(sources: Vec) -> Self { + // Per file: its chunks' term counts, computed in parallel. + let per_file: Vec = sources + .into_par_iter() + .map(|f| { + let st = stemmer(); + let rel = f.path.to_string_lossy().replace('\\', "/"); + let stem_path = match rel.rsplit_once('.') { + Some((head, ext)) if !ext.contains('/') => head.to_string(), + _ => rel.clone(), + }; + let path_terms = terms_of(&stem_path, &st, true); + let lines: Vec<&str> = f.source.lines().collect(); + let head = 1..HEAD_LINES.min(lines.len()); + let mut chunks = Vec::new(); + for span in std::iter::once(head).chain(f.spans.iter().cloned()) { + let start = span.start.max(1) - 1; + let end = span.end.min(lines.len()).min(start + MAX_CHUNK_LINES); + if start >= end { + continue; + } + let mut counts: HashMap = HashMap::new(); + for t in terms_of(&lines[start..end].join("\n"), &st, false) + .into_iter() + .chain(path_terms.iter().cloned()) + { + let c = counts.entry(t).or_insert(0); + *c = c.saturating_add(1); + } + if !counts.is_empty() { + chunks.push(counts); + } + } + (f.id, f.path, chunks) + }) + .collect(); + let mut idx = ChunkRankIndex::default(); + for (id, path, chunks) in per_file { + let file = idx.files.len() as u32; + idx.files.push((id, path)); + for counts in chunks { + let chunk = idx.file_of.len() as u32; + idx.file_of.push(file); + idx.len.push(counts.values().map(|&c| c as u32).sum()); + for (t, c) in counts { + idx.post.entry(t).or_default().push((chunk, c)); + } + } + } + let n = idx.len.len().max(1) as f32; + idx.avg_len = (idx.len.iter().map(|&l| l as f32).sum::() / n).max(1.0); + idx + } + + /// Files by their best chunk for `prompt`, best first. + pub(crate) fn rank(&self, prompt: &str, limit: usize) -> Vec { + let n = self.len.len() as f32; + if n == 0.0 { + return Vec::new(); + } + let st = stemmer(); + let body = neuromesh_parser::strip_issue_boilerplate(prompt); + let title = body + .lines() + .map(str::trim) + .find(|l| !l.is_empty()) + .unwrap_or(""); + let mut q: HashMap = HashMap::new(); + for t in terms_of(&body, &st, false) { + q.insert(t, 1.0); + } + for t in terms_of(title, &st, false) { + *q.entry(t).or_insert(0.0) += TITLE_EXTRA; + } + let mut terms: Vec<(String, f32)> = q.into_iter().collect(); + terms.sort_by(|a, b| a.0.cmp(&b.0)); + terms.truncate(256); + let mut score: HashMap = HashMap::new(); + for (t, w) in &terms { + let Some(list) = self.post.get(t) else { + continue; + }; + let df = list.len() as f32; + let idf = ((n - df + 0.5) / (df + 0.5) + 1.0).ln(); + for &(chunk, tf) in list { + let tf = tf as f32; + let norm = 1.0 - B + B * self.len[chunk as usize] as f32 / self.avg_len; + *score.entry(chunk).or_insert(0.0) += w * idf * tf * (K1 + 1.0) / (tf + K1 * norm); + } + } + let mut best: HashMap = HashMap::new(); + for (chunk, s) in score { + let f = self.file_of[chunk as usize]; + let e = best.entry(f).or_insert(0.0); + if s > *e { + *e = s; + } + } + let mut ranked: Vec = best + .into_iter() + .map(|(f, s)| { + let (id, path) = &self.files[f as usize]; + RankedFile { + id: id.clone(), + path: path.clone(), + score: s, + matched: 0, + } + }) + .collect(); + ranked.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.path.cmp(&b.path)) + }); + ranked.truncate(limit); + ranked + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn src(id: &str, path: &str, source: &str, spans: Vec>) -> ChunkSource { + ChunkSource { + id: NodeId::new(id), + path: PathBuf::from(path), + source: source.to_string(), + spans, + } + } + + #[test] + fn a_report_reaches_the_definition_it_describes_inside_a_big_file() { + // `big.py` spells the report's words once each across unrelated + // definitions; `small.py` has one definition that is all about them. + let mut big = String::new(); + for i in 0..60 { + big.push_str(&format!("def helper_{i}():\n return {i}\n")); + } + big.push_str("def parse_header():\n timezone = 0\n"); + let small = "def normalize_timezone_offset(header):\n \ + # parse the timezone offset of a header\n \ + return header.timezone.offset\n"; + let index = ChunkRankIndex::build(vec![ + src( + "big", + "pkg/big.py", + &big, + vec![std::ops::Range { + start: 121, + end: 122, + }], + ), + src( + "small", + "pkg/dates.py", + small, + vec![std::ops::Range { start: 1, end: 3 }], + ), + ]); + let ranked = index.rank("Timezone offset in header is parsed wrong", 5); + assert_eq!(ranked[0].path, PathBuf::from("pkg/dates.py")); + } +} diff --git a/crates/neuromesh-graph/src/file_rank.rs b/crates/neuromesh-graph/src/file_rank.rs index c063b57..b19b2b0 100644 --- a/crates/neuromesh-graph/src/file_rank.rs +++ b/crates/neuromesh-graph/src/file_rank.rs @@ -214,18 +214,21 @@ pub(crate) struct FileRankIndex { avg_body_len: f32, } -fn stemmer() -> Stemmer { +pub(crate) fn stemmer() -> Stemmer { Stemmer::create(Algorithm::English) } /// Lowercase, split identifiers into words, drop short/non-alphabetic /// pieces and stopwords, stem. -fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec { +pub(crate) fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec { let mut out = Vec::new(); for raw in text.split(|c: char| !c.is_ascii_alphanumeric() && c != '_') { if raw.is_empty() { continue; } + for code in raw.split('_').filter(|p| is_code_token(p)) { + out.push(code.to_ascii_lowercase()); + } for part in neuromesh_parser::tokenize_ident(raw) { let w = part.to_lowercase(); if w.len() < 2 || !w.chars().all(|c| c.is_ascii_alphabetic()) { @@ -240,6 +243,19 @@ fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec { out } +/// `L031`, `LT02`, `E501`, `W0611`, `utf8`, `int64`: letters then digits, the +/// way linters, encodings and widths are named. Word splitting drops the +/// digits, which threw away the one term a report about rule L031 shares +/// with `rules/L031.py` (SWE-bench dev, sqlfluff). Kept whole, unstemmed. +fn is_code_token(s: &str) -> bool { + let letters = s.bytes().take_while(u8::is_ascii_alphabetic).count(); + let digits = s.len() - letters; + (1..=4).contains(&letters) + && (1..=5).contains(&digits) + && s.len() >= 3 + && s.bytes().skip(letters).all(|b| b.is_ascii_digit()) +} + /// Stemmed content words of any text (a symbol name, a doc summary), /// stopwords dropped — comparable with [`query_terms`]. pub fn text_terms(text: &str) -> Vec { @@ -263,6 +279,7 @@ const SYNONYM_WEIGHT: f32 = 0.35; /// [`SYNONYM_WEIGHT`]): a question word in a cluster also searches the /// rest of its cluster. pub fn weighted_query_terms(prompt: &str) -> Vec<(String, f32)> { + let prompt = &neuromesh_parser::strip_issue_boilerplate(prompt); let st = stemmer(); let mut out: Vec<(String, f32)> = Vec::new(); let mut seen = HashSet::new(); diff --git a/crates/neuromesh-graph/src/graph.rs b/crates/neuromesh-graph/src/graph.rs index 6bb9ef6..5f8bc4d 100644 --- a/crates/neuromesh-graph/src/graph.rs +++ b/crates/neuromesh-graph/src/graph.rs @@ -140,6 +140,8 @@ struct DerivedIndexes { file_rank_key: Option<(u64, u64)>, file_rank: Arc, file_hint_revision: Option, + chunk_rank_key: Option<(u64, u64)>, + chunk_rank: Arc, file_hint_table: Arc)>>, } @@ -763,6 +765,73 @@ impl NeuralProjectGraph { self.file_rank_index().rank(&terms, limit) } + /// Files ranked by their best definition for a long report — see + /// [`crate::chunk_rank`]. Builds the definition index from the sources on + /// first use. + pub fn chunk_rank(&self, prompt: &str, limit: usize) -> Vec { + let mut end = prompt.len().min(32 * 1024); + while !prompt.is_char_boundary(end) { + end -= 1; + } + self.chunk_rank_index().rank(&prompt[..end], limit) + } + + fn chunk_rank_index(&self) -> Arc { + let (key, files) = { + let data = self.inner.read(); + let key = (data.generation, data.file_to_nodes.len() as u64); + { + let cached = self.derived.read(); + if cached.chunk_rank_key == Some(key) { + return Arc::clone(&cached.chunk_rank); + } + } + let mut files = Vec::new(); + for (path, ids) in &data.file_to_nodes { + if neuromesh_core::is_low_priority_source_path(path) { + continue; + } + let mut file_id = None; + let mut spans = Vec::new(); + for id in ids { + let Some(node) = data.mesh.node(id) else { + continue; + }; + if node.node_type == NodeType::File { + file_id = Some(id.clone()); + } else if let Some(r) = &node.line_range { + if r.end > r.start { + spans.push(r.clone()); + } + } + } + if let Some(id) = file_id { + files.push((id, path.clone(), spans)); + } + } + (key, files) + }; + let t0 = std::time::Instant::now(); + let sources: Vec = files + .into_par_iter() + .filter_map(|(id, path, spans)| { + let source = self.read_source(&path)?; + Some(crate::chunk_rank::ChunkSource { + id, + path, + source, + spans, + }) + }) + .collect(); + let index = Arc::new(crate::chunk_rank::ChunkRankIndex::build(sources)); + crate::timing("chunk_rank build", t0); + let mut cached = self.derived.write(); + cached.chunk_rank_key = Some(key); + cached.chunk_rank = Arc::clone(&index); + index + } + fn file_rank_index(&self) -> Arc { let data = self.inner.read(); // Structural key only: learning updates bump the node revision on diff --git a/crates/neuromesh-graph/src/lib.rs b/crates/neuromesh-graph/src/lib.rs index 00dda62..7d8b03e 100644 --- a/crates/neuromesh-graph/src/lib.rs +++ b/crates/neuromesh-graph/src/lib.rs @@ -1,4 +1,5 @@ pub mod activation; +mod chunk_rank; pub mod concept_index; pub mod edge; pub mod embeddings; diff --git a/crates/neuromesh-parser/src/lib.rs b/crates/neuromesh-parser/src/lib.rs index f83fecf..9d97d88 100644 --- a/crates/neuromesh-parser/src/lib.rs +++ b/crates/neuromesh-parser/src/lib.rs @@ -32,5 +32,7 @@ pub use identifiers::{ stem_search_queries, tokenize_camel_chunk, tokenize_ident, PromptAnchors, }; pub use semantic::{SemanticTypeExtractor, SemanticTypeMap, TypeDefinition}; -pub use text_normalize::{normalize_keyword, normalize_prompt_tokens, normalize_unicode}; +pub use text_normalize::{ + normalize_keyword, normalize_prompt_tokens, normalize_unicode, strip_issue_boilerplate, +}; pub use types::{AstAnalysisResult, ParsedImport, ParsedRelationship, ParsedSymbol}; diff --git a/crates/neuromesh-parser/src/text_normalize.rs b/crates/neuromesh-parser/src/text_normalize.rs index d47816b..f130484 100644 --- a/crates/neuromesh-parser/src/text_normalize.rs +++ b/crates/neuromesh-parser/src/text_normalize.rs @@ -9,6 +9,51 @@ const LRM: char = '\u{200e}'; const RLM: char = '\u{200f}'; const BOM: char = '\u{feff}'; +/// Words from which a prompt reads as a report (an issue, a bug template) +/// rather than a question; below it nothing is stripped. +pub const REPORT_WORDS: usize = 60; + +/// The part of a bug report that is about this bug. +/// +/// Issue trackers wrap every report in the same template: HTML comments with +/// instructions, checklists ("- [X] I searched the issues and found no +/// similar issues", "I agree to follow the Code of Conduct"), section +/// headings ("### Expected Behaviour"), and links. Read as words, that +/// scaffolding is the same in every report of a project and ranked +/// `CODE_OF_CONDUCT.md` and the CLI module first on SWE-bench dev +/// (sqlfluff). Short prompts are returned unchanged. +pub fn strip_issue_boilerplate(prompt: &str) -> String { + use std::sync::OnceLock; + static COMMENT: OnceLock = OnceLock::new(); + static URL: OnceLock = OnceLock::new(); + if prompt.split_whitespace().count() < REPORT_WORDS { + return prompt.to_string(); + } + let comment = COMMENT.get_or_init(|| regex::Regex::new(r"(?s)").unwrap()); + let url = URL.get_or_init(|| regex::Regex::new(r"https?://\S+").unwrap()); + let text = comment.replace_all(prompt, " "); + let text = url.replace_all(&text, " "); + // Inside a fenced block a `# ...` line is a code comment, not a heading. + let mut fenced = false; + let mut kept = Vec::new(); + for line in text.lines() { + let t = line.trim_start(); + if t.starts_with("```") { + fenced = !fenced; + kept.push(line); + continue; + } + let checklist = ["- [", "* [", "+ ["] + .iter() + .any(|p| t.starts_with(p) && t.get(4..5) == Some("]")); + let heading = !fenced && t.starts_with('#') && t.trim_start_matches('#').starts_with(' '); + if !checklist && !heading { + kept.push(line); + } + } + kept.join("\n") +} + /// Normalize a client keyword: trim, NFKC, strip control/format chars, collapse space. pub fn normalize_keyword(raw: &str) -> String { let s = normalize_unicode(raw); @@ -48,6 +93,25 @@ fn apply_optional_script_fixes(s: &mut String) { mod tests { use super::*; + #[test] + fn issue_template_scaffolding_is_dropped_code_comments_kept() { + let filler = "word ".repeat(REPORT_WORDS); + let report = format!( + "L031 triggers on a query without a join\n\ + ### Search before asking\n\ + - [X] I searched the [issues](https://github.com/o/r/issues) and found no similar issues.\n\ + \n\ + ```python\n# keep this comment\nx = 1\n```\n{filler}" + ); + let out = strip_issue_boilerplate(&report); + assert!(out.contains("L031 triggers")); + assert!(out.contains("# keep this comment")); + assert!(!out.contains("Search before asking")); + assert!(!out.contains("similar issues")); + assert!(!out.contains("describe the bug")); + assert_eq!(strip_issue_boilerplate("### short question"), "### short question"); + } + #[test] fn nfkc_and_zwnj() { let raw = "مدل\u{200c}محصول"; diff --git a/crates/neuromesh-task/src/signature.rs b/crates/neuromesh-task/src/signature.rs index ad3357d..a1f1430 100644 --- a/crates/neuromesh-task/src/signature.rs +++ b/crates/neuromesh-task/src/signature.rs @@ -1,6 +1,7 @@ use neuromesh_core::{TaskIntent, TaskRisk, TaskSignature}; use neuromesh_parser::{ extract_embedded_code_tokens, extract_prompt_anchors, is_imperative_verb, normalize_unicode, + strip_issue_boilerplate, }; use uuid::Uuid; @@ -8,7 +9,7 @@ pub struct TaskSignatureExtractor; impl TaskSignatureExtractor { pub fn extract(prompt: &str) -> TaskSignature { - let prompt = normalize_unicode(prompt); + let prompt = strip_issue_boilerplate(&normalize_unicode(prompt)); let lower = prompt.to_lowercase(); let anchors = extract_prompt_anchors(&prompt); let embedded = extract_embedded_code_tokens(&prompt); diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index e63a631..244a703 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -118,3 +118,107 @@ across many anchors and fills the packet with the wrong files (including test fi whole text (BM25) does better. This is the first standard-benchmark result and it is a weakness — the next engine work targets it (whole-question ranking for long reports, traceback file paths as anchors). Published reference points to beat: Agentless and LocAgent file-level Acc@k (LLM-based). + +### 8.1 Session 17 (2026-10-08): protocol, reference points, first full test-split number + +**Protocol.** Tuning set = SWE-bench *dev* split (225 issues; pvlib, pydicom, sqlfluff, astroid, +pyvista, marshmallow — no repository shared with Lite test). Iteration subset `dev-fast` = first 10 +instances per dev repository by id (59). Holdout = SWE-bench Lite *test* (300). The 24 Lite-test +instances from flask/requests/seaborn/xarray/pylint that session 16 tuned issue mode on are +dev-class and reported separately. Metric = file-level Acc@k as in LocAgent (all gold files in the +top k). + +**Published reference points** (LocAgent, arXiv 2503.09089, Table 4; SWE-bench Lite, file level): + +| method | Acc@1 | Acc@3 | Acc@5 | +|---|---|---|---| +| BM25 | 38.69 | 51.82 | 61.68 | +| Jina-Code-v2 (embedding) | 43.43 | 71.17 | 80.29 | +| CodeRankEmbed (embedding) | 52.55 | 77.74 | 84.67 | +| Agentless (Claude-3.5) | 72.63 | 79.20 | 79.56 | +| SWE-agent (Claude-3.5) | 77.37 | 87.23 | 90.15 | +| LocAgent (Claude-3.5) | 77.74 | 91.97 | 94.16 | + +**Lite test, v1.1.0 + issue mode (#131), single run, 237 of 276 non-dev instances** (39 lost to +git worktree lock contention between harness shards, re-run separately): + +| | hit@1 | hit@3 | hit@5 | packet | +|---|---|---|---|---| +| ours (packet order) | 0.344 | 0.506 | 0.534 | 0.538 | +| ours BM25F ranking alone | 0.259 | 0.470 | 0.571 | — | +| plain BM25 (our implementation) | 0.295 | 0.506 | 0.578 | — | + +Honest reading: on real issues the engine is level with BM25 and well below dense retrievers and +LLM agents. Hit@5 ≈ packet recall because the packet holds 1–5 files: Acc@5 is capped by packet +size, which motivates a separate localisation list (packet first, then ranked runners-up). + +**Dense retrieval cost (measured, negative for local-first use).** Jina-Code-v2 (int8 ONNX, CPU, +6 threads, loaded machine) embedded astroid's 3,364 function/class chunks (256 tokens) in 1,314 s +— 2.6 chunks/s. At ~70 GFLOP per 256-token chunk for a 137–161M encoder, whole-repository chunk +embedding of django (~40k chunks) is hours on a laptop CPU. The published embedding numbers above +are bought with GPU-scale compute; a local engine has to get there lexically/structurally or embed +only a candidate short list. + +### 8.2 Index speed (C10) + +`NM_TIMING=1` on a django checkout (3.5k files) located the cold-start delay the upstream author +reported: `finalize_links` was 147 s of a ~200 s first packet. Causes and fixes (#132): + +| stage | before | after | cause | +|---|---|---|---| +| resolve_file_hint | 84 s | 2 s | every hint normalised every path (4.7k × 3.5k) | +| imported_files_of | 22 s | 0.3 s | neighbour walk per relation; now once per file per pass | +| call resolution | 58 s | 7 s | same (file, name) resolved repeatedly; memoised per pass | +| scan | 16 s | 3 s | serial canonicalize (root re-canonicalised per file) + serial reads | +| whole cold packet | 3m20s | 26 s | | + +Same graph: the nine sets and three concept sets did not move. Paper use: indexing cost table +(item 5.6) and the claim that a structural index is cheap enough to build per checkout. + +### 8.3 Issue-mode ranking on SWE-bench dev (session 17, dev-class) + +`dev-fast` = 59 issues (10 per dev repository), paired runs, file-level Acc@k (all gold files in +top k; dev issues average 1.9 gold files, so numbers sit below Lite's single-file ones). 95% bootstrap +CI on 57 issues is about ±0.12, so only differences of ~0.05+ that repeat across k are read as real. + +| run | change | Acc@1 | Acc@3 | Acc@5 | Acc@10 | +|---|---|---|---|---|---| +| plain BM25 (file) | — | 0.175 | 0.386 | 0.474 | 0.526 | +| a (v1.1 + #131 + #132) | packet order | 0.246 | 0.421 | 0.456 | — | +| b | + issue boilerplate stripped, code tokens (`L031`), localisation list (packet, then ranked runners-up, tests/docs last) | 0.211 | 0.456 | 0.544 | 0.667 | +| c | b + real body term frequency in BM25F | 0.193 | 0.439 | 0.509 | 0.649 | +| **d** | b + definition-level BM25 (title ×3), RRF with the list | **0.263** | **0.509** | **0.596** | **0.719** | +| e | d + the same fusion choosing the long-report seeds | = d | = d | = d | = d | + +Prototype first (Python, `scripts/research/lex_variants.py`): definition-level BM25 with the title +counted three times, fused by RRF with b's list, gave 0.298/0.544/0.614/0.702 — the engine port (d) +reproduces it within noise. Kept: b, d (C11, C12). Rejected with numbers: c (body tf; also lowered +nothing on the twelve sets but did not help), e (no change). + +Failure classes read on dev (sqlfluff): rule codes `L031`/`LT02` dropped by every word tokenizer +(ours and BM25) although the gold file is `rules/L031.py`; issue-template scaffolding ("Search before +asking", "found no similar issues", "Code of Conduct", links) ranked `CODE_OF_CONDUCT.md` and the CLI +module; test fixtures (`.sql`) entering the list. All three are generic, not sqlfluff-specific. + +Twelve sets unchanged with d (dev 0.938, large 0.675, holdout 0.700, holdout-c 0.589, holdout-lang +0.589, holdout-ml 0.478, holdout-ml2 0.632, holdout-cfg 0.767, holdout-web 0.917/0.643, concept +0.679/0.398, concept-holdout 0.500/0.156, concept-holdout2 0.958/0.342). + +### 8.4 Cross-encoder reranker v2 (jina-reranker-v2, int8 ONNX, 278M) — split verdict + +Rerank of the engine's top-10 localisation list, file digest = path + signature/doc lines +(`scripts/research/rerank_test.py`, `rerank_swe.py`): + +| set | first stage R@3 | reranker alone | 50/50 blend | +|---|---|---|---| +| ripgrep plain-language (holdout, 12 q) | 0.208 | 0.583 | 0.667 | +| click plain-language (holdout, 12 q) | 0.750 | 0.958 | 0.792 | +| this repo plain-language (dev, 14 q) | 0.643 | 0.679 | 0.643 | +| SWE-bench dev-fast issues (59), Acc@3 | 0.492 | 0.373 | 0.458 | + +v1-turbo on ripgrep: 0.458 alone, 0.583 blend. Latency on issues: 8.0 s p50 / 14.5 s p90 for 10 +pairs on CPU. Verdict: a code-trained cross-encoder orders *short plain-language questions* better +than lexical ranking on all three sets, and hurts on *long issue reports* (truncated 512-token query +loses the report's specifics; the first stage already reads the whole report). The earlier v1 +rejection (§4) was a different experiment — it changed packet *content* (recall collapsed); this one +only reorders. Paper: the query-length split is a finding in itself. diff --git a/scripts/research/dense_loc.py b/scripts/research/dense_loc.py new file mode 100644 index 0000000..a88a546 --- /dev/null +++ b/scripts/research/dense_loc.py @@ -0,0 +1,247 @@ +"""Research harness: file-level localisation on SWE-bench with lexical, dense and +fused rankers, scores kept so fusion can be studied offline. + + py -3 scripts/research/dense_loc.py --data rows.json --repos --out run.jsonl + [--only repo1,repo2] [--single-gold] [--shard i/n] [--model jina-code-v2] + +Per instance: check out `base_commit` (git worktree), split every .py file into +chunks (module head, each function, each method; Python `ast`, line windows +when a file does not parse), embed the chunks with a code embedding model (ONNX, +mean pooling), and rank files by their best chunk against the issue text. +Embeddings are cached per chunk text across commits (`--cache`), so one repository +is embedded once and later commits only embed what changed. + +Each output line keeps the top-100 files with scores for: bm25 (file text + +path, as `compare_baselines.py`), dense (max chunk cosine), so fusions can be +computed without re-running. Gold = files the reference patch edits. +""" +import argparse +import ast +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time + +import numpy as np + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from compare_baselines import Bm25 # noqa: E402 + +MODELS = os.path.join(os.environ.get("LOCALAPPDATA", ""), "neuromesh", "models") + + +def gold_files(patch): + return sorted(set(re.findall(r"^diff --git a/(\S+)", patch, re.M))) + + +def checkout(clone, commit, dest): + if os.path.isdir(dest): + subprocess.run(["git", "-C", clone, "worktree", "remove", "--force", dest], capture_output=True) + shutil.rmtree(dest, ignore_errors=True) + subprocess.run(["git", "-C", clone, "worktree", "prune"], capture_output=True) + r = subprocess.run(["git", "-C", clone, "worktree", "add", "--detach", dest, commit], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(r.stderr.strip()[-300:]) + + +def py_files(root): + r = subprocess.run(["git", "-C", root, "ls-files", "*.py"], capture_output=True, text=True) + return [f for f in r.stdout.splitlines() if f] + + +def chunks_of(path, text, max_chars=1500): + """(label, text) chunks of one Python file.""" + lines = text.splitlines() + out = [] + head = "\n".join(lines[:40]) + out.append(("", f"{path}\n{head}"[:max_chars])) + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + for i in range(40, len(lines), 60): + out.append((f"L{i}", f"{path}\n" + "\n".join(lines[i:i + 60])[:max_chars])) + return out + + def seg(node): + end = getattr(node, "end_lineno", node.lineno) + return "\n".join(lines[node.lineno - 1:end]) + + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + out.append((node.name, f"{path}\n{seg(node)}"[:max_chars])) + elif isinstance(node, ast.ClassDef): + body = [n for n in node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))] + head_end = body[0].lineno - 1 if body else getattr(node, "end_lineno", node.lineno) + out.append((node.name, f"{path}\n" + "\n".join(lines[node.lineno - 1:head_end])[:max_chars])) + for m in body: + out.append((f"{node.name}.{m.name}", f"{path}\nclass {node.name}:\n{seg(m)}"[:max_chars])) + return out + + +class Embedder: + def __init__(self, model, max_tokens, threads=0): + import onnxruntime as ort + from tokenizers import Tokenizer + + d = os.path.join(MODELS, model) + so = ort.SessionOptions() + if threads: + so.intra_op_num_threads = threads + self.sess = ort.InferenceSession(os.path.join(d, "model_quantized.onnx"), so, + providers=["CPUExecutionProvider"]) + self.tok = Tokenizer.from_file(os.path.join(d, "tokenizer.json")) + self.max_tokens = max_tokens + + def embed(self, texts, max_tokens=None, batch=16): + max_tokens = max_tokens or self.max_tokens + enc = [self.tok.encode(t) for t in texts] + order = sorted(range(len(texts)), key=lambda i: len(enc[i].ids)) + out = np.zeros((len(texts), 768), dtype=np.float32) + for s in range(0, len(order), batch): + idx = order[s:s + batch] + ids = [enc[i].ids[:max_tokens] for i in idx] + n = max(len(x) for x in ids) + arr = np.zeros((len(ids), n), dtype=np.int64) + mask = np.zeros((len(ids), n), dtype=np.int64) + for j, x in enumerate(ids): + arr[j, :len(x)] = x + mask[j, :len(x)] = 1 + h = self.sess.run(None, {"input_ids": arr, "attention_mask": mask})[0] + m = mask[..., None].astype(np.float32) + v = (h * m).sum(1) / np.maximum(m.sum(1), 1) + v /= np.maximum(np.linalg.norm(v, axis=1, keepdims=True), 1e-9) + out[idx] = v + return out + + +class Cache: + def __init__(self, path): + self.path = path + self.vec = {} + if os.path.exists(path): + z = np.load(path, allow_pickle=False) + for k, v in zip(z["keys"], z["vecs"]): + self.vec[str(k)] = v + self.dirty = False + + def save(self): + if not self.dirty: + return + keys = np.array(list(self.vec.keys())) + vecs = np.stack(list(self.vec.values())).astype(np.float16) + tmp = self.path + ".tmp.npz" + np.savez(tmp, keys=keys, vecs=vecs) + os.replace(tmp, self.path) + self.dirty = False + + +def key(text): + return hashlib.sha1(text.encode("utf-8", "ignore")).hexdigest() + + +def top(scores, n=100): + items = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))[:n] + return [[f, round(float(s), 5)] for f, s in items] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--only", default="") + ap.add_argument("--shard", default="") + ap.add_argument("--single-gold", action="store_true") + ap.add_argument("--model", default="jina-code-v2") + ap.add_argument("--max-tokens", type=int, default=256) + ap.add_argument("--query-tokens", type=int, default=1024) + ap.add_argument("--threads", type=int, default=0) + ap.add_argument("--cache", default="") + ap.add_argument("--work", default="") + args = ap.parse_args() + + rows = json.load(open(args.data, encoding="utf-8")) + if args.only: + keep = set(args.only.split(",")) + rows = [r for r in rows if r["repo"] in keep] + if args.single_gold: + rows = [r for r in rows if len(gold_files(r["patch"])) == 1] + done = set() + if os.path.exists(args.out): + done = {json.loads(l)["instance_id"] for l in open(args.out, encoding="utf-8") if l.strip()} + todo = [r for r in rows if r["instance_id"] not in done] + if args.shard: + i, n = (int(x) for x in args.shard.split("/")) + todo = todo[i::n] + emb = Embedder(args.model, args.max_tokens, args.threads) + cache_dir = args.cache or os.path.join(os.path.dirname(args.out) or ".", "emb-cache") + os.makedirs(cache_dir, exist_ok=True) + work = os.path.abspath(args.work or os.path.join(os.path.dirname(args.out) or ".", "wt-dense")) + caches = {} + with open(args.out, "a", encoding="utf-8") as out: + for n_done, row in enumerate(todo, 1): + name = row["repo"].split("/")[1] + t0 = time.time() + rec = {"instance_id": row["instance_id"], "repo": row["repo"], "gold": gold_files(row["patch"])} + try: + dest = os.path.join(work, name) + checkout(os.path.abspath(os.path.join(args.repos, name)), row["base_commit"], dest) + files = py_files(dest) + cache = caches.get(name) + if cache is None: + cache = caches[name] = Cache(os.path.join(cache_dir, f"{name}-{args.model}-{args.max_tokens}.npz")) + labels, keys, missing = [], [], {} + for f in files: + try: + text = open(os.path.join(dest, f), encoding="utf-8", errors="ignore").read() + except OSError: + continue + for label, ch in chunks_of(f, text): + k = key(ch) + labels.append((f, label)) + keys.append(k) + if k not in cache.vec: + missing[k] = ch + t_emb = time.time() + if missing: + ks = list(missing) + vecs = emb.embed([missing[k] for k in ks]) + for k, v in zip(ks, vecs): + cache.vec[k] = v.astype(np.float16) + cache.dirty = True + rec["embedded"] = len(missing) + rec["chunks"] = len(keys) + rec["embed_s"] = round(time.time() - t_emb, 1) + q = emb.embed([row["problem_statement"]], max_tokens=args.query_tokens)[0] + mat = np.stack([cache.vec[k] for k in keys]).astype(np.float32) + sims = mat @ q + dense = {} + best_chunk = {} + for (f, label), s in zip(labels, sims): + if s > dense.get(f, -2): + dense[f] = float(s) + best_chunk[f] = label + rec["dense"] = top(dense) + rec["dense_chunk"] = {f: best_chunk[f] for f, _ in rec["dense"][:10]} + bm = Bm25(dest, files) + ranked = bm.rank(row["problem_statement"]) + rec["bm25"] = [[f, None] for f in ranked[:100]] + cache.save() + except Exception as e: # recorded, not fatal + rec["error"] = str(e)[-300:] + rec["secs"] = round(time.time() - t0, 1) + out.write(json.dumps(rec) + "\n") + out.flush() + g = rec["gold"] + pos = next((i for i, (f, _) in enumerate(rec.get("dense", [])) if f in g), None) + print(f"[{n_done}/{len(todo)}] {rec['instance_id']} dense_pos={pos} chunks={rec.get('chunks')} " + f"new={rec.get('embedded')} {rec['secs']}s {rec.get('error', '')[:80]}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/research/eval_loc.py b/scripts/research/eval_loc.py new file mode 100644 index 0000000..6186a9b --- /dev/null +++ b/scripts/research/eval_loc.py @@ -0,0 +1,108 @@ +"""File-level Acc@k (all gold files in the top k, as LocAgent reports) for ranked +lists stored in result jsonl files, on the instances every given run covers. + + py -3 scripts/research/eval_loc.py RUN.jsonl:key[,key...] [RUN2.jsonl:key ...] + [--subset rows.json] [--single-gold] [--k 1,3,5,10] [--fuse a+b] + +A key names a list field (`loc_files`, `ours_files`, `chunk`, `file`, ...) or a +precomputed hit field prefix (`bm25_hit`, `ours_hit`, `loc_hit`). `--fuse` +adds reciprocal-rank fusion of two list keys (k = 60). +""" +import argparse +import json +import random +from collections import defaultdict + + +def load(path): + out = {} + for line in open(path, encoding="utf-8"): + if line.strip(): + r = json.loads(line) + if "error" not in r: + out[r["instance_id"]] = r + return out + + +def norm(f): + return f.replace("\\", "/") + + +def acc(lst, gold, k): + top = [norm(f) for f in lst[:k]] + return int(all(any(t.endswith(g) or g.endswith(t) for t in top) for g in gold)) + + +def rrf(a, b, k=60): + s = defaultdict(float) + for lst in (a, b): + for i, f in enumerate(lst): + s[norm(f)] += 1 / (k + i + 1) + return [f for f, _ in sorted(s.items(), key=lambda kv: (-kv[1], kv[0]))] + + +def boot_ci(vals, n=2000, seed=0): + rnd = random.Random(seed) + m = len(vals) + means = sorted(sum(rnd.choice(vals) for _ in range(m)) / m for _ in range(n)) + return means[int(0.025 * n)], means[int(0.975 * n)] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("runs", nargs="+") + ap.add_argument("--subset", default="") + ap.add_argument("--single-gold", action="store_true") + ap.add_argument("--k", default="1,3,5,10") + ap.add_argument("--fuse", action="append", default=[]) + ap.add_argument("--ci", action="store_true") + args = ap.parse_args() + ks = [int(x) for x in args.k.split(",")] + series = [] # (label, {iid: list or hits}) + for spec in args.runs: + path, keys = spec.rsplit(":", 1) + data = load(path) + for key in keys.split(","): + series.append((f"{path.split('/')[-1]}:{key}", key, data)) + ids = None + for _, key, data in series: + have = {i for i, r in data.items() if isinstance(r.get(key), list) or f"{key}@1" in r} + ids = have if ids is None else ids & have + if args.subset: + ids &= {r["instance_id"] for r in json.load(open(args.subset, encoding="utf-8"))} + any_data = series[0][2] + if args.single_gold: + ids = {i for i in ids if len(any_data[i]["gold"]) == 1} + ids = sorted(ids) + print(f"instances: {len(ids)}") + rows = [] + for label, key, data in series: + vals = {} + for k in ks: + if ids and isinstance(data[ids[0]].get(key), list): + v = [acc(data[i][key], data[i]["gold"], k) for i in ids] + else: + v = [data[i].get(f"{key}@{k}", 0) for i in ids] + vals[k] = v + rows.append((label, vals)) + for fuse in args.fuse: + a, b = fuse.split("+") + la = next(s for s in series if s[1] == a) + lb = next(s for s in series if s[1] == b) + vals = {k: [acc(rrf(la[2][i][a], lb[2][i][b]), la[2][i]["gold"], k) for i in ids] for k in ks} + rows.append((f"rrf({a},{b})", vals)) + for label, vals in rows: + cells = [] + for k in ks: + v = vals[k] + m = sum(v) / max(len(v), 1) + if args.ci and v: + lo, hi = boot_ci(v) + cells.append(f"@{k} {m:.3f} [{lo:.2f},{hi:.2f}]") + else: + cells.append(f"@{k} {m:.3f}") + print(f"{label:40s} " + " ".join(cells)) + + +if __name__ == "__main__": + main() diff --git a/scripts/research/lex_variants.py b/scripts/research/lex_variants.py new file mode 100644 index 0000000..14db9c7 --- /dev/null +++ b/scripts/research/lex_variants.py @@ -0,0 +1,119 @@ +"""Cheap lexical variants for issue → file localisation, positions kept per gold +file so variants can be compared and fused offline. + + file BM25 over whole files (body + path words) + chunk BM25 over functions/classes/module heads (Python `ast`), a file + scores its best chunk (LocAgent indexes entity contents this way) + *_title same, with the issue's first line counted three times + + py -3 scripts/research/lex_variants.py --data rows.json --repos --out lex.jsonl --work +""" +import argparse +import json +import math +import os +import sys +from collections import Counter, defaultdict + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from compare_baselines import repo_files, words # noqa: E402 +from swebench_localize import checkout, gold_files # noqa: E402 +from dense_loc import chunks_of # noqa: E402 + +K1, B = 1.2, 0.75 + + +class Bm25: + def __init__(self, docs): + self.docs = docs # list of Counter + self.lens = [sum(d.values()) for d in docs] + self.avg = sum(self.lens) / max(len(docs), 1) + self.df = Counter() + for d in docs: + self.df.update(d.keys()) + self.n = len(docs) + self.post = defaultdict(list) + for i, d in enumerate(docs): + for w, c in d.items(): + self.post[w].append((i, c)) + + def scores(self, q): + out = defaultdict(float) + for w, qc in q.items(): + if w not in self.post: + continue + idf = math.log((self.n - self.df[w] + 0.5) / (self.df[w] + 0.5) + 1) + for i, t in self.post[w]: + out[i] += qc * idf * t * (K1 + 1) / (t + K1 * (1 - B + B * self.lens[i] / max(self.avg, 1))) + return out + + +def ranked_files(scores, owners): + best = {} + for i, s in scores.items(): + f = owners[i] + if s > best.get(f, 0): + best[f] = s + return [f for f, _ in sorted(best.items(), key=lambda kv: (-kv[1], kv[0]))] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--work", required=True) + args = ap.parse_args() + rows = json.load(open(args.data, encoding="utf-8")) + done = set() + if os.path.exists(args.out): + done = {json.loads(l)["instance_id"] for l in open(args.out, encoding="utf-8") if l.strip()} + with open(args.out, "a", encoding="utf-8") as out: + for row in rows: + if row["instance_id"] in done: + continue + name = row["repo"].split("/")[1] + dest = os.path.join(args.work, name) + gold = gold_files(row["patch"]) + rec = {"instance_id": row["instance_id"], "repo": row["repo"], "gold": gold} + try: + checkout(os.path.join(args.repos, name), row["base_commit"], dest) + files = repo_files(dest) + fdocs, cdocs, owners = [], [], [] + for f in files: + try: + text = open(os.path.join(dest, f), encoding="utf-8", errors="ignore").read() + except OSError: + text = "" + path_words = Counter(words(f.rsplit(".", 1)[0])) + fdocs.append(Counter(words(text)) + path_words) + if f.endswith(".py"): + for _, ch in chunks_of(f, text, max_chars=20000): + cdocs.append(Counter(words(ch)) + path_words) + owners.append(f) + else: + cdocs.append(fdocs[-1]) + owners.append(f) + fidx, cidx = Bm25(fdocs), Bm25(cdocs) + text = row["problem_statement"] + title = text.strip().splitlines()[0] if text.strip() else "" + q = Counter(set(words(text))) + qt = Counter(set(words(text))) + for w in set(words(title)): + qt[w] += 2 + lists = { + "file": ranked_files(fidx.scores(q), files), + "file_title": ranked_files(fidx.scores(qt), files), + "chunk": ranked_files(cidx.scores(q), owners), + "chunk_title": ranked_files(cidx.scores(qt), owners), + } + for k, lst in lists.items(): + rec[k] = lst[:50] + except Exception as e: # recorded, not fatal + rec["error"] = str(e)[-300:] + out.write(json.dumps(rec) + "\n") + out.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/research/lexical_ablation.py b/scripts/research/lexical_ablation.py new file mode 100644 index 0000000..a9f0edf --- /dev/null +++ b/scripts/research/lexical_ablation.py @@ -0,0 +1,115 @@ +"""Lexical ablation on SWE-bench-style issues: which parts of BM25 matter for +long bug reports? Variants share one tokenizer (`compare_baselines.words`): + + tf classic BM25 (k1 1.2, b 0.75), body + path words, query as a set + binary term present/absent, length = distinct words (what our body field does) + binary4 binary, words of 4+ letters only (our body field's vocabulary) + qtf tf, and a query word counts as often as the report repeats it + path3 tf, path words weighted x3 as a separate field (BM25F-style) + + py -3 scripts/research/lexical_ablation.py --data rows.json --repos --out abl.jsonl +""" +import argparse +import json +import math +import os +import sys +from collections import Counter + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from compare_baselines import repo_files, words # noqa: E402 +from swebench_localize import checkout, gold_files, hit # noqa: E402 + +K1, B = 1.2, 0.75 + + +class Index: + def __init__(self, root, files): + self.files = files + self.body = [] + self.path = [] + for f in files: + try: + text = open(os.path.join(root, f), encoding="utf-8", errors="ignore").read() + except OSError: + text = "" + self.body.append(Counter(words(text))) + self.path.append(Counter(words(f.rsplit(".", 1)[0]))) + self.n = len(files) + + def rank(self, query, variant): + qwords = words(query) + q = Counter(qwords) if variant == "qtf" else Counter(set(qwords)) + docs = [] + for body, path in zip(self.body, self.path): + if variant in ("binary", "binary4"): + d = {w: 1 for w in body if variant == "binary" or len(w) >= 4} + for w in path: + d[w] = 1 + elif variant == "path3": + d = dict(body) + for w, c in path.items(): + d[w] = d.get(w, 0) + 3 * c + else: + d = dict(body) + for w, c in path.items(): + d[w] = d.get(w, 0) + c + docs.append(d) + df = Counter() + for d in docs: + df.update(d.keys()) + lens = [sum(d.values()) if variant not in ("binary", "binary4") else len(d) for d in docs] + avg = sum(lens) / max(len(lens), 1) + scores = [] + for f, d, ln in zip(self.files, docs, lens): + s = 0.0 + for w, qc in q.items(): + t = d.get(w, 0) + if t: + idf = math.log((self.n - df[w] + 0.5) / (df[w] + 0.5) + 1) + s += qc * idf * t * (K1 + 1) / (t + K1 * (1 - B + B * ln / max(avg, 1))) + if s > 0: + scores.append((-s, f)) + scores.sort() + return [f for _, f in scores] + + +VARIANTS = ["tf", "binary", "binary4", "qtf", "path3"] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--only", default="") + ap.add_argument("--work", required=True) + args = ap.parse_args() + rows = json.load(open(args.data, encoding="utf-8")) + if args.only: + rows = [r for r in rows if r["repo"] in set(args.only.split(","))] + done = set() + if os.path.exists(args.out): + done = {json.loads(l)["instance_id"] for l in open(args.out, encoding="utf-8") if l.strip()} + with open(args.out, "a", encoding="utf-8") as out: + for row in rows: + if row["instance_id"] in done: + continue + name = row["repo"].split("/")[1] + dest = os.path.join(args.work, name) + gold = gold_files(row["patch"]) + rec = {"instance_id": row["instance_id"], "repo": row["repo"], "gold": gold} + try: + checkout(os.path.join(args.repos, name), row["base_commit"], dest) + idx = Index(dest, repo_files(dest)) + for v in VARIANTS: + ranked = idx.rank(row["problem_statement"], v) + rec[v] = {f"@{k}": hit(ranked, gold, k) for k in (1, 3, 5, 10)} + except Exception as e: # recorded, not fatal + rec["error"] = str(e)[-300:] + out.write(json.dumps(rec) + "\n") + out.flush() + + +if __name__ == "__main__": + main() diff --git a/scripts/research/rerank_swe.py b/scripts/research/rerank_swe.py new file mode 100644 index 0000000..ebd69e6 --- /dev/null +++ b/scripts/research/rerank_swe.py @@ -0,0 +1,90 @@ +"""Cross-encoder rerank of the engine's localisation list on SWE-bench issues. + +Reads a harness result file (needs `loc_files`), checks out each instance, +reranks the top N candidates with the issue text (boilerplate stripped +the engine's way: comments, links, checklists, headings) and writes the new +order as `rr` (pure reranker) and `rrb` (50/50 blend with first-stage rank). + + py -3 scripts/research/rerank_swe.py --results run.jsonl --data rows.json --repos + --work --out rr.jsonl [--model jina-reranker-v2] [--top 10] +""" +import argparse +import json +import os +import re +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, os.path.dirname(__file__)) +from swebench_localize import checkout, remove_worktrees # noqa: E402 +from rerank_test import Reranker, digest # noqa: E402 + + +def strip_boilerplate(text): + text = re.sub(r"(?s)", " ", text) + text = re.sub(r"https?://\S+", " ", text) + out, fenced = [], False + for line in text.splitlines(): + t = line.lstrip() + if t.startswith("```"): + fenced = not fenced + out.append(line) + continue + if re.match(r"[-*+] \[[ xX]\]", t) or (not fenced and re.match(r"#+ ", t)): + continue + out.append(line) + return "\n".join(out) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True) + ap.add_argument("--data", required=True) + ap.add_argument("--repos", required=True) + ap.add_argument("--work", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--model", default="jina-reranker-v2") + ap.add_argument("--top", type=int, default=10) + ap.add_argument("--subset", default="") + args = ap.parse_args() + rows = {r["instance_id"]: r for r in json.load(open(args.data, encoding="utf-8"))} + keep = None + if args.subset: + keep = {r["instance_id"] for r in json.load(open(args.subset, encoding="utf-8"))} + res = [json.loads(l) for l in open(args.results, encoding="utf-8") if l.strip()] + res = [r for r in res if "loc_files" in r and (keep is None or r["instance_id"] in keep)] + rr = Reranker(args.model) + done = set() + if os.path.exists(args.out): + done = {json.loads(l)["instance_id"] for l in open(args.out, encoding="utf-8")} + try: + with open(args.out, "a", encoding="utf-8") as out: + for r in res: + if r["instance_id"] in done: + continue + row = rows[r["instance_id"]] + name = row["repo"].split("/")[1] + dest = os.path.join(args.work, name) + checkout(os.path.join(args.repos, name), row["base_commit"], dest) + cands = r["loc_files"][: args.top] + q = strip_boilerplate(row["problem_statement"]) + t0 = time.time() + s = rr.scores(q, [digest(dest, c) for c in cands]) + secs = time.time() - t0 + lo, hi = min(s), max(s) + span = (hi - lo) or 1.0 + order = [c for _, c in sorted(zip(s, cands), key=lambda x: -x[0])] + blend = {c: 0.5 * (si - lo) / span + 0.5 / (1 + i) for i, (c, si) in enumerate(zip(cands, s))} + rec = {"instance_id": r["instance_id"], "repo": r["repo"], "gold": r["gold"], + "loc": r["loc_files"], "rr": order + r["loc_files"][args.top:], + "rrb": sorted(cands, key=lambda c: -blend[c]) + r["loc_files"][args.top:], + "rr_secs": round(secs, 2)} + out.write(json.dumps(rec) + "\n") + out.flush() + finally: + remove_worktrees(args.repos, args.work) + + +if __name__ == "__main__": + main() diff --git a/scripts/research/rerank_test.py b/scripts/research/rerank_test.py new file mode 100644 index 0000000..0737306 --- /dev/null +++ b/scripts/research/rerank_test.py @@ -0,0 +1,114 @@ +"""Does a cross-encoder reorder the engine's candidates better? (F92 follow-up) + +For every task of a gold set: the engine's localisation list (top 10) is the +first stage; a cross-encoder (ONNX under %LOCALAPPDATA%/neuromesh/models/) +scores (question, file digest) pairs; the final order blends both 50/50 (min-max +reranker score + reciprocal first-stage rank), as the rejected v1-turbo WIP did. +Reports R@3 / P@3 for first stage, reranker alone, and the blend. + + py -3 scripts/research/rerank_test.py --gold tests/.../gold_tasks.toml --repo + --bin --model jina-reranker-v2 +""" +import argparse +import json +import os +import re +import shutil +import subprocess +import tempfile +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib + +import numpy as np + +MODELS = os.path.join(os.environ.get("LOCALAPPDATA", ""), "neuromesh", "models") +SIG = re.compile(r"^\s*(pub |async |def |class |fn |struct |enum |trait |impl |type |interface |export |///|//!|#\s|\"\"\")") + + +def digest(root, path, max_chars=1800): + try: + text = open(os.path.join(root, path), encoding="utf-8", errors="ignore").read() + except OSError: + return path + keep = [l.strip() for l in text.splitlines() if SIG.match(l)] + return (path + "\n" + "\n".join(keep))[:max_chars] + + +class Reranker: + def __init__(self, model, max_len=512): + import onnxruntime as ort + from tokenizers import Tokenizer + + d = os.path.join(MODELS, model) + self.sess = ort.InferenceSession(os.path.join(d, "model_quantized.onnx"), + providers=["CPUExecutionProvider"]) + self.tok = Tokenizer.from_file(os.path.join(d, "tokenizer.json")) + self.tok.enable_truncation(max_len) + self.names = {i.name for i in self.sess.get_inputs()} + + def scores(self, query, docs): + out = [] + for doc in docs: + e = self.tok.encode(query, doc) + feed = {"input_ids": np.array([e.ids], dtype=np.int64), + "attention_mask": np.array([e.attention_mask], dtype=np.int64)} + if "token_type_ids" in self.names: + feed["token_type_ids"] = np.array([e.type_ids], dtype=np.int64) + out.append(float(self.sess.run(None, feed)[0].reshape(-1)[0])) + return out + + +def packet(binary, repo, prompt): + home = tempfile.mkdtemp(prefix="nmhome-") + try: + r = subprocess.run([binary, "packet", "--json", "--query", prompt], cwd=repo, + env=dict(os.environ, NEUROMESH_HOME=home, NEUROMESH_NO_BROWSER="1"), + capture_output=True, text=True, timeout=900) + out = r.stdout + return json.loads(out[out.find("{"):]) + finally: + shutil.rmtree(home, ignore_errors=True) + + +def at3(order, gold): + top = order[:3] + hit = sum(1 for g in gold if g in top) + return hit / len(gold), hit / max(len(top), 1) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--gold", required=True) + ap.add_argument("--repo", required=True) + ap.add_argument("--bin", required=True) + ap.add_argument("--model", default="jina-reranker-v2") + args = ap.parse_args() + tasks = tomllib.load(open(args.gold, "rb"))["task"] + rr = Reranker(args.model) + agg = {"first": [0, 0], "rerank": [0, 0], "blend": [0, 0]} + for t in tasks: + pkt = packet(args.bin, args.repo, t["prompt"]) + cands = (pkt.get("localization") or pkt.get("selected_paths") or [])[:10] + if not cands: + continue + s = rr.scores(t["prompt"], [digest(args.repo, c) for c in cands]) + lo, hi = min(s), max(s) + span = (hi - lo) or 1.0 + rer = [c for _, c in sorted(zip(s, cands), key=lambda x: -x[0])] + blend_score = {c: 0.5 * (si - lo) / span + 0.5 * (1.0 / (1 + i)) for i, (c, si) in enumerate(zip(cands, s))} + blend = sorted(cands, key=lambda c: -blend_score[c]) + gold = t["gold_files"] + for name, order in (("first", cands), ("rerank", rer), ("blend", blend)): + r, p = at3(order, gold) + agg[name][0] += r + agg[name][1] += p + print(t["id"], "first", [c.split('/')[-1] for c in cands[:3]], "blend", [c.split('/')[-1] for c in blend[:3]]) + n = len(tasks) + for name, (r, p) in agg.items(): + print(f"{name:7s} R@3 {r / n:.3f} P@3 {p / n:.3f}") + + +if __name__ == "__main__": + main() From 44cd0d14f7dfa41dd3676ebb89f7c1e170d2038f Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Fri, 9 Oct 2026 01:41:02 +0330 Subject: [PATCH 2/2] fmt Co-Authored-By: Claude Opus 5.5 --- crates/neuromesh-parser/src/text_normalize.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/neuromesh-parser/src/text_normalize.rs b/crates/neuromesh-parser/src/text_normalize.rs index f130484..6a73b04 100644 --- a/crates/neuromesh-parser/src/text_normalize.rs +++ b/crates/neuromesh-parser/src/text_normalize.rs @@ -109,7 +109,10 @@ mod tests { assert!(!out.contains("Search before asking")); assert!(!out.contains("similar issues")); assert!(!out.contains("describe the bug")); - assert_eq!(strip_issue_boilerplate("### short question"), "### short question"); + assert_eq!( + strip_issue_boilerplate("### short question"), + "### short question" + ); } #[test]