Issue mode: definition-level ranking + report hygiene (SWE dev Acc@5 0.456 → 0.596)

crates/neuromesh-cli/src/commands/packet.rs

@@ -41,6 +41,9 @@ struct PacketJsonOut {
     /// Repository-relative paths, best first (highest activation of any
     /// node in the file): what an evaluation needs for hit@k.
     selected_paths: Vec<String>,
+    /// Packet files, then the next source files of the whole-question ranking
+    /// (20 deep): where to look, best first.
+    localization: Vec<String>,
     /// The whole-question file ranking (BM25F), top 10, for evaluation.
     ranked_paths: Vec<String>,
     identifiers: Vec<String>,
@@ -96,6 +99,7 @@ pub fn execute(args: &[String]) -> Result<()> {
     neuromesh_graph::timing("activate_tiered", started);
 
     let selected_paths = neuromesh_context::gold::packet_file_order(&graph, &prompt, &view);
+    let localization = neuromesh_context::gold::localization_order(&graph, &prompt, &view, 20);
     let mut files: Vec<String> = packet_file_names(&view).into_iter().collect();
     files.sort();
     let reduction = if workspace_tokens > 0 {
@@ -125,6 +129,7 @@ pub fn execute(args: &[String]) -> Result<()> {
         selected_files_count: files.len(),
         selected_files: files.clone(),
         selected_paths,
+        localization,
         ranked_paths: graph
             .file_rank(&prompt, 10)
             .into_iter()

crates/neuromesh-context/src/gold.rs

@@ -1045,6 +1045,67 @@ pub fn packet_file_order(
     fused.into_iter().map(|(p, _)| p).collect()
 }
 
+/// Where to look, best first, `k` deep: the packet's files in
+/// [`packet_file_order`], then the whole-question ranking's next source files
+/// (tests, docs and examples left out). A packet is 1–5 files; a localisation
+/// list ("which file does this issue need changed?", file-level Acc@k as
+/// SWE-bench localisation papers report it) needs more candidates than the
+/// packet ships, and the runners-up are the files an agent should open next.
+pub fn localization_order(
+    graph: &neuromesh_graph::NeuralProjectGraph,
+    prompt: &str,
+    view: &ContextView,
+    k: usize,
+) -> Vec<String> {
+    // A packet may carry a test, a fixture or a doc page as context; where to
+    // change code is a source file, so those go last.
+    let (mut out, aside): (Vec<String>, Vec<String>) = packet_file_order(graph, prompt, view)
+        .into_iter()
+        .partition(|p| !crate::selector::is_noise_path(std::path::Path::new(p)));
+    let mut seen: std::collections::HashSet<String> =
+        out.iter().chain(aside.iter()).cloned().collect();
+    for r in graph.file_rank(prompt, 4 * k.max(10)) {
+        if out.len() >= k {
+            break;
+        }
+        if crate::selector::is_noise_path(&r.path) {
+            continue;
+        }
+        let p = r.path.to_string_lossy().replace('\\', "/");
+        if seen.insert(p.clone()) {
+            out.push(p);
+        }
+    }
+    // A long report also votes definition by definition: reciprocal-rank
+    // fusion (k = 60, equal weight) of this list with the definition-level
+    // ranking (SWE-bench dev-fast, 57 issues: Acc@1 0.211 → 0.298, @5 0.544
+    // → 0.614 in the prototype).
+    if prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS {
+        const K: f32 = 60.0;
+        let by_def: Vec<String> = graph
+            .chunk_rank(prompt, 4 * k.max(10))
+            .into_iter()
+            .filter(|r| !crate::selector::is_noise_path(&r.path))
+            .map(|r| r.path.to_string_lossy().replace('\\', "/"))
+            .collect();
+        let mut fused: std::collections::HashMap<String, f32> = std::collections::HashMap::new();
+        for list in [&out, &by_def] {
+            for (i, p) in list.iter().enumerate() {
+                *fused.entry(p.clone()).or_insert(0.0) += 1.0 / (K + i as f32 + 1.0);
+            }
+        }
+        let mut order: Vec<(String, f32)> = fused.into_iter().collect();
+        order.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
+        out = order
+            .into_iter()
+            .map(|(p, _)| p)
+            .take(k.max(out.len()))
+            .collect();
+    }
+    out.extend(aside);
+    out
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/neuromesh-graph/src/chunk_rank.rs

@@ -0,0 +1,218 @@
+//! Definition-level BM25 for long reports (issues, bug templates).
+//!
+//! A whole-file ranker dilutes a report's words over every definition of a
+//! big module; read definition by definition, the function the report is
+//! about scores on its own (LocAgent indexes entity contents the same way).
+//! Chunks are the file head (first 40 lines) and every definition the graph
+//! recorded with a line span, each carrying its file's path words; a file
+//! scores its best chunk. The report's first line — an issue's title — counts
+//! three times.
+//!
+//! SWE-bench dev (`dev-fast`, 57 issues): fused by reciprocal rank with the
+//! packet's localisation list, Acc@1 0.211 → 0.298, @5 0.544 → 0.614, @10
+//! 0.667 → 0.702 (prototype, `scripts/research/lex_variants.py`).
+//!
+//! Built lazily from the sources on disk on the first long report, cached
+//! with the file ranker's key.
+
+use crate::file_rank::{stemmer, terms_of, RankedFile};
+use neuromesh_core::NodeId;
+use rayon::prelude::*;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+const K1: f32 = 1.2;
+const B: f32 = 0.75;
+const HEAD_LINES: usize = 40;
+const MAX_CHUNK_LINES: usize = 400;
+const TITLE_EXTRA: f32 = 2.0;
+
+/// One file as the index needs it: id, path, source, definition spans
+/// (1-based lines, end inclusive).
+pub(crate) struct ChunkSource {
+    pub id: NodeId,
+    pub path: PathBuf,
+    pub source: String,
+    pub spans: Vec<std::ops::Range<usize>>,
+}
+
+/// A file's chunks as term counts, before they are numbered.
+type FileChunks = (NodeId, PathBuf, Vec<HashMap<String, u16>>);
+
+#[derive(Default)]
+pub(crate) struct ChunkRankIndex {
+    files: Vec<(NodeId, PathBuf)>,
+    file_of: Vec<u32>,
+    len: Vec<u32>,
+    avg_len: f32,
+    post: HashMap<String, Vec<(u32, u16)>>,
+}
+
+impl ChunkRankIndex {
+    pub(crate) fn build(sources: Vec<ChunkSource>) -> Self {
+        // Per file: its chunks' term counts, computed in parallel.
+        let per_file: Vec<FileChunks> = sources
+            .into_par_iter()
+            .map(|f| {
+                let st = stemmer();
+                let rel = f.path.to_string_lossy().replace('\\', "/");
+                let stem_path = match rel.rsplit_once('.') {
+                    Some((head, ext)) if !ext.contains('/') => head.to_string(),
+                    _ => rel.clone(),
+                };
+                let path_terms = terms_of(&stem_path, &st, true);
+                let lines: Vec<&str> = f.source.lines().collect();
+                let head = 1..HEAD_LINES.min(lines.len());
+                let mut chunks = Vec::new();
+                for span in std::iter::once(head).chain(f.spans.iter().cloned()) {
+                    let start = span.start.max(1) - 1;
+                    let end = span.end.min(lines.len()).min(start + MAX_CHUNK_LINES);
+                    if start >= end {
+                        continue;
+                    }
+                    let mut counts: HashMap<String, u16> = HashMap::new();
+                    for t in terms_of(&lines[start..end].join("\n"), &st, false)
+                        .into_iter()
+                        .chain(path_terms.iter().cloned())
+                    {
+                        let c = counts.entry(t).or_insert(0);
+                        *c = c.saturating_add(1);
+                    }
+                    if !counts.is_empty() {
+                        chunks.push(counts);
+                    }
+                }
+                (f.id, f.path, chunks)
+            })
+            .collect();
+        let mut idx = ChunkRankIndex::default();
+        for (id, path, chunks) in per_file {
+            let file = idx.files.len() as u32;
+            idx.files.push((id, path));
+            for counts in chunks {
+                let chunk = idx.file_of.len() as u32;
+                idx.file_of.push(file);
+                idx.len.push(counts.values().map(|&c| c as u32).sum());
+                for (t, c) in counts {
+                    idx.post.entry(t).or_default().push((chunk, c));
+                }
+            }
+        }
+        let n = idx.len.len().max(1) as f32;
+        idx.avg_len = (idx.len.iter().map(|&l| l as f32).sum::<f32>() / n).max(1.0);
+        idx
+    }
+
+    /// Files by their best chunk for `prompt`, best first.
+    pub(crate) fn rank(&self, prompt: &str, limit: usize) -> Vec<RankedFile> {
+        let n = self.len.len() as f32;
+        if n == 0.0 {
+            return Vec::new();
+        }
+        let st = stemmer();
+        let body = neuromesh_parser::strip_issue_boilerplate(prompt);
+        let title = body
+            .lines()
+            .map(str::trim)
+            .find(|l| !l.is_empty())
+            .unwrap_or("");
+        let mut q: HashMap<String, f32> = HashMap::new();
+        for t in terms_of(&body, &st, false) {
+            q.insert(t, 1.0);
+        }
+        for t in terms_of(title, &st, false) {
+            *q.entry(t).or_insert(0.0) += TITLE_EXTRA;
+        }
+        let mut terms: Vec<(String, f32)> = q.into_iter().collect();
+        terms.sort_by(|a, b| a.0.cmp(&b.0));
+        terms.truncate(256);
+        let mut score: HashMap<u32, f32> = HashMap::new();
+        for (t, w) in &terms {
+            let Some(list) = self.post.get(t) else {
+                continue;
+            };
+            let df = list.len() as f32;
+            let idf = ((n - df + 0.5) / (df + 0.5) + 1.0).ln();
+            for &(chunk, tf) in list {
+                let tf = tf as f32;
+                let norm = 1.0 - B + B * self.len[chunk as usize] as f32 / self.avg_len;
+                *score.entry(chunk).or_insert(0.0) += w * idf * tf * (K1 + 1.0) / (tf + K1 * norm);
+            }
+        }
+        let mut best: HashMap<u32, f32> = HashMap::new();
+        for (chunk, s) in score {
+            let f = self.file_of[chunk as usize];
+            let e = best.entry(f).or_insert(0.0);
+            if s > *e {
+                *e = s;
+            }
+        }
+        let mut ranked: Vec<RankedFile> = best
+            .into_iter()
+            .map(|(f, s)| {
+                let (id, path) = &self.files[f as usize];
+                RankedFile {
+                    id: id.clone(),
+                    path: path.clone(),
+                    score: s,
+                    matched: 0,
+                }
+            })
+            .collect();
+        ranked.sort_by(|a, b| {
+            b.score
+                .partial_cmp(&a.score)
+                .unwrap_or(std::cmp::Ordering::Equal)
+                .then_with(|| a.path.cmp(&b.path))
+        });
+        ranked.truncate(limit);
+        ranked
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn src(id: &str, path: &str, source: &str, spans: Vec<std::ops::Range<usize>>) -> ChunkSource {
+        ChunkSource {
+            id: NodeId::new(id),
+            path: PathBuf::from(path),
+            source: source.to_string(),
+            spans,
+        }
+    }
+
+    #[test]
+    fn a_report_reaches_the_definition_it_describes_inside_a_big_file() {
+        // `big.py` spells the report's words once each across unrelated
+        // definitions; `small.py` has one definition that is all about them.
+        let mut big = String::new();
+        for i in 0..60 {
+            big.push_str(&format!("def helper_{i}():\n    return {i}\n"));
+        }
+        big.push_str("def parse_header():\n    timezone = 0\n");
+        let small = "def normalize_timezone_offset(header):\n    \
+                     # parse the timezone offset of a header\n    \
+                     return header.timezone.offset\n";
+        let index = ChunkRankIndex::build(vec![
+            src(
+                "big",
+                "pkg/big.py",
+                &big,
+                vec![std::ops::Range {
+                    start: 121,
+                    end: 122,
+                }],
+            ),
+            src(
+                "small",
+                "pkg/dates.py",
+                small,
+                vec![std::ops::Range { start: 1, end: 3 }],
+            ),
+        ]);
+        let ranked = index.rank("Timezone offset in header is parsed wrong", 5);
+        assert_eq!(ranked[0].path, PathBuf::from("pkg/dates.py"));
+    }
+}

crates/neuromesh-graph/src/file_rank.rs

@@ -214,18 +214,21 @@ pub(crate) struct FileRankIndex {
     avg_body_len: f32,
 }
 
-fn stemmer() -> Stemmer {
+pub(crate) fn stemmer() -> Stemmer {
     Stemmer::create(Algorithm::English)
 }
 
 /// Lowercase, split identifiers into words, drop short/non-alphabetic
 /// pieces and stopwords, stem.
-fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec<String> {
+pub(crate) fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec<String> {
     let mut out = Vec::new();
     for raw in text.split(|c: char| !c.is_ascii_alphanumeric() && c != '_') {
         if raw.is_empty() {
             continue;
         }
+        for code in raw.split('_').filter(|p| is_code_token(p)) {
+            out.push(code.to_ascii_lowercase());
+        }
         for part in neuromesh_parser::tokenize_ident(raw) {
             let w = part.to_lowercase();
             if w.len() < 2 || !w.chars().all(|c| c.is_ascii_alphabetic()) {
@@ -240,6 +243,19 @@ fn terms_of(text: &str, st: &Stemmer, keep_stopwords: bool) -> Vec<String> {
     out
 }
 
+/// `L031`, `LT02`, `E501`, `W0611`, `utf8`, `int64`: letters then digits, the
+/// way linters, encodings and widths are named. Word splitting drops the
+/// digits, which threw away the one term a report about rule L031 shares
+/// with `rules/L031.py` (SWE-bench dev, sqlfluff). Kept whole, unstemmed.
+fn is_code_token(s: &str) -> bool {
+    let letters = s.bytes().take_while(u8::is_ascii_alphabetic).count();
+    let digits = s.len() - letters;
+    (1..=4).contains(&letters)
+        && (1..=5).contains(&digits)
+        && s.len() >= 3
+        && s.bytes().skip(letters).all(|b| b.is_ascii_digit())
+}
+
 /// Stemmed content words of any text (a symbol name, a doc summary),
 /// stopwords dropped — comparable with [`query_terms`].
 pub fn text_terms(text: &str) -> Vec<String> {
@@ -263,6 +279,7 @@ const SYNONYM_WEIGHT: f32 = 0.35;
 /// [`SYNONYM_WEIGHT`]): a question word in a cluster also searches the
 /// rest of its cluster.
 pub fn weighted_query_terms(prompt: &str) -> Vec<(String, f32)> {
+    let prompt = &neuromesh_parser::strip_issue_boilerplate(prompt);
     let st = stemmer();
     let mut out: Vec<(String, f32)> = Vec::new();
     let mut seen = HashSet::new();