diff --git a/Cargo.lock b/Cargo.lock index 984a9b9..6f0f1f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1759,7 +1759,7 @@ dependencies = [ [[package]] name = "neuromesh-api" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "futures-util", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "neuromesh-cache" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "neuromesh-core", @@ -1800,7 +1800,7 @@ dependencies = [ [[package]] name = "neuromesh-cli" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "dirs", @@ -1830,7 +1830,7 @@ dependencies = [ [[package]] name = "neuromesh-context" -version = "1.3.0" +version = "1.4.0" dependencies = [ "blake3", "chrono", @@ -1849,7 +1849,7 @@ dependencies = [ [[package]] name = "neuromesh-core" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "dirs", @@ -1863,7 +1863,7 @@ dependencies = [ [[package]] name = "neuromesh-embed" -version = "1.3.0" +version = "1.4.0" dependencies = [ "dirs", "fastembed", @@ -1878,7 +1878,7 @@ dependencies = [ [[package]] name = "neuromesh-graph" -version = "1.3.0" +version = "1.4.0" dependencies = [ "bincode", "chrono", @@ -1900,7 +1900,7 @@ dependencies = [ [[package]] name = "neuromesh-graph-proxy" -version = "1.3.0" +version = "1.4.0" dependencies = [ "dirs", "neuromesh-core", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "neuromesh-index" -version = "1.3.0" +version = "1.4.0" dependencies = [ "blake3", "chrono", @@ -1931,7 +1931,7 @@ dependencies = [ [[package]] name = "neuromesh-local-ai" -version = "1.3.0" +version = "1.4.0" dependencies = [ "neuromesh-core", "parking_lot", @@ -1942,7 +1942,7 @@ dependencies = [ [[package]] name = "neuromesh-mcp" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "neuromesh-cache", @@ -1967,7 +1967,7 @@ dependencies = [ [[package]] name = "neuromesh-memory" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "dirs", @@ -1981,7 +1981,7 @@ dependencies = [ [[package]] name = "neuromesh-observability" -version = "1.3.0" +version = "1.4.0" dependencies = [ "chrono", "dirs", @@ -1996,7 +1996,7 @@ dependencies = [ [[package]] name = "neuromesh-parser" -version = "1.3.0" +version = "1.4.0" dependencies = [ "neuromesh-core", "neuromesh-index", @@ -2027,7 +2027,7 @@ dependencies = [ [[package]] name = "neuromesh-provider" -version = "1.3.0" +version = "1.4.0" dependencies = [ "futures-util", "neuromesh-core", @@ -2040,7 +2040,7 @@ dependencies = [ [[package]] name = "neuromesh-router" -version = "1.3.0" +version = "1.4.0" dependencies = [ "neuromesh-context", "neuromesh-core", @@ -2053,7 +2053,7 @@ dependencies = [ [[package]] name = "neuromesh-task" -version = "1.3.0" +version = "1.4.0" dependencies = [ "neuromesh-core", "neuromesh-parser", diff --git a/Cargo.toml b/Cargo.toml index a454700..64c39b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ exclude = ["tests/fixtures/mini-axum"] [workspace.package] -version = "1.3.0" +version = "1.4.0" edition = "2021" rust-version = "1.80" authors = ["NeuroMesh Team "] diff --git a/crates/neuromesh-cli/src/commands/packet.rs b/crates/neuromesh-cli/src/commands/packet.rs index f7698e5..6115740 100644 --- a/crates/neuromesh-cli/src/commands/packet.rs +++ b/crates/neuromesh-cli/src/commands/packet.rs @@ -48,6 +48,10 @@ struct PacketJsonOut { /// lines), 20 deep — function-level localisation. #[serde(skip_serializing_if = "Vec::is_empty")] definitions: Vec, + /// Research runs (`NM_DIAG=1`): the same list under other settings. + #[serde(skip_serializing_if = "Option::is_none")] + diag_definitions: + Option>>, /// The whole-question file ranking (BM25F), top 10, for evaluation. ranked_paths: Vec, identifiers: Vec, @@ -104,12 +108,11 @@ pub fn execute(args: &[String]) -> Result<()> { let selected_paths = neuromesh_context::gold::packet_file_order(&graph, &prompt, &view); let localization = neuromesh_context::gold::localization_order(&graph, &prompt, &view, 20); + // Every prompt gets the function list: a short issue still has a function + // to change (30 of LocAgent's 274 Lite instances are under 60 words and + // scored as misses without one). let definitions = - if prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS { - neuromesh_context::gold::definition_order(&graph, &prompt, &localization, def_depth()) - } else { - Vec::new() - }; + neuromesh_context::gold::definition_order(&graph, &prompt, &localization, def_depth()); let mut files: Vec = packet_file_names(&view).into_iter().collect(); files.sort(); let reduction = if workspace_tokens > 0 { @@ -139,6 +142,7 @@ pub fn execute(args: &[String]) -> Result<()> { selected_files_count: files.len(), selected_files: files.clone(), selected_paths, + diag_definitions: diag_definitions(&graph, &prompt, &localization), localization, definitions, ranked_paths: graph @@ -417,6 +421,79 @@ fn def_depth() -> usize { .unwrap_or(20) } +/// Definition lists under research settings, only when `NM_DIAG=1` (one run +/// measures several variants on the same checkouts). +fn diag_definitions( + graph: &neuromesh_graph::NeuralProjectGraph, + prompt: &str, + localization: &[String], +) -> Option>> { + if std::env::var("NM_DIAG").ok().as_deref() != Some("1") { + return None; + } + use neuromesh_context::gold::{definition_order_with, DefinitionOpts}; + use neuromesh_graph::ChunkOpts; + let base = DefinitionOpts::default(); + let variants = [ + ( + "raw", + DefinitionOpts { + file_prior: false, + traceback: false, + ..base + }, + ), + ( + "no_trace", + DefinitionOpts { + traceback: false, + ..base + }, + ), + ( + "no_stem", + DefinitionOpts { + chunk: ChunkOpts { + no_stem: true, + owner_terms: false, + }, + ..base + }, + ), + ( + "owner", + DefinitionOpts { + chunk: ChunkOpts { + no_stem: false, + owner_terms: true, + }, + ..base + }, + ), + ( + "no_stem_owner", + DefinitionOpts { + chunk: ChunkOpts { + no_stem: true, + owner_terms: true, + }, + ..base + }, + ), + ]; + Some( + variants + .into_iter() + .map(|(name, opts)| { + ( + name.to_string(), + definition_order_with(graph, prompt, localization, def_depth(), opts), + ) + }) + .collect(), + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/neuromesh-context/src/gold.rs b/crates/neuromesh-context/src/gold.rs index a584ca5..b0c3aee 100644 --- a/crates/neuromesh-context/src/gold.rs +++ b/crates/neuromesh-context/src/gold.rs @@ -1145,6 +1145,35 @@ pub fn definition_order( prompt: &str, files: &[String], k: usize, +) -> Vec { + definition_order_with(graph, prompt, files, k, DefinitionOpts::default()) +} + +/// Switches for research runs (`NM_DIAG`); the default is what ships. +#[derive(Debug, Clone, Copy)] +pub struct DefinitionOpts { + pub chunk: neuromesh_graph::ChunkOpts, + pub file_prior: bool, + pub traceback: bool, +} + +impl Default for DefinitionOpts { + fn default() -> Self { + Self { + chunk: neuromesh_graph::ChunkOpts::default(), + file_prior: true, + traceback: true, + } + } +} + +/// [`definition_order`] with research switches. +pub fn definition_order_with( + graph: &neuromesh_graph::NeuralProjectGraph, + prompt: &str, + files: &[String], + k: usize, + opts: DefinitionOpts, ) -> Vec { const K: f32 = 60.0; let file_rank: std::collections::HashMap<&str, usize> = files @@ -1152,8 +1181,13 @@ pub fn definition_order( .enumerate() .map(|(i, f)| (f.as_str(), i)) .collect(); + let file_rank = if opts.file_prior { + file_rank + } else { + std::collections::HashMap::new() + }; let mut scored: Vec<(f32, usize, neuromesh_graph::RankedDefinition)> = graph - .rank_definitions(prompt, 300) + .rank_definitions_with(prompt, 300, opts.chunk) .into_iter() .filter(|d| !crate::selector::is_noise_path(std::path::Path::new(&d.path))) .enumerate() @@ -1165,38 +1199,167 @@ pub fn definition_order( (s, i, d) }) .collect(); + // The functions a traceback runs through vote at double weight, deepest + // frame first, and enter the list even when the ranking missed them. + // SWE-bench dev, func Acc@5: dev-fast 0.212 → 0.250, dev-rest 0.253 → + // 0.278 (prototype `scripts/research/priors.py --prior traceback`). + const TRACE_WEIGHT: f32 = 2.0; + let frames = if opts.traceback { + traceback_definitions(graph, prompt) + } else { + Vec::new() + }; + for (i, d) in frames.into_iter().enumerate() { + let vote = TRACE_WEIGHT / (K + i as f32 + 1.0); + match scored + .iter_mut() + .find(|(_, _, s)| s.path == d.path && s.name == d.name) + { + Some(hit) => hit.0 += vote, + None => { + let mut s = vote; + if let Some(r) = file_rank.get(d.path.as_str()) { + s += 1.0 / (K + *r as f32 + 1.0); + } + let at = scored.len(); + scored.push((s, at, d)); + } + } + } scored.sort_by(|a, b| b.0.total_cmp(&a.0).then_with(|| a.1.cmp(&b.1))); scored.into_iter().take(k).map(|(_, _, d)| d).collect() } -/// Indexed files a report names, best pointer first: traceback frames -/// (`File "…/pkg/mod.py", line 12`) from the deepest up, then any other path -/// or file name in order of mention. A path resolves by its longest suffix -/// that names exactly one indexed file (site-packages prefixes fall away); a -/// bare file name only when one non-test file has it. -fn named_files(graph: &neuromesh_graph::NeuralProjectGraph, prompt: &str) -> Vec { +/// The definitions a report's traceback runs through, deepest frame first. +/// A frame (`File "…/pkg/mod.py", line 12, in name`, or pytest's +/// `pkg/mod.py:12: in name`) resolves to an indexed file as in +/// [`named_files`], then to the definition called `name` there (the one +/// containing the line when several share the name) or, for `` and +/// names the index lacks, the innermost definition containing the line. +/// Frames in tests are skipped: the test that reproduces a bug is not where +/// it is fixed. Line numbers come from the reporter's version, so the name +/// decides first. +pub fn traceback_definitions( + graph: &neuromesh_graph::NeuralProjectGraph, + prompt: &str, +) -> Vec { use std::sync::OnceLock; static FRAME: OnceLock = OnceLock::new(); - static PATH: OnceLock = OnceLock::new(); - let frame = FRAME.get_or_init(|| regex::Regex::new(r#"File "([^"\n]+)", line \d+"#).unwrap()); - let path = - PATH.get_or_init(|| regex::Regex::new(r"[\w./\\-]+\.[A-Za-z][A-Za-z0-9]{0,5}\b").unwrap()); - let files: Vec = graph - .file_node_paths() - .into_iter() - .map(|(_, p)| p.to_string_lossy().replace('\\', "/")) - .collect(); - let mut by_name: std::collections::HashMap<&str, Vec<&String>> = - std::collections::HashMap::new(); - for f in &files { - if !neuromesh_core::source_path::is_test_path(std::path::Path::new(f)) { - by_name - .entry(f.rsplit('/').next().unwrap_or(f)) - .or_default() - .push(f); + static PYTEST: OnceLock = OnceLock::new(); + let frame = FRAME.get_or_init(|| { + regex::Regex::new(r#"File "([^"\n]+\.py)", line (\d+), in ([\w<>.]+)"#).unwrap() + }); + let pytest = PYTEST.get_or_init(|| { + regex::Regex::new(r"(?m)^([\w./\\-]+\.py):(\d+):(?: in ([\w<>.]+))?").unwrap() + }); + let mut frames: Vec<(String, usize, Option)> = Vec::new(); + for c in frame.captures_iter(prompt) { + let line = c[2].parse().unwrap_or(0); + frames.push((c[1].to_string(), line, Some(c[3].to_string()))); + } + for c in pytest.captures_iter(prompt) { + let line = c[2].parse().unwrap_or(0); + frames.push(( + c[1].to_string(), + line, + c.get(3).map(|m| m.as_str().to_string()), + )); + } + if frames.is_empty() { + return Vec::new(); + } + let resolver = FileResolver::new(graph); + let mut out: Vec = Vec::new(); + for (token, line, func) in frames.iter().rev() { + let Some(path) = resolver.resolve(token) else { + continue; + }; + if neuromesh_core::source_path::is_test_path(std::path::Path::new(&path)) { + continue; } + // (qualified name, first line, last line) of every definition that + // is not a container (a class whose methods name it as parent). + let nodes = graph.nodes_in_file(std::path::Path::new(&path)); + let parents: std::collections::HashSet<&str> = + nodes.iter().filter_map(|n| n.parent.as_deref()).collect(); + let defs: Vec<(String, usize, usize)> = nodes + .iter() + .filter(|n| n.node_type != neuromesh_core::NodeType::File) + .filter_map(|n| { + let r = n.line_range.as_ref()?; + if r.end <= r.start { + return None; + } + let name = match &n.parent { + Some(p) if !p.is_empty() => format!("{p}.{}", n.name), + _ => n.name.clone(), + }; + (!parents.contains(name.as_str())).then_some((name, r.start, r.end)) + }) + .collect(); + let wanted = func + .as_deref() + .filter(|f| *f != "") + .map(|f| f.rsplit('.').next().unwrap_or(f)); + let inside = |d: &&(String, usize, usize)| d.1 <= *line && *line <= d.2; + let named: Vec<&(String, usize, usize)> = defs + .iter() + .filter(|d| wanted.is_some_and(|w| d.0.rsplit('.').next() == Some(w))) + .collect(); + let pick = if named.is_empty() { + defs.iter().filter(inside).min_by_key(|d| d.2 - d.1) + } else { + named + .iter() + .copied() + .filter(inside) + .min_by_key(|d| d.2 - d.1) + .or_else(|| named.iter().copied().min_by_key(|d| d.1.abs_diff(*line))) + }; + if let Some((name, start, end)) = pick { + if !out.iter().any(|d| d.path == path && &d.name == name) { + out.push(neuromesh_graph::RankedDefinition { + path: path.clone(), + name: name.clone(), + lines: (*start, *end), + score: 0.0, + }); + } + } + } + out +} + +/// Resolves a path or file name written in a report to an indexed file: the +/// longest suffix that names exactly one indexed file (site-packages and +/// home-directory prefixes fall away), else a bare file name that exactly one +/// non-test file has. +struct FileResolver { + files: Vec, + by_name: std::collections::HashMap>, +} + +impl FileResolver { + fn new(graph: &neuromesh_graph::NeuralProjectGraph) -> Self { + let files: Vec = graph + .file_node_paths() + .into_iter() + .map(|(_, p)| p.to_string_lossy().replace('\\', "/")) + .collect(); + let mut by_name: std::collections::HashMap> = + std::collections::HashMap::new(); + for f in &files { + if !neuromesh_core::source_path::is_test_path(std::path::Path::new(f)) { + by_name + .entry(f.rsplit('/').next().unwrap_or(f).to_string()) + .or_default() + .push(f.clone()); + } + } + Self { files, by_name } } - let resolve = |token: &str| -> Option { + + fn resolve(&self, token: &str) -> Option { let t = token.replace('\\', "/"); let t = t.trim_start_matches("./"); let parts: Vec<&str> = t.split('/').filter(|s| !s.is_empty()).collect(); @@ -1205,9 +1368,11 @@ fn named_files(graph: &neuromesh_graph::NeuralProjectGraph, prompt: &str) -> Vec } for i in 0..parts.len() { let suffix = parts[i..].join("/"); - let hits: Vec<&String> = files + let tail = format!("/{suffix}"); + let hits: Vec<&String> = self + .files .iter() - .filter(|f| **f == suffix || f.ends_with(&format!("/{suffix}"))) + .filter(|f| **f == suffix || f.ends_with(&tail)) .collect(); match hits.len() { 1 => return Some(hits[0].clone()), @@ -1216,11 +1381,27 @@ fn named_files(graph: &neuromesh_graph::NeuralProjectGraph, prompt: &str) -> Vec _ => return None, } } - match by_name.get(parts[parts.len() - 1]) { + match self.by_name.get(parts[parts.len() - 1]) { Some(c) if c.len() == 1 => Some(c[0].clone()), _ => None, } - }; + } +} + +/// Indexed files a report names, best pointer first: traceback frames +/// (`File "…/pkg/mod.py", line 12`) from the deepest up, then any other path +/// or file name in order of mention. A path resolves by its longest suffix +/// that names exactly one indexed file (site-packages prefixes fall away); a +/// bare file name only when one non-test file has it. +fn named_files(graph: &neuromesh_graph::NeuralProjectGraph, prompt: &str) -> Vec { + use std::sync::OnceLock; + static FRAME: OnceLock = OnceLock::new(); + static PATH: OnceLock = OnceLock::new(); + let frame = FRAME.get_or_init(|| regex::Regex::new(r#"File "([^"\n]+)", line \d+"#).unwrap()); + let path = + PATH.get_or_init(|| regex::Regex::new(r"[\w./\\-]+\.[A-Za-z][A-Za-z0-9]{0,5}\b").unwrap()); + let resolver = FileResolver::new(graph); + let resolve = |token: &str| resolver.resolve(token); let mut out: Vec = Vec::new(); let frames: Vec<&str> = frame .captures_iter(prompt) @@ -1304,6 +1485,46 @@ mod tests { } } + #[test] + fn traceback_frames_name_the_functions_deepest_first() { + use neuromesh_index::{IndexedFile, SourceLanguage}; + use neuromesh_parser::CodeIntelligenceEngine; + use std::path::PathBuf; + let graph = NeuralProjectGraph::new(ProjectId::new("tb")); + let core = "def helper(x):\n return x + 1\n\n\nclass Frame:\n def render(self, x):\n y = helper(x)\n return y\n\n def close(self):\n pass\n"; + let test = "from pkg.core import Frame\n\ndef test_render():\n Frame().render(1)\n"; + for (rel, src) in [("pkg/core.py", core), ("tests/test_core.py", test)] { + let file = IndexedFile { + project_id: ProjectId::new("tb"), + relative_path: PathBuf::from(rel), + full_path: PathBuf::from(rel), + blake3_hash: rel.into(), + byte_size: src.len() as u64, + token_count: 40, + language: SourceLanguage::Python, + last_modified: chrono::Utc::now(), + }; + graph.ingest_file( + &file, + &CodeIntelligenceEngine::analyze(&PathBuf::from(rel), src, SourceLanguage::Python), + Some(src), + ); + } + graph.finalize_links(); + let report = "Rendering fails:\n\nTraceback (most recent call last):\n File \"tests/test_core.py\", line 4, in test_render\n Frame().render(1)\n File \"/usr/lib/python3/site-packages/pkg/core.py\", line 7, in render\n y = helper(x)\n File \"/usr/lib/python3/site-packages/pkg/core.py\", line 2, in helper\n return x + 1\nTypeError: boom\n"; + let got: Vec<(String, String)> = traceback_definitions(&graph, report) + .into_iter() + .map(|d| (d.path, d.name)) + .collect(); + assert_eq!( + got, + vec![ + ("pkg/core.py".to_string(), "helper".to_string()), + ("pkg/core.py".to_string(), "Frame.render".to_string()), + ] + ); + } + #[test] fn parses_forbidden_files_from_toml() { let tasks = parse_gold_toml( diff --git a/crates/neuromesh-graph/src/chunk_rank.rs b/crates/neuromesh-graph/src/chunk_rank.rs index 08c20f6..6a9581e 100644 --- a/crates/neuromesh-graph/src/chunk_rank.rs +++ b/crates/neuromesh-graph/src/chunk_rank.rs @@ -15,7 +15,7 @@ //! Built lazily from the sources on disk on the first long report, cached //! with the file ranker's key. -use crate::file_rank::{stemmer, terms_of, RankedFile}; +use crate::file_rank::{stemmer, terms_of_opt, RankedFile}; use neuromesh_core::NodeId; use rayon::prelude::*; use std::collections::HashMap; @@ -41,6 +41,16 @@ pub(crate) struct ChunkSource { pub containers: Vec, } +/// How chunks and queries are turned into terms. The default is what ships; +/// the other settings exist for research runs (`NM_DIAG`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ChunkOpts { + /// Exact words instead of Snowball stems. + pub no_stem: bool, + /// A method's chunk also carries its class name's words. + pub owner_terms: bool, +} + /// One definition ranked for a report: where it is and how well it matched. #[derive(Debug, Clone, serde::Serialize)] pub struct RankedDefinition { @@ -65,21 +75,27 @@ pub(crate) struct ChunkRankIndex { post: HashMap>, /// Per chunk: definition name (`` for the file head) and lines. label: Vec