Plain-language list fusion; two fresh holdouts (cobra, axios); baseline path-match fix

crates/neuromesh-context/src/gold.rs

@@ -1080,10 +1080,15 @@ pub fn localization_order(
     // fusion (k = 60, equal weight) of this list with the definition-level
     // ranking (SWE-bench dev-fast, 57 issues: Acc@1 0.211 → 0.298, @5 0.544
     // → 0.614 in the prototype).
-    if prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS {
+    let long = prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS;
+    {
         const K: f32 = 60.0;
-        let by_def: Vec<String> = graph
-            .chunk_rank(prompt, 4 * k.max(10))
+        let ranked = if long {
+            graph.chunk_rank(prompt, 4 * k.max(10))
+        } else {
+            graph.file_rank(prompt, 4 * k.max(10))
+        };
+        let by_def: Vec<String> = ranked
             .into_iter()
             .filter(|r| !crate::selector::is_noise_path(&r.path))
             .map(|r| r.path.to_string_lossy().replace('\\', "/"))

docs/research/contributions-log.md

@@ -262,3 +262,35 @@ sharding overlapped when a resumed run started (now hash sharding).
 Paper next: (1) head-to-head on LocAgent's exact 274-instance subset; (2) an LLM stage over
 `where_to_look` (Agentless-style file choice) to measure Acc@1 at a fraction of agent tokens;
 (3) ablation table from §8.3 on the holdout (one run per component removed).
+
+### 8.6 Plain-language questions: two fresh holdouts, reranker verdict, list fusion (2026-10-09)
+
+Two new holdouts, gold written from source and committed before any run: **concept-holdout3**
+(spf13/cobra v1.8.1, Go, 12 q) and **concept-holdout4** (axios v1.7.7, JavaScript, 12 q).
+
+Cross-encoder (jina-reranker-v2) on cobra, run once: R@3 of the localisation list 0.792 → **0.708**
+(engine), 0.625 (pure rerank). It had helped ripgrep and click (§8.4) — it does not generalise.
+**Rejected**; code kept on branch `phase-o/rerank-v2` only. This is the holdout principle catching a
+result two looked-at sets suggested.
+
+Short-question list fusion (S1): the localisation list for a short question is now the RRF of the
+packet order and the whole-question ranking (a packet's activation order puts hub files like
+cobra's `command.go` first). R@3, decided on four sets that had been looked at:
+
+| set | before | S1 | S2 (+definition ranking) | BM25 R@3 |
+|---|---|---|---|---|
+| ripgrep | 0.208 | **0.500** | 0.417 | 0.500 |
+| click | 0.750 | **0.917** | 0.917 | 0.750 |
+| this repo | 0.643 | 0.643 | 0.643 | 0.357 |
+| cobra (looked at after its single run) | 0.792 | **0.875** | 0.792 | 0.875 |
+| **axios (fresh, run once)** | 0.833 | 0.833 | — | 0.583 |
+
+Kept S1: gains on three sets, no loss anywhere, neutral on the fresh holdout (so the gain is
+dev-class until another holdout confirms it). Packets are unchanged (list order only); fourteen sets
+unchanged; packet recall/precision on the new sets: cobra 0.833/0.357, axios 0.667/0.528.
+
+**Evaluation bug found and fixed:** the Python baselines matched gold by plain `endswith`, so a gold
+`completions.go` was "found" by `zsh_completions.go`. Only sets with root-level gold files are
+affected: cobra BM25 R@3 is 0.875, not the 0.958 first printed. Re-run on every published set
+(holdout-2/c/ml2/lang, ripgrep, click, this repo): identical numbers. SWE-bench Lite has no root-level
+gold file (dev: 4 of 225, Verified: 1 of 500), and the Rust gold harness matches bare names exactly.

scripts/benchmark-fast.sh

@@ -27,6 +27,8 @@ declare -A MANIFEST=(
   [holdout-web]="tests/third_party/holdout-web/repos.toml"
   [concept-holdout]="tests/third_party/concept-holdout/repos.toml"
   [concept-holdout2]="tests/third_party/concept-holdout2/repos.toml"
+  [concept-holdout3]="tests/third_party/concept-holdout3/repos.toml"
+  [concept-holdout4]="tests/third_party/concept-holdout4/repos.toml"
 )
 declare -A TEST=(
   [dev]="third_party_gold"
@@ -41,6 +43,8 @@ declare -A TEST=(
   [concept]="third_party_private_gold"
   [concept-holdout]="third_party_private_gold"
   [concept-holdout2]="third_party_private_gold"
+  [concept-holdout3]="third_party_private_gold"
+  [concept-holdout4]="third_party_private_gold"
 )
 sets=("$@")
 if [ ${#sets[@]} -eq 0 ]; then
@@ -76,6 +80,8 @@ run_set() {
     concept) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept" NM_PRIVATE_DIR="$root/..") ;;
     concept-holdout) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout") ;;
     concept-holdout2) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout2" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout2") ;;
+    concept-holdout3) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout3" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout3") ;;
+    concept-holdout4) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout4" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout4") ;;
   esac
   # The harness resolves the workspace from its manifest dir at build time.
   (cd crates/neuromesh-context && env "${envs[@]}" "$bin" --nocapture >"$root/$out/$set.log" 2>&1 || true)

scripts/compare_baselines.py

@@ -119,7 +119,10 @@ def rank(self, prompt):
 
 def score(ranked, gold, k):
     top = [f.replace("\\", "/") for f in ranked[:k]]
-    hits = sum(1 for g in gold if any(t.endswith(g) for t in top))
+    # On a path-segment boundary: `zsh_completions.go` is not `completions.go`
+    # (plain `endswith` counted it, inflating BM25 on repositories with
+    # root-level files — cobra R@3 0.958 was really lower).
+    hits = sum(1 for g in gold if any(t == g or t.endswith("/" + g) for t in top))
     return hits / len(gold), (hits / len(top) if top else 0.0)
 
 

scripts/research/eval_loc.py

@@ -30,7 +30,8 @@ def norm(f):
 
 def acc(lst, gold, k):
     top = [norm(f) for f in lst[:k]]
-    return int(all(any(t.endswith(g) or g.endswith(t) for t in top) for g in gold))
+    same = lambda t, g: t == g or t.endswith("/" + g) or g.endswith("/" + t)  # noqa: E731
+    return int(all(any(same(t, g) for t in top) for g in gold))
 
 
 def rrf(a, b, k=60):

scripts/swebench_localize.py

@@ -95,7 +95,8 @@ def run_packet(binary, workspace, query):
 def hit(files, gold, k=None):
     top = files if k is None else files[:k]
     top = [f.replace("\\", "/") for f in top]
-    return int(all(any(t.endswith(g) or g.endswith(t) for t in top) for g in gold))
+    same = lambda t, g: t == g or t.endswith("/" + g) or g.endswith("/" + t)  # noqa: E731
+    return int(all(any(same(t, g) for t in top) for g in gold))
 
 
 def main():

tests/third_party/concept-holdout3/cobra/gold_tasks.toml

@@ -0,0 +1,86 @@
+# Locked 2026-10-09 before any engine run on cobra. Plain-language
+# questions only; gold from reading the source (grep-checked).
+
+[[task]]
+id = "cobra_did_you_mean"
+prompt = "When a user mistypes a subcommand, how does it come up with the did-you-mean suggestions?"
+gold_files = ["command.go", "cobra.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_explorer_launch"
+prompt = "What happens on Windows when the program is started by double-clicking it in Explorer?"
+gold_files = ["command_win.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_flags_together"
+prompt = "How can I require that two flags are always given together, or forbid using them at the same time?"
+gold_files = ["flag_groups.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_positional_count"
+prompt = "How are the positional arguments a command accepts checked, for example requiring an exact number of them?"
+gold_files = ["args.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_fish"
+prompt = "How is the tab-completion script for the fish shell produced?"
+gold_files = ["fish_completions.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_man_pages"
+prompt = "How does it generate Unix manual pages for the commands?"
+gold_files = ["doc/man_docs.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_hint_messages"
+prompt = "Where are the extra hint messages shown under completion suggestions handled, and how can a user turn them off through an environment variable?"
+gold_files = ["active_help.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_offer_completions"
+prompt = "How does it work out which completions to offer while the user is typing a command line?"
+gold_files = ["completions.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_powershell"
+prompt = "How is completion supported in PowerShell?"
+gold_files = ["powershell_completions.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_markdown_docs"
+prompt = "How can it write a documentation page in Markdown for every command?"
+gold_files = ["doc/md_docs.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_find_subcommand"
+prompt = "How does it walk the tree of commands to figure out which subcommand the arguments refer to?"
+gold_files = ["command.go"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "cobra_zsh"
+prompt = "How are completions for zsh generated?"
+gold_files = ["zsh_completions.go"]
+forbidden_files = []
+expect_seeds_missed = false

tests/third_party/concept-holdout3/repos.toml

@@ -0,0 +1,11 @@
+# Third plain-language holdout (2026-10-09): written to confirm the
+# cross-encoder reorder (jina-reranker-v2) on a repository and language
+# (Go) that no plain-language decision has looked at. Gold written from
+# reading the source before any engine run; locked by this commit.
+#
+#   cobra — spf13/cobra v1.8.1: Go CLI framework, 14 source files + doc/
+
+[[repo]]
+name = "cobra"
+url = "https://github.com/spf13/cobra"
+rev = "e94f6d0dd9a5e5738dca6bce03c4b1207ffbc0ec"

tests/third_party/concept-holdout4/axios/gold_tasks.toml

@@ -0,0 +1,86 @@
+# Locked 2026-10-09 before any engine run on axios. Plain-language
+# questions only; gold from reading the source (grep-checked).
+
+[[task]]
+id = "ax_status_success"
+prompt = "How does it decide whether a response with a given status code counts as a success or a failure?"
+gold_files = ["lib/core/settle.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_merge_options"
+prompt = "How are two sets of request options combined when a call overrides the instance defaults?"
+gold_files = ["lib/core/mergeConfig.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_cancel_token"
+prompt = "How can a request that is already in flight be cancelled with a token?"
+gold_files = ["lib/cancel/CancelToken.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_hooks"
+prompt = "How do the hooks that run before a request is sent or after a response arrives get registered and removed?"
+gold_files = ["lib/core/InterceptorManager.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_cookies"
+prompt = "In the browser, how does it read and write cookies?"
+gold_files = ["lib/helpers/cookies.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_same_origin"
+prompt = "How does it check whether the request URL is on the same origin as the current page?"
+gold_files = ["lib/helpers/isURLSameOrigin.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_speed"
+prompt = "How is the transfer speed measured for progress reporting?"
+gold_files = ["lib/helpers/speedometer.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_data_uri"
+prompt = "How does it turn a data: URI into a buffer or a blob?"
+gold_files = ["lib/helpers/fromDataURI.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_parse_headers"
+prompt = "How are raw response header lines parsed into an object, and what happens with duplicate headers?"
+gold_files = ["lib/helpers/parseHeaders.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_pick_transport"
+prompt = "How does it pick which transport to use: the browser XHR, fetch, or the Node http module?"
+gold_files = ["lib/adapters/adapters.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_node_redirects"
+prompt = "How does it follow redirects and decompress gzip responses when running in Node?"
+gold_files = ["lib/adapters/http.js"]
+forbidden_files = []
+expect_seeds_missed = false
+
+[[task]]
+id = "ax_nested_form"
+prompt = "How are nested form field names like foo[x][y] turned into a JSON object?"
+gold_files = ["lib/helpers/formDataToJSON.js"]
+forbidden_files = []
+expect_seeds_missed = false

tests/third_party/concept-holdout4/repos.toml

@@ -0,0 +1,11 @@
+# Fourth plain-language holdout (2026-10-09), JavaScript: written to confirm
+# the short-question list fusion decided on ripgrep/click/this repo/cobra.
+# Gold written from reading the source before any engine run; locked by
+# this commit.
+#
+#   axios — axios/axios v1.7.7: HTTP client for the browser and Node
+
+[[repo]]
+name = "axios"
+url = "https://github.com/axios/axios"
+rev = "5b8a826771b77ab30081d033fdba9ef3b90e439a"