From 6cb9e3c8e1ff84a8758b09607863929b72a9a097 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Fri, 9 Oct 2026 10:11:36 +0330 Subject: [PATCH 1/3] concept-holdout3 (cobra, Go): 12 plain-language questions, gold locked before any run Co-Authored-By: Claude Opus 5.5 --- .../concept-holdout3/cobra/gold_tasks.toml | 86 +++++++++++++++++++ tests/third_party/concept-holdout3/repos.toml | 11 +++ 2 files changed, 97 insertions(+) create mode 100644 tests/third_party/concept-holdout3/cobra/gold_tasks.toml create mode 100644 tests/third_party/concept-holdout3/repos.toml diff --git a/tests/third_party/concept-holdout3/cobra/gold_tasks.toml b/tests/third_party/concept-holdout3/cobra/gold_tasks.toml new file mode 100644 index 0000000..ab4bbb6 --- /dev/null +++ b/tests/third_party/concept-holdout3/cobra/gold_tasks.toml @@ -0,0 +1,86 @@ +# Locked 2026-10-09 before any engine run on cobra. Plain-language +# questions only; gold from reading the source (grep-checked). + +[[task]] +id = "cobra_did_you_mean" +prompt = "When a user mistypes a subcommand, how does it come up with the did-you-mean suggestions?" +gold_files = ["command.go", "cobra.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_explorer_launch" +prompt = "What happens on Windows when the program is started by double-clicking it in Explorer?" +gold_files = ["command_win.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_flags_together" +prompt = "How can I require that two flags are always given together, or forbid using them at the same time?" +gold_files = ["flag_groups.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_positional_count" +prompt = "How are the positional arguments a command accepts checked, for example requiring an exact number of them?" +gold_files = ["args.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_fish" +prompt = "How is the tab-completion script for the fish shell produced?" +gold_files = ["fish_completions.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_man_pages" +prompt = "How does it generate Unix manual pages for the commands?" +gold_files = ["doc/man_docs.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_hint_messages" +prompt = "Where are the extra hint messages shown under completion suggestions handled, and how can a user turn them off through an environment variable?" +gold_files = ["active_help.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_offer_completions" +prompt = "How does it work out which completions to offer while the user is typing a command line?" +gold_files = ["completions.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_powershell" +prompt = "How is completion supported in PowerShell?" +gold_files = ["powershell_completions.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_markdown_docs" +prompt = "How can it write a documentation page in Markdown for every command?" +gold_files = ["doc/md_docs.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_find_subcommand" +prompt = "How does it walk the tree of commands to figure out which subcommand the arguments refer to?" +gold_files = ["command.go"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "cobra_zsh" +prompt = "How are completions for zsh generated?" +gold_files = ["zsh_completions.go"] +forbidden_files = [] +expect_seeds_missed = false diff --git a/tests/third_party/concept-holdout3/repos.toml b/tests/third_party/concept-holdout3/repos.toml new file mode 100644 index 0000000..bdb8a14 --- /dev/null +++ b/tests/third_party/concept-holdout3/repos.toml @@ -0,0 +1,11 @@ +# Third plain-language holdout (2026-10-09): written to confirm the +# cross-encoder reorder (jina-reranker-v2) on a repository and language +# (Go) that no plain-language decision has looked at. Gold written from +# reading the source before any engine run; locked by this commit. +# +# cobra — spf13/cobra v1.8.1: Go CLI framework, 14 source files + doc/ + +[[repo]] +name = "cobra" +url = "https://github.com/spf13/cobra" +rev = "e94f6d0dd9a5e5738dca6bce03c4b1207ffbc0ec" From 757fc092b4e3bbf1277fb88f586985f07c8ed233 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Fri, 9 Oct 2026 10:33:01 +0330 Subject: [PATCH 2/3] concept-holdout4 (axios, JavaScript): 12 plain-language questions, gold locked before any run Co-Authored-By: Claude Opus 5.5 --- .../concept-holdout4/axios/gold_tasks.toml | 86 +++++++++++++++++++ tests/third_party/concept-holdout4/repos.toml | 11 +++ 2 files changed, 97 insertions(+) create mode 100644 tests/third_party/concept-holdout4/axios/gold_tasks.toml create mode 100644 tests/third_party/concept-holdout4/repos.toml diff --git a/tests/third_party/concept-holdout4/axios/gold_tasks.toml b/tests/third_party/concept-holdout4/axios/gold_tasks.toml new file mode 100644 index 0000000..e3848f4 --- /dev/null +++ b/tests/third_party/concept-holdout4/axios/gold_tasks.toml @@ -0,0 +1,86 @@ +# Locked 2026-10-09 before any engine run on axios. Plain-language +# questions only; gold from reading the source (grep-checked). + +[[task]] +id = "ax_status_success" +prompt = "How does it decide whether a response with a given status code counts as a success or a failure?" +gold_files = ["lib/core/settle.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_merge_options" +prompt = "How are two sets of request options combined when a call overrides the instance defaults?" +gold_files = ["lib/core/mergeConfig.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_cancel_token" +prompt = "How can a request that is already in flight be cancelled with a token?" +gold_files = ["lib/cancel/CancelToken.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_hooks" +prompt = "How do the hooks that run before a request is sent or after a response arrives get registered and removed?" +gold_files = ["lib/core/InterceptorManager.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_cookies" +prompt = "In the browser, how does it read and write cookies?" +gold_files = ["lib/helpers/cookies.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_same_origin" +prompt = "How does it check whether the request URL is on the same origin as the current page?" +gold_files = ["lib/helpers/isURLSameOrigin.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_speed" +prompt = "How is the transfer speed measured for progress reporting?" +gold_files = ["lib/helpers/speedometer.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_data_uri" +prompt = "How does it turn a data: URI into a buffer or a blob?" +gold_files = ["lib/helpers/fromDataURI.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_parse_headers" +prompt = "How are raw response header lines parsed into an object, and what happens with duplicate headers?" +gold_files = ["lib/helpers/parseHeaders.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_pick_transport" +prompt = "How does it pick which transport to use: the browser XHR, fetch, or the Node http module?" +gold_files = ["lib/adapters/adapters.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_node_redirects" +prompt = "How does it follow redirects and decompress gzip responses when running in Node?" +gold_files = ["lib/adapters/http.js"] +forbidden_files = [] +expect_seeds_missed = false + +[[task]] +id = "ax_nested_form" +prompt = "How are nested form field names like foo[x][y] turned into a JSON object?" +gold_files = ["lib/helpers/formDataToJSON.js"] +forbidden_files = [] +expect_seeds_missed = false diff --git a/tests/third_party/concept-holdout4/repos.toml b/tests/third_party/concept-holdout4/repos.toml new file mode 100644 index 0000000..a81462d --- /dev/null +++ b/tests/third_party/concept-holdout4/repos.toml @@ -0,0 +1,11 @@ +# Fourth plain-language holdout (2026-10-09), JavaScript: written to confirm +# the short-question list fusion decided on ripgrep/click/this repo/cobra. +# Gold written from reading the source before any engine run; locked by +# this commit. +# +# axios — axios/axios v1.7.7: HTTP client for the browser and Node + +[[repo]] +name = "axios" +url = "https://github.com/axios/axios" +rev = "5b8a826771b77ab30081d033fdba9ef3b90e439a" From 3e496afe58965beb4d6ddb96f2fed340a5f825b8 Mon Sep 17 00:00:00 2001 From: ParsaVictor <1.parsa.karkooti@gmail.com> Date: Fri, 9 Oct 2026 11:04:33 +0330 Subject: [PATCH 3/3] plain-language list fusion (S1); concept-holdout3/4 (cobra, axios); baseline path-match fix - localization for a short question: RRF of the packet order and the whole- question ranking. R@3: ripgrep 0.208 -> 0.500, click 0.750 -> 0.917, cobra 0.792 -> 0.875, this repo and the fresh axios holdout unchanged. List order only; packets unchanged. - Two plain-language holdouts with gold locked before any run: cobra (Go) and axios (JavaScript), wired into benchmark-fast.sh (packet 0.833/0.357 and 0.667/0.528). - Python baselines matched gold with a plain endswith (`zsh_completions.go` "was" `completions.go`): now on a path-segment boundary. Published numbers re-run unchanged; only the new cobra set was affected (BM25 R@3 0.875). - jina-reranker-v2 rejected on the fresh cobra holdout (R@3 0.792 -> 0.708). Fourteen sets unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/neuromesh-context/src/gold.rs | 11 +++++++--- docs/research/contributions-log.md | 32 ++++++++++++++++++++++++++++ scripts/benchmark-fast.sh | 6 ++++++ scripts/compare_baselines.py | 5 ++++- scripts/research/eval_loc.py | 3 ++- scripts/swebench_localize.py | 3 ++- 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/crates/neuromesh-context/src/gold.rs b/crates/neuromesh-context/src/gold.rs index f835bbb..642c9b3 100644 --- a/crates/neuromesh-context/src/gold.rs +++ b/crates/neuromesh-context/src/gold.rs @@ -1080,10 +1080,15 @@ pub fn localization_order( // fusion (k = 60, equal weight) of this list with the definition-level // ranking (SWE-bench dev-fast, 57 issues: Acc@1 0.211 → 0.298, @5 0.544 // → 0.614 in the prototype). - if prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS { + let long = prompt.split_whitespace().count() >= neuromesh_parser::text_normalize::REPORT_WORDS; + { const K: f32 = 60.0; - let by_def: Vec = graph - .chunk_rank(prompt, 4 * k.max(10)) + let ranked = if long { + graph.chunk_rank(prompt, 4 * k.max(10)) + } else { + graph.file_rank(prompt, 4 * k.max(10)) + }; + let by_def: Vec = ranked .into_iter() .filter(|r| !crate::selector::is_noise_path(&r.path)) .map(|r| r.path.to_string_lossy().replace('\\', "/")) diff --git a/docs/research/contributions-log.md b/docs/research/contributions-log.md index ca4d6de..2b45819 100644 --- a/docs/research/contributions-log.md +++ b/docs/research/contributions-log.md @@ -262,3 +262,35 @@ sharding overlapped when a resumed run started (now hash sharding). Paper next: (1) head-to-head on LocAgent's exact 274-instance subset; (2) an LLM stage over `where_to_look` (Agentless-style file choice) to measure Acc@1 at a fraction of agent tokens; (3) ablation table from §8.3 on the holdout (one run per component removed). + +### 8.6 Plain-language questions: two fresh holdouts, reranker verdict, list fusion (2026-10-09) + +Two new holdouts, gold written from source and committed before any run: **concept-holdout3** +(spf13/cobra v1.8.1, Go, 12 q) and **concept-holdout4** (axios v1.7.7, JavaScript, 12 q). + +Cross-encoder (jina-reranker-v2) on cobra, run once: R@3 of the localisation list 0.792 → **0.708** +(engine), 0.625 (pure rerank). It had helped ripgrep and click (§8.4) — it does not generalise. +**Rejected**; code kept on branch `phase-o/rerank-v2` only. This is the holdout principle catching a +result two looked-at sets suggested. + +Short-question list fusion (S1): the localisation list for a short question is now the RRF of the +packet order and the whole-question ranking (a packet's activation order puts hub files like +cobra's `command.go` first). R@3, decided on four sets that had been looked at: + +| set | before | S1 | S2 (+definition ranking) | BM25 R@3 | +|---|---|---|---|---| +| ripgrep | 0.208 | **0.500** | 0.417 | 0.500 | +| click | 0.750 | **0.917** | 0.917 | 0.750 | +| this repo | 0.643 | 0.643 | 0.643 | 0.357 | +| cobra (looked at after its single run) | 0.792 | **0.875** | 0.792 | 0.875 | +| **axios (fresh, run once)** | 0.833 | 0.833 | — | 0.583 | + +Kept S1: gains on three sets, no loss anywhere, neutral on the fresh holdout (so the gain is +dev-class until another holdout confirms it). Packets are unchanged (list order only); fourteen sets +unchanged; packet recall/precision on the new sets: cobra 0.833/0.357, axios 0.667/0.528. + +**Evaluation bug found and fixed:** the Python baselines matched gold by plain `endswith`, so a gold +`completions.go` was "found" by `zsh_completions.go`. Only sets with root-level gold files are +affected: cobra BM25 R@3 is 0.875, not the 0.958 first printed. Re-run on every published set +(holdout-2/c/ml2/lang, ripgrep, click, this repo): identical numbers. SWE-bench Lite has no root-level +gold file (dev: 4 of 225, Verified: 1 of 500), and the Rust gold harness matches bare names exactly. diff --git a/scripts/benchmark-fast.sh b/scripts/benchmark-fast.sh index 7c2ccb7..3787128 100644 --- a/scripts/benchmark-fast.sh +++ b/scripts/benchmark-fast.sh @@ -27,6 +27,8 @@ declare -A MANIFEST=( [holdout-web]="tests/third_party/holdout-web/repos.toml" [concept-holdout]="tests/third_party/concept-holdout/repos.toml" [concept-holdout2]="tests/third_party/concept-holdout2/repos.toml" + [concept-holdout3]="tests/third_party/concept-holdout3/repos.toml" + [concept-holdout4]="tests/third_party/concept-holdout4/repos.toml" ) declare -A TEST=( [dev]="third_party_gold" @@ -41,6 +43,8 @@ declare -A TEST=( [concept]="third_party_private_gold" [concept-holdout]="third_party_private_gold" [concept-holdout2]="third_party_private_gold" + [concept-holdout3]="third_party_private_gold" + [concept-holdout4]="third_party_private_gold" ) sets=("$@") if [ ${#sets[@]} -eq 0 ]; then @@ -76,6 +80,8 @@ run_set() { concept) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept" NM_PRIVATE_DIR="$root/..") ;; concept-holdout) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout") ;; concept-holdout2) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout2" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout2") ;; + concept-holdout3) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout3" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout3") ;; + concept-holdout4) envs=(NM_PRIVATE_SET_DIR="$root/tests/third_party/concept-holdout4" NM_PRIVATE_DIR="$root/target/third_party/concept-holdout4") ;; esac # The harness resolves the workspace from its manifest dir at build time. (cd crates/neuromesh-context && env "${envs[@]}" "$bin" --nocapture >"$root/$out/$set.log" 2>&1 || true) diff --git a/scripts/compare_baselines.py b/scripts/compare_baselines.py index 5060ca6..3d361ab 100644 --- a/scripts/compare_baselines.py +++ b/scripts/compare_baselines.py @@ -119,7 +119,10 @@ def rank(self, prompt): def score(ranked, gold, k): top = [f.replace("\\", "/") for f in ranked[:k]] - hits = sum(1 for g in gold if any(t.endswith(g) for t in top)) + # On a path-segment boundary: `zsh_completions.go` is not `completions.go` + # (plain `endswith` counted it, inflating BM25 on repositories with + # root-level files — cobra R@3 0.958 was really lower). + hits = sum(1 for g in gold if any(t == g or t.endswith("/" + g) for t in top)) return hits / len(gold), (hits / len(top) if top else 0.0) diff --git a/scripts/research/eval_loc.py b/scripts/research/eval_loc.py index 6186a9b..692e6e7 100644 --- a/scripts/research/eval_loc.py +++ b/scripts/research/eval_loc.py @@ -30,7 +30,8 @@ def norm(f): def acc(lst, gold, k): top = [norm(f) for f in lst[:k]] - return int(all(any(t.endswith(g) or g.endswith(t) for t in top) for g in gold)) + same = lambda t, g: t == g or t.endswith("/" + g) or g.endswith("/" + t) # noqa: E731 + return int(all(any(same(t, g) for t in top) for g in gold)) def rrf(a, b, k=60): diff --git a/scripts/swebench_localize.py b/scripts/swebench_localize.py index 12d3027..6e959ec 100644 --- a/scripts/swebench_localize.py +++ b/scripts/swebench_localize.py @@ -95,7 +95,8 @@ def run_packet(binary, workspace, query): def hit(files, gold, k=None): top = files if k is None else files[:k] top = [f.replace("\\", "/") for f in top] - return int(all(any(t.endswith(g) or g.endswith(t) for t in top) for g in gold)) + same = lambda t, g: t == g or t.endswith("/" + g) or g.endswith("/" + t) # noqa: E731 + return int(all(any(same(t, g) for t in top) for g in gold)) def main():