diff --git a/MANIFEST.in b/MANIFEST.in index cb6b46c..fd0b06e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,3 +14,4 @@ recursive-include commands *.toml recursive-include skills *.md recursive-include docs *.md recursive-include deeprefine_skill/gemini_extension * +recursive-include deeprefine_skill/benchmark_suites *.json *.jsonl *.md *.txt diff --git a/README.md b/README.md index 33a0b1e..276bcd7 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,9 @@ Run from your KB project root. | `deeprefine apply --refresh-wiki --trace-file T --refinement-file F` | Apply actions and regenerate `graphify-out/wiki` from the refined graph; graph + Wiki are committed together | | `deeprefine apply --refresh-wiki --allow-low-confidence --trace-file T --refinement-file F` | Refresh the Wiki while explicitly overriding the LOW-confidence guard | | `deeprefine loop finish --trace-file T [--refinement-file F]` | Persist results and mark history refined | +| `deeprefine benchmark prepare ...` | Prepare a deterministic smoke/Re-DocRED/2Wiki mini suite | +| `deeprefine benchmark evaluate ...` | Compare baseline and candidate graphs without modifying either | +| `deeprefine benchmark report ...` | Merge result JSON files into a README-ready Markdown table | ### Evidence-aware review and safe apply @@ -186,6 +189,110 @@ When the knowledge base was created with Graphify Wiki output, use `--refresh-wi --- +## Lightweight graph-quality benchmark + +`deeprefine benchmark` compares a Graphify baseline with a refined candidate +without importing DeepRefine's FAISS/model stack. Default scoring is offline, +deterministic, read-only, and uses only the Python standard library. + +It reports two complementary views: + +- **Intrinsic quality:** entity and strict `(head, relation, tail)` + precision/recall/F1 against an annotated graph, plus lightweight + GraphJudge-style G-BLEU and G-ROUGE. +- **Downstream quality:** query-relevant evidence-edge recall, complete + multi-hop path rate, answer reachability, and before/after repair-regression + transitions. Optional prediction JSONL adds Hit@K, MRR, supporting-fact + Recall@K, answer EM, and token F1. + +The Wiki is a derived view of `graph.json`. Optional `--baseline-wiki` and +`--candidate-wiki` arguments check `index.md`, local links, and orphan pages; +those integrity checks are not treated as semantic quality scores. + +### Zero-cost smoke run + +The package includes a self-authored fixture solely to verify the evaluator: + +```bash +deeprefine benchmark evaluate \ + --suite synthetic-smoke-v1 \ + --baseline-graph deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_graph.json \ + --candidate-graph deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_graph.json \ + --baseline-predictions deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_predictions.jsonl \ + --candidate-predictions deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_predictions.jsonl \ + --output-dir graphify-out/.deeprefine/benchmark/smoke +``` + +Expected self-check values: + +| Suite | Metric | Synthetic Before | Synthetic After | Δ | +|---|---|---:|---:|---:| +| synthetic-smoke-v1 | Entity F1 | 97.73% | 100.00% | +2.27 pp | +| synthetic-smoke-v1 | Strict Triple F1 | 82.86% | 100.00% | +17.14 pp | +| synthetic-smoke-v1 | Evidence Edge Recall | 47.92% | 100.00% | +52.08 pp | +| synthetic-smoke-v1 | Complete Path Rate | 25.00% | 100.00% | +75.00 pp | +| synthetic-smoke-v1 | Answer F1 | 25.00% | 100.00% | +75.00 pp | + +These values describe an intentionally damaged and repaired test graph. They +are not a DeepRefine performance claim. + +### Prepare a real mini suite + +Raw third-party text is not bundled. Supply an official upstream JSON file: + +```bash +deeprefine benchmark prepare \ + --suite redocred-mini-v1 \ + --profile readme \ + --source /path/to/Re-DocRED/dev_revised.json \ + --output-dir prepared/redocred-mini-v1 + +deeprefine benchmark prepare \ + --suite 2wiki-mini-v1 \ + --profile readme \ + --source /path/to/2wikimultihopqa.json \ + --output-dir prepared/2wiki-mini-v1 +``` + +`quick` selects 10 Re-DocRED documents or 16 balanced 2Wiki questions. +`readme` selects 50 documents or 64 questions. Selection is deterministic, and +`suite.lock.json` records source and generated-file SHA-256 hashes. Prepared +corpora retain every selected 2Wiki question's distractor passages. + +After building and refining graphs from the prepared corpus, score the pair: + +```bash +deeprefine benchmark evaluate \ + --suite prepared/2wiki-mini-v1 \ + --baseline-graph runs/before/graph.json \ + --candidate-graph runs/after/graph.json \ + --graphify-version 0.9.12 \ + --deeprefine-version COMMIT_OR_MODEL \ + --output-dir graphify-out/.deeprefine/benchmark/2wiki + +deeprefine benchmark report \ + --result graphify-out/.deeprefine/benchmark/redocred/result.json \ + --result graphify-out/.deeprefine/benchmark/2wiki/result.json \ + --format markdown \ + --output graphify-out/.deeprefine/benchmark/README-results.md +``` + +For optional G-BERTScore, install +`pip install "deeprefine-cli[benchmark-semantic]"` and pass +`--semantic-model roberta-large`. It is intentionally excluded from default +CI because it downloads a large model and PyTorch runtime. + +2Wiki `evidences` are answer paths, not an exhaustive graph. Accordingly, +2Wiki reports evidence recall/path coverage but never full-graph edge +precision. This micro-benchmark supports regression checks and README +demonstrations; it does not replace full GraphRAG evaluation. See +[benchmark data attribution](docs/benchmark-data.md) for sources and licensing. +Real Re-DocRED/2Wiki scores should only be added here together with their +committed `result.json`, exact commands, versions, hashes, token counts, and +cost. + +--- + ## Codex Integration
@@ -563,7 +670,7 @@ deeprefine refine # dry-run by default ```bash deeprefine --help -# Expect: cursor, copilot, codex, claude, opencode, gemini, history, index, refine, review, apply, loop +# Expect: cursor, copilot, codex, claude, opencode, gemini, benchmark, history, index, refine, review, apply, loop ```
diff --git a/deeprefine_skill/benchmark_suites/README.md b/deeprefine_skill/benchmark_suites/README.md new file mode 100644 index 0000000..bb13e59 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/README.md @@ -0,0 +1,35 @@ +# Benchmark suites and third-party data + +The package ships only the self-authored `synthetic-smoke-v1` fixture. It is +licensed under the same MIT license as DeepRefine-Skill and exists to test the +evaluator; its scores are not a model-quality claim. + +Real mini suites are prepared locally from an upstream file supplied by the +user. DeepRefine-Skill records the source SHA-256 and selected sample IDs, but +does not redistribute the original text. + +## Re-DocRED + +- Paper: https://arxiv.org/abs/2205.12696 +- Repository: https://github.com/tonytan48/Re-DocRED +- Expected input: the official revised dev JSON (`dev_revised.json`), with + `rel_info.json` in the same directory when human-readable relation names are + available. + +Review and comply with the dataset repository's current terms and the terms of +its underlying Wikipedia/Wikidata content before use. + +## 2WikiMultiHopQA + +- Paper: https://arxiv.org/abs/2011.01060 +- Repository: https://github.com/Alab-NII/2wikimultihop +- Expected input: an official JSON split, or the compatible + `DeepRefine/benchmark/2wikimultihopqa.json` file. + +The evaluator treats `evidences` as query-relevant reasoning paths only. They +are not an exhaustive gold graph, so the 2Wiki suite never reports full-graph +edge precision. + +Generated prepared-suite directories retain source paths, hashes, provenance, +and paper links in `suite.json` and `suite.lock.json`. Those generated +directories should not be redistributed without checking the upstream terms. diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_graph.json b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_graph.json new file mode 100644 index 0000000..a0a6ad9 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_graph.json @@ -0,0 +1,60 @@ +{ + "directed": true, + "multigraph": true, + "nodes": [ + {"aliases": ["M. Chen"], "id": "b01", "label": "Mira Chen", "source_file": "corpus\\doc-alpha.txt"}, + {"aliases": ["Aurora"], "id": "b02", "label": "Aurora University", "source_file": "corpus\\doc-alpha.txt"}, + {"id": "b03", "label": "Aster City", "source_file": "corpus\\doc-alpha.txt"}, + {"aliases": ["Borealis"], "id": "b04", "label": "Borealis Republic", "source_file": "corpus\\doc-alpha.txt"}, + {"id": "b05", "label": "1998", "source_file": "corpus\\doc-alpha.txt"}, + {"id": "b06", "label": "Leo Hart", "source_file": "corpus\\doc-beta.txt"}, + {"aliases": ["Quill"], "id": "b07", "label": "Quill Labs", "source_file": "corpus\\doc-beta.txt"}, + {"aliases": ["Orion"], "id": "b08", "label": "Orion Database", "source_file": "corpus\\doc-beta.txt"}, + {"id": "b09", "label": "Cedar City", "source_file": "corpus\\doc-beta.txt"}, + {"id": "b10", "label": "2012", "source_file": "corpus\\doc-beta.txt"}, + {"id": "b11", "label": "Mystery Lab", "source_file": "corpus\\doc-beta.txt"}, + {"id": "b12", "label": "Nia Vale", "source_file": "corpus\\doc-gamma.txt"}, + {"aliases": ["Silver"], "id": "b13", "label": "Silver Observatory", "source_file": "corpus\\doc-gamma.txt"}, + {"aliases": ["Helios"], "id": "b14", "label": "Helios Comet", "source_file": "corpus\\doc-gamma.txt"}, + {"id": "b15", "label": "Ridge Province", "source_file": "corpus\\doc-gamma.txt"}, + {"id": "b16", "label": "2004", "source_file": "corpus\\doc-gamma.txt"}, + {"id": "b17", "label": "Oren Pike", "source_file": "corpus\\doc-delta.txt"}, + {"aliases": ["Bluebird"], "id": "b18", "label": "Bluebird Press", "source_file": "corpus\\doc-delta.txt"}, + {"aliases": ["Ember"], "id": "b19", "label": "Ember Novel", "source_file": "corpus\\doc-delta.txt"}, + {"id": "b20", "label": "Lumen Town", "source_file": "corpus\\doc-delta.txt"}, + {"id": "b21", "label": "2018", "source_file": "corpus\\doc-delta.txt"} + ], + "edges": [ + {"relation": "located_in", "source": "b02", "target": "b03"}, + {"relation": "largest_city_of", "source": "b03", "target": "b04"}, + {"relation": "founded_in", "source": "b02", "target": "b05"}, + {"relation": "citizen_of", "source": "b01", "target": "b04"}, + {"relation": "contains", "source": "b04", "target": "b03"}, + {"relation": "born_in", "source": "b01", "target": "b03"}, + {"relation": "enrolls", "source": "b02", "target": "b01"}, + {"relation": "founded", "source": "b06", "target": "b07"}, + {"relation": "built", "source": "b07", "target": "b08"}, + {"relation": "headquartered_in", "source": "b07", "target": "b09"}, + {"relation": "announced_in", "source": "b08", "target": "b10"}, + {"relation": "lives_in", "source": "b06", "target": "b09"}, + {"relation": "hosts", "source": "b09", "target": "b07"}, + {"relation": "developed_by", "source": "b08", "target": "b07"}, + {"relation": "created", "source": "b06", "target": "b08"}, + {"relation": "owns", "source": "b11", "target": "b08"}, + {"relation": "works_at", "source": "b12", "target": "b13"}, + {"relation": "observed", "source": "b13", "target": "b14"}, + {"relation": "located_in", "source": "b13", "target": "b15"}, + {"relation": "discovered_in", "source": "b14", "target": "b16"}, + {"relation": "observed", "source": "b12", "target": "b14"}, + {"relation": "contains", "source": "b15", "target": "b13"}, + {"relation": "observed_by", "source": "b14", "target": "b12"}, + {"relation": "lives_in", "source": "b12", "target": "b15"}, + {"relation": "wrote", "source": "b17", "target": "b19"}, + {"relation": "published", "source": "b18", "target": "b19"}, + {"relation": "published_in", "source": "b19", "target": "b21"}, + {"relation": "lives_in", "source": "b17", "target": "b20"}, + {"relation": "hosts", "source": "b20", "target": "b18"}, + {"relation": "author", "source": "b19", "target": "b17"}, + {"relation": "works_with", "source": "b17", "target": "b18"} + ] +} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_predictions.jsonl b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_predictions.jsonl new file mode 100644 index 0000000..4ac989e --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/baseline_predictions.jsonl @@ -0,0 +1,8 @@ +{"answer":"Borealis Republic","case_id":"qa-01","retrieved_nodes":["Borealis Republic","Aster City"],"retrieved_supporting_facts":[]} +{"answer":"Aster City","case_id":"qa-02","retrieved_nodes":["Aster City","Borealis Republic"],"retrieved_supporting_facts":["doc-alpha#1"]} +{"answer":"Quill Labs","case_id":"qa-03","retrieved_nodes":["Quill Labs","Cedar City"],"retrieved_supporting_facts":["doc-beta#0"]} +{"answer":"2011","case_id":"qa-04","retrieved_nodes":["Cedar City","2012"],"retrieved_supporting_facts":[]} +{"answer":"Silver Observatory","case_id":"qa-05","retrieved_nodes":["Silver Observatory","Ridge Province"],"retrieved_supporting_facts":["doc-gamma#0"]} +{"answer":"Ridge Province","case_id":"qa-06","retrieved_nodes":["Ridge Province"],"retrieved_supporting_facts":["doc-gamma#2"]} +{"answer":"Ember Novel","case_id":"qa-07","retrieved_nodes":["Ember Novel"],"retrieved_supporting_facts":["doc-delta#0"]} +{"answer":"Bluebird Press","case_id":"qa-08","retrieved_nodes":["Bluebird Press","Lumen Town"],"retrieved_supporting_facts":["doc-delta#0"]} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_graph.json b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_graph.json new file mode 100644 index 0000000..5435f8a --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_graph.json @@ -0,0 +1,60 @@ +{ + "directed": true, + "multigraph": true, + "nodes": [ + {"aliases": ["M. Chen"], "id": "c01", "label": "Mira Chen", "source_file": "corpus/doc-alpha.txt"}, + {"aliases": ["Aurora"], "id": "c02", "label": "Aurora University", "source_file": "corpus/doc-alpha.txt"}, + {"id": "c03", "label": "Aster City", "source_file": "corpus/doc-alpha.txt"}, + {"aliases": ["Borealis"], "id": "c04", "label": "Borealis Republic", "source_file": "corpus/doc-alpha.txt"}, + {"id": "c05", "label": "1998", "source_file": "corpus/doc-alpha.txt"}, + {"id": "c06", "label": "Leo Hart", "source_file": "corpus/doc-beta.txt"}, + {"aliases": ["Quill"], "id": "c07", "label": "Quill Labs", "source_file": "corpus/doc-beta.txt"}, + {"aliases": ["Orion"], "id": "c08", "label": "Orion Database", "source_file": "corpus/doc-beta.txt"}, + {"id": "c09", "label": "Cedar City", "source_file": "corpus/doc-beta.txt"}, + {"id": "c10", "label": "2012", "source_file": "corpus/doc-beta.txt"}, + {"id": "c11", "label": "Nia Vale", "source_file": "corpus/doc-gamma.txt"}, + {"aliases": ["Silver"], "id": "c12", "label": "Silver Observatory", "source_file": "corpus/doc-gamma.txt"}, + {"aliases": ["Helios"], "id": "c13", "label": "Helios Comet", "source_file": "corpus/doc-gamma.txt"}, + {"id": "c14", "label": "Ridge Province", "source_file": "corpus/doc-gamma.txt"}, + {"id": "c15", "label": "2004", "source_file": "corpus/doc-gamma.txt"}, + {"id": "c16", "label": "Oren Pike", "source_file": "corpus/doc-delta.txt"}, + {"aliases": ["Bluebird"], "id": "c17", "label": "Bluebird Press", "source_file": "corpus/doc-delta.txt"}, + {"aliases": ["Ember"], "id": "c18", "label": "Ember Novel", "source_file": "corpus/doc-delta.txt"}, + {"id": "c19", "label": "Lumen Town", "source_file": "corpus/doc-delta.txt"}, + {"id": "c20", "label": "2018", "source_file": "corpus/doc-delta.txt"} + ], + "links": [ + {"relation": "studied_at", "source": "c01", "target": "c02"}, + {"relation": "located_in", "source": "c02", "target": "c03"}, + {"relation": "capital_of", "source": "c03", "target": "c04"}, + {"relation": "founded_in", "source": "c02", "target": "c05"}, + {"relation": "citizen_of", "source": "c01", "target": "c04"}, + {"relation": "contains", "source": "c04", "target": "c03"}, + {"relation": "born_in", "source": "c01", "target": "c03"}, + {"relation": "enrolls", "source": "c02", "target": "c01"}, + {"relation": "founded", "source": "c06", "target": "c07"}, + {"relation": "created", "source": "c07", "target": "c08"}, + {"relation": "headquartered_at", "source": "c07", "target": "c09"}, + {"relation": "released_in", "source": "c08", "target": "c10"}, + {"relation": "lives_in", "source": "c06", "target": "c09"}, + {"relation": "hosts", "source": "c09", "target": "c07"}, + {"relation": "developed_by", "source": "c08", "target": "c07"}, + {"relation": "created", "source": "c06", "target": "c08"}, + {"relation": "works_at", "source": "c11", "target": "c12"}, + {"relation": "discovered", "source": "c12", "target": "c13"}, + {"relation": "located_in", "source": "c12", "target": "c14"}, + {"relation": "discovered_in", "source": "c13", "target": "c15"}, + {"relation": "observed", "source": "c11", "target": "c13"}, + {"relation": "contains", "source": "c14", "target": "c12"}, + {"relation": "observed_by", "source": "c13", "target": "c11"}, + {"relation": "lives_in", "source": "c11", "target": "c14"}, + {"relation": "wrote", "source": "c16", "target": "c18"}, + {"relation": "published", "source": "c17", "target": "c18"}, + {"relation": "located_in", "source": "c17", "target": "c19"}, + {"relation": "published_in", "source": "c18", "target": "c20"}, + {"relation": "lives_in", "source": "c16", "target": "c19"}, + {"relation": "hosts", "source": "c19", "target": "c17"}, + {"relation": "author", "source": "c18", "target": "c16"}, + {"relation": "works_with", "source": "c16", "target": "c17"} + ] +} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_predictions.jsonl b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_predictions.jsonl new file mode 100644 index 0000000..82ed6ce --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/candidate_predictions.jsonl @@ -0,0 +1,8 @@ +{"answer":"Aurora University","case_id":"qa-01","retrieved_nodes":["Aurora University"],"retrieved_supporting_facts":["doc-alpha#0"]} +{"answer":"Borealis Republic","case_id":"qa-02","retrieved_nodes":["Borealis Republic","Aster City","Aurora University"],"retrieved_supporting_facts":["doc-alpha#0","doc-alpha#1","doc-alpha#2"]} +{"answer":"Orion Database","case_id":"qa-03","retrieved_nodes":["Orion Database","Quill Labs"],"retrieved_supporting_facts":["doc-beta#0","doc-beta#1"]} +{"answer":"2012","case_id":"qa-04","retrieved_nodes":["2012"],"retrieved_supporting_facts":["doc-beta#2"]} +{"answer":"Helios Comet","case_id":"qa-05","retrieved_nodes":["Helios Comet","Silver Observatory"],"retrieved_supporting_facts":["doc-gamma#0","doc-gamma#1"]} +{"answer":"Ridge Province","case_id":"qa-06","retrieved_nodes":["Ridge Province"],"retrieved_supporting_facts":["doc-gamma#2"]} +{"answer":"Ember Novel","case_id":"qa-07","retrieved_nodes":["Ember Novel"],"retrieved_supporting_facts":["doc-delta#0"]} +{"answer":"Lumen Town","case_id":"qa-08","retrieved_nodes":["Lumen Town","Bluebird Press"],"retrieved_supporting_facts":["doc-delta#0","doc-delta#1"]} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-alpha.txt b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-alpha.txt new file mode 100644 index 0000000..f7e89de --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-alpha.txt @@ -0,0 +1,3 @@ +Mira Chen studied at Aurora University and was born in Aster City. +Aurora University is located in Aster City and was founded in 1998. +Aster City is the capital of the Borealis Republic. diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-beta.txt b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-beta.txt new file mode 100644 index 0000000..acc1dc0 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-beta.txt @@ -0,0 +1,3 @@ +Leo Hart founded Quill Labs in Cedar City. +Quill Labs created the Orion Database and is headquartered in Cedar City. +Orion Database was released in 2012. diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-delta.txt b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-delta.txt new file mode 100644 index 0000000..3b7b56c --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-delta.txt @@ -0,0 +1,3 @@ +Oren Pike wrote Ember Novel and works with Bluebird Press. +Bluebird Press published Ember Novel in 2018 and is located in Lumen Town. +Oren Pike lives in Lumen Town. diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-gamma.txt b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-gamma.txt new file mode 100644 index 0000000..dc670b5 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/corpus/doc-gamma.txt @@ -0,0 +1,3 @@ +Nia Vale works at Silver Observatory in Ridge Province. +Silver Observatory discovered Helios Comet in 2004. +Nia Vale observed Helios Comet and lives in Ridge Province. diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/queries.jsonl b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/queries.jsonl new file mode 100644 index 0000000..3280a10 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/queries.jsonl @@ -0,0 +1,8 @@ +{"case_id":"qa-01","question":"Where did Mira Chen study?"} +{"case_id":"qa-02","question":"Which country contains the university where Mira studied?"} +{"case_id":"qa-03","question":"What database was created by the company Leo founded?"} +{"case_id":"qa-04","question":"When was Orion Database released?"} +{"case_id":"qa-05","question":"What comet was discovered by Nia's observatory?"} +{"case_id":"qa-06","question":"Where is Silver Observatory located?"} +{"case_id":"qa-07","question":"What novel did Oren Pike write?"} +{"case_id":"qa-08","question":"Where is the publisher that works with Oren located?"} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.json b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.json new file mode 100644 index 0000000..d22e609 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.json @@ -0,0 +1,218 @@ +{ + "cases": [ + { + "directed": true, + "gold_edges": [ + {"relation": "studied at", "source": "d1e1", "target": "d1e2"}, + {"relation": "located in", "source": "d1e2", "target": "d1e3"}, + {"relation": "capital of", "source": "d1e3", "target": "d1e4"}, + {"relation": "founded in", "source": "d1e2", "target": "d1e5"}, + {"relation": "citizen of", "source": "d1e1", "target": "d1e4"}, + {"relation": "contains", "source": "d1e4", "target": "d1e3"}, + {"relation": "born in", "source": "d1e1", "target": "d1e3"}, + {"relation": "enrolls", "source": "d1e2", "target": "d1e1"} + ], + "gold_entities": [ + {"aliases": ["M. Chen"], "id": "d1e1", "label": "Mira Chen", "source_file": "doc-alpha.txt"}, + {"aliases": ["Aurora"], "id": "d1e2", "label": "Aurora University", "source_file": "doc-alpha.txt"}, + {"id": "d1e3", "label": "Aster City", "source_file": "doc-alpha.txt"}, + {"aliases": ["Borealis"], "id": "d1e4", "label": "Borealis Republic", "source_file": "doc-alpha.txt"}, + {"id": "d1e5", "label": "1998", "source_file": "doc-alpha.txt"} + ], + "id": "doc-alpha", + "source_files": ["doc-alpha.txt"], + "task": "intrinsic", + "title": "Aurora University" + }, + { + "directed": true, + "gold_edges": [ + {"relation": "founded", "source": "d2e1", "target": "d2e2"}, + {"relation": "created", "source": "d2e2", "target": "d2e3"}, + {"accepted_relations": ["headquartered at"], "relation": "headquartered in", "source": "d2e2", "target": "d2e4"}, + {"relation": "released in", "source": "d2e3", "target": "d2e5"}, + {"relation": "lives in", "source": "d2e1", "target": "d2e4"}, + {"relation": "hosts", "source": "d2e4", "target": "d2e2"}, + {"relation": "developed by", "source": "d2e3", "target": "d2e2"}, + {"relation": "created", "source": "d2e1", "target": "d2e3"} + ], + "gold_entities": [ + {"id": "d2e1", "label": "Leo Hart", "source_file": "doc-beta.txt"}, + {"aliases": ["Quill"], "id": "d2e2", "label": "Quill Labs", "source_file": "doc-beta.txt"}, + {"aliases": ["Orion"], "id": "d2e3", "label": "Orion Database", "source_file": "doc-beta.txt"}, + {"id": "d2e4", "label": "Cedar City", "source_file": "doc-beta.txt"}, + {"id": "d2e5", "label": "2012", "source_file": "doc-beta.txt"} + ], + "id": "doc-beta", + "source_files": ["doc-beta.txt"], + "task": "intrinsic", + "title": "Quill Labs" + }, + { + "directed": true, + "gold_edges": [ + {"relation": "works at", "source": "d3e1", "target": "d3e2"}, + {"relation": "discovered", "source": "d3e2", "target": "d3e3"}, + {"relation": "located in", "source": "d3e2", "target": "d3e4"}, + {"relation": "discovered in", "source": "d3e3", "target": "d3e5"}, + {"relation": "observed", "source": "d3e1", "target": "d3e3"}, + {"relation": "contains", "source": "d3e4", "target": "d3e2"}, + {"relation": "observed by", "source": "d3e3", "target": "d3e1"}, + {"relation": "lives in", "source": "d3e1", "target": "d3e4"} + ], + "gold_entities": [ + {"id": "d3e1", "label": "Nia Vale", "source_file": "doc-gamma.txt"}, + {"aliases": ["Silver"], "id": "d3e2", "label": "Silver Observatory", "source_file": "doc-gamma.txt"}, + {"aliases": ["Helios"], "id": "d3e3", "label": "Helios Comet", "source_file": "doc-gamma.txt"}, + {"id": "d3e4", "label": "Ridge Province", "source_file": "doc-gamma.txt"}, + {"id": "d3e5", "label": "2004", "source_file": "doc-gamma.txt"} + ], + "id": "doc-gamma", + "source_files": ["doc-gamma.txt"], + "task": "intrinsic", + "title": "Silver Observatory" + }, + { + "directed": true, + "gold_edges": [ + {"relation": "wrote", "source": "d4e1", "target": "d4e3"}, + {"relation": "published", "source": "d4e2", "target": "d4e3"}, + {"relation": "located in", "source": "d4e2", "target": "d4e4"}, + {"relation": "published in", "source": "d4e3", "target": "d4e5"}, + {"relation": "lives in", "source": "d4e1", "target": "d4e4"}, + {"relation": "hosts", "source": "d4e4", "target": "d4e2"}, + {"relation": "author", "source": "d4e3", "target": "d4e1"}, + {"relation": "works with", "source": "d4e1", "target": "d4e2"} + ], + "gold_entities": [ + {"id": "d4e1", "label": "Oren Pike", "source_file": "doc-delta.txt"}, + {"aliases": ["Bluebird"], "id": "d4e2", "label": "Bluebird Press", "source_file": "doc-delta.txt"}, + {"aliases": ["Ember"], "id": "d4e3", "label": "Ember Novel", "source_file": "doc-delta.txt"}, + {"id": "d4e4", "label": "Lumen Town", "source_file": "doc-delta.txt"}, + {"id": "d4e5", "label": "2018", "source_file": "doc-delta.txt"} + ], + "id": "doc-delta", + "source_files": ["doc-delta.txt"], + "task": "intrinsic", + "title": "Bluebird Press" + }, + { + "answer_entities": ["Aurora University"], + "answers": ["Aurora University"], + "evidence_edges": [ + {"relation": "studied at", "source": "Mira Chen", "target": "Aurora University"} + ], + "id": "qa-01", + "max_hops": 1, + "question": "Where did Mira Chen study?", + "seed_entities": ["Mira Chen"], + "supporting_facts": ["doc-alpha#0"], + "task": "downstream" + }, + { + "answer_entities": ["Borealis Republic", "Borealis"], + "answers": ["Borealis Republic"], + "evidence_edges": [ + {"relation": "studied at", "source": "Mira Chen", "target": "Aurora University"}, + {"relation": "located in", "source": "Aurora University", "target": "Aster City"}, + {"relation": "capital of", "source": "Aster City", "target": "Borealis Republic"} + ], + "id": "qa-02", + "max_hops": 3, + "question": "Which country contains the university where Mira studied?", + "seed_entities": ["Mira Chen"], + "supporting_facts": ["doc-alpha#0", "doc-alpha#1", "doc-alpha#2"], + "task": "downstream" + }, + { + "answer_entities": ["Orion Database", "Orion"], + "answers": ["Orion Database"], + "evidence_edges": [ + {"relation": "founded", "source": "Leo Hart", "target": "Quill Labs"}, + {"relation": "created", "source": "Quill Labs", "target": "Orion Database"} + ], + "id": "qa-03", + "max_hops": 2, + "question": "What database was created by the company Leo founded?", + "seed_entities": ["Leo Hart"], + "supporting_facts": ["doc-beta#0", "doc-beta#1"], + "task": "downstream" + }, + { + "answer_entities": ["2012"], + "answers": ["2012"], + "evidence_edges": [ + {"relation": "released in", "source": "Orion Database", "target": "2012"} + ], + "id": "qa-04", + "max_hops": 1, + "question": "When was Orion Database released?", + "seed_entities": ["Orion Database"], + "supporting_facts": ["doc-beta#2"], + "task": "downstream" + }, + { + "answer_entities": ["Helios Comet", "Helios"], + "answers": ["Helios Comet"], + "evidence_edges": [ + {"relation": "works at", "source": "Nia Vale", "target": "Silver Observatory"}, + {"relation": "discovered", "source": "Silver Observatory", "target": "Helios Comet"} + ], + "id": "qa-05", + "max_hops": 2, + "question": "What comet was discovered by Nia's observatory?", + "seed_entities": ["Nia Vale"], + "supporting_facts": ["doc-gamma#0", "doc-gamma#1"], + "task": "downstream" + }, + { + "answer_entities": ["Ridge Province"], + "answers": ["Ridge Province"], + "evidence_edges": [ + {"relation": "located in", "source": "Silver Observatory", "target": "Ridge Province"} + ], + "id": "qa-06", + "max_hops": 1, + "question": "Where is Silver Observatory located?", + "seed_entities": ["Silver Observatory"], + "supporting_facts": ["doc-gamma#2"], + "task": "downstream" + }, + { + "answer_entities": ["Ember Novel", "Ember"], + "answers": ["Ember Novel"], + "evidence_edges": [ + {"relation": "wrote", "source": "Oren Pike", "target": "Ember Novel"} + ], + "id": "qa-07", + "max_hops": 1, + "question": "What novel did Oren Pike write?", + "seed_entities": ["Oren Pike"], + "supporting_facts": ["doc-delta#0"], + "task": "downstream" + }, + { + "answer_entities": ["Lumen Town"], + "answers": ["Lumen Town"], + "evidence_edges": [ + {"relation": "works with", "source": "Oren Pike", "target": "Bluebird Press"}, + {"relation": "located in", "source": "Bluebird Press", "target": "Lumen Town"} + ], + "id": "qa-08", + "max_hops": 2, + "question": "Where is the publisher that works with Oren located?", + "seed_entities": ["Oren Pike"], + "supporting_facts": ["doc-delta#0", "doc-delta#1"], + "task": "downstream" + } + ], + "profile": "smoke", + "schema_version": 1, + "source": { + "dataset": "Self-authored synthetic fixture", + "license": "MIT", + "provenance": "DeepRefine-Skill test data; not a performance claim" + }, + "suite_id": "synthetic-smoke-v1", + "suite_version": "1" +} diff --git a/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.lock.json b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.lock.json new file mode 100644 index 0000000..8211956 --- /dev/null +++ b/deeprefine_skill/benchmark_suites/synthetic-smoke-v1/suite.lock.json @@ -0,0 +1,22 @@ +{ + "files": { + "baseline_graph.json": "edb8203410356b650b4027f30f2388c2f18f1ad4c6ae7f53d87080ac6f2c1fc6", + "baseline_predictions.jsonl": "ec17daaf34783aa04a2f7bff98c2d4e4b536a17cd22c2d6803ab2463374d029d", + "candidate_graph.json": "683e80be81fa6dcdc7c6eededcaa528fd023d9f8ca69ef180a776ca6f9a5f3d1", + "candidate_predictions.jsonl": "f9a8d4364c28dd50d903de41eec844f690198db869950efadce3c5879e144c2a", + "corpus/doc-alpha.txt": "1e2a7d558baf4f27da169a0e53f5439f4f91bf5684aa00e3e75e6d92b7a958fa", + "corpus/doc-beta.txt": "e486d3e14f6b6188d4055c13c4a329520f9420543b0e92a75b942c4ad5d63d64", + "corpus/doc-delta.txt": "438726bc79e78c00433fe9ea8728a13ed337d610843e0421362baae444cd7f27", + "corpus/doc-gamma.txt": "0b74f07bd31be3d9b4f183cbb60cdf66703d30afa2689123634ce25d1014fb8d", + "queries.jsonl": "f5f9bb8cdb2dd4f8b945aadac747e471d336e35164f2b8997b28cf53cacbe457", + "suite.json": "3071ebe7026790274c8c85216e343a9f1249f03cbb3a91b99f578b65f4019f8b" + }, + "profile": "smoke", + "schema_version": 1, + "selection_seed": "deeprefine-benchmark-v1", + "source": { + "dataset": "Self-authored synthetic fixture", + "license": "MIT" + }, + "suite_id": "synthetic-smoke-v1" +} diff --git a/deeprefine_skill/benchmarking/__init__.py b/deeprefine_skill/benchmarking/__init__.py new file mode 100644 index 0000000..5537f07 --- /dev/null +++ b/deeprefine_skill/benchmarking/__init__.py @@ -0,0 +1,12 @@ +"""Offline, reproducible benchmarks for Graphify/DeepRefine graphs. + +The default benchmark implementation intentionally depends only on the Python +standard library. Public helpers are imported lazily by the CLI so the +existing DeepRefine commands keep their current dependency footprint. +""" + +from .evaluator import evaluate_suite +from .prepare import prepare_suite +from .report import render_markdown + +__all__ = ["evaluate_suite", "prepare_suite", "render_markdown"] diff --git a/deeprefine_skill/benchmarking/cli.py b/deeprefine_skill/benchmarking/cli.py new file mode 100644 index 0000000..c6f551a --- /dev/null +++ b/deeprefine_skill/benchmarking/cli.py @@ -0,0 +1,190 @@ +"""Argument-parser integration for ``deeprefine benchmark``.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from .evaluator import evaluate_suite +from .prepare import SUPPORTED_SUITES, prepare_suite +from .report import render_markdown + + +def _fail(exc: Exception) -> int: + print(f"benchmark: {exc}", file=sys.stderr) + return 2 + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def cmd_benchmark_prepare(args: argparse.Namespace) -> int: + """Prepare a deterministic suite from an official upstream data file.""" + + try: + profile = args.profile or ( + "smoke" if args.suite == "synthetic-smoke-v1" else "quick" + ) + destination = prepare_suite( + args.suite, + profile, + args.output_dir, + source=args.source, + ) + except (OSError, ValueError) as exc: + return _fail(exc) + print(f"Prepared benchmark suite: {destination}") + print(f"Suite manifest: {destination / 'suite.json'}") + return 0 + + +def _metadata(args: argparse.Namespace) -> dict[str, Any]: + return { + "graphify_version": args.graphify_version, + "deeprefine_version": args.deeprefine_version, + "model": args.model, + "temperature": args.temperature, + "prompt_config_hash": args.prompt_config_hash, + "llm_calls": args.llm_calls, + "input_tokens": args.input_tokens, + "output_tokens": args.output_tokens, + } + + +def cmd_benchmark_evaluate(args: argparse.Namespace) -> int: + """Evaluate a graph pair and write result.json plus report.md.""" + + try: + result = evaluate_suite( + args.suite, + args.baseline_graph, + args.candidate_graph, + baseline_predictions=args.baseline_predictions, + candidate_predictions=args.candidate_predictions, + baseline_wiki=args.baseline_wiki, + candidate_wiki=args.candidate_wiki, + semantic_model=args.semantic_model, + metadata=_metadata(args), + ) + output_dir = Path(args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + result_path = output_dir / "result.json" + report_path = output_dir / "report.md" + _write_json(result_path, result) + report_path.write_text(render_markdown(result), encoding="utf-8") + except (OSError, RuntimeError, TypeError, ValueError) as exc: + return _fail(exc) + print(f"Benchmark result: {result_path}") + print(f"Markdown report: {report_path}") + return 0 + + +def cmd_benchmark_report(args: argparse.Namespace) -> int: + """Merge one or more result JSON files into a Markdown report.""" + + results: list[dict[str, Any]] = [] + try: + for value in args.result: + path = Path(value) + loaded = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(loaded, dict) or loaded.get("schema_version") != 1: + raise ValueError(f"Unsupported benchmark result: {path}") + results.append(loaded) + markdown = render_markdown(results) + if args.output == "-": + print(markdown, end="") + else: + output = Path(args.output).resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(markdown, encoding="utf-8") + print(f"Markdown report: {output}") + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + return _fail(exc) + return 0 + + +def register_benchmark_commands(subparsers: Any) -> None: + """Register the benchmark command group on an argparse subparser action.""" + + parser = subparsers.add_parser( + "benchmark", + help="Prepare and evaluate lightweight graph-quality benchmarks", + ) + commands = parser.add_subparsers(dest="benchmark_cmd", required=True) + + prepare = commands.add_parser( + "prepare", + help="Prepare a deterministic suite from upstream data", + ) + prepare.add_argument("--suite", required=True, choices=sorted(SUPPORTED_SUITES)) + prepare.add_argument( + "--profile", + default=None, + choices=("smoke", "quick", "readme"), + help="Default: smoke for synthetic, quick for real suites", + ) + prepare.add_argument( + "--source", + default=None, + help="Official upstream JSON file (not needed for synthetic-smoke-v1)", + ) + prepare.add_argument("--output-dir", required=True) + prepare.set_defaults(func=cmd_benchmark_prepare) + + evaluate = commands.add_parser( + "evaluate", + help="Compare baseline and candidate Graphify graph.json files", + ) + evaluate.add_argument( + "--suite", + required=True, + help="Prepared suite directory, suite.json, or built-in suite ID", + ) + evaluate.add_argument("--baseline-graph", required=True) + evaluate.add_argument("--candidate-graph", required=True) + evaluate.add_argument("--baseline-predictions", default=None) + evaluate.add_argument("--candidate-predictions", default=None) + evaluate.add_argument( + "--baseline-wiki", + default=None, + help="Optional baseline Wiki directory for local-link integrity checks", + ) + evaluate.add_argument( + "--candidate-wiki", + default=None, + help="Optional candidate Wiki directory for local-link integrity checks", + ) + evaluate.add_argument( + "--semantic-model", + default=None, + help="Opt in to G-BERTScore with this bert-score model (for example roberta-large)", + ) + evaluate.add_argument("--output-dir", required=True) + evaluate.add_argument("--graphify-version", default=None) + evaluate.add_argument("--deeprefine-version", default=None) + evaluate.add_argument("--model", default=None) + evaluate.add_argument("--temperature", type=float, default=None) + evaluate.add_argument("--prompt-config-hash", default=None) + evaluate.add_argument("--llm-calls", type=int, default=0) + evaluate.add_argument("--input-tokens", type=int, default=0) + evaluate.add_argument("--output-tokens", type=int, default=0) + evaluate.set_defaults(func=cmd_benchmark_evaluate) + + report = commands.add_parser( + "report", + help="Render one or more result.json files as Markdown", + ) + report.add_argument("--result", action="append", required=True) + report.add_argument("--format", choices=("markdown",), default="markdown") + report.add_argument("--output", required=True, help="Output file or '-' for stdout") + report.set_defaults(func=cmd_benchmark_report) diff --git a/deeprefine_skill/benchmarking/evaluator.py b/deeprefine_skill/benchmarking/evaluator.py new file mode 100644 index 0000000..235b56d --- /dev/null +++ b/deeprefine_skill/benchmarking/evaluator.py @@ -0,0 +1,683 @@ +"""Intrinsic and downstream evaluation for prepared benchmark suites.""" + +from __future__ import annotations + +import platform +import sys +import time +from pathlib import Path +from statistics import fmean +from typing import Any, Iterable, Mapping, Sequence + +from deeprefine_skill import __version__ + +from .graph import ( + GraphData, + GraphNode, + align_entities, + bounded_bfs, + canonical_edge, + load_graphify_graph, + normalize_relation, + normalize_text, + source_matches, +) +from .metrics import ( + answer_exact_match, + answer_token_f1, + bertscore_graph_score, + hit_at_k, + lexical_graph_score, + precision_recall_f1, + reciprocal_rank, + supporting_recall, + triple_sentence, +) +from .suite import load_predictions, load_suite, sha256_file +from .wiki import inspect_wiki_directory + + +def _list_strings(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + return [str(item) for item in value] + return [str(value)] + + +def _gold_nodes(case: Mapping[str, Any]) -> list[GraphNode]: + result: list[GraphNode] = [] + for index, item in enumerate(case.get("gold_entities", [])): + if not isinstance(item, Mapping): + raise ValueError(f"{case['id']}: gold_entities[{index}] must be an object") + node_id = str(item.get("id", "")).strip() + if not node_id: + raise ValueError(f"{case['id']}: gold_entities[{index}] is missing id") + result.append( + GraphNode( + id=node_id, + label=str(item.get("label", node_id)), + aliases=tuple(_list_strings(item.get("aliases"))), + source_file=str(item.get("source_file", "")), + data=dict(item), + ) + ) + return result + + +def _scope_node_ids( + case: Mapping[str, Any], + graph: GraphData, + aligned_predicted_ids: Iterable[str], + *, + only_intrinsic_case: bool, +) -> set[str]: + sources = _list_strings(case.get("source_files")) + source_scoped = { + node.id + for node in graph.nodes.values() + if node.source_file + and any(source_matches(node.source_file, source) for source in sources) + } + if source_scoped: + return source_scoped.union(aligned_predicted_ids) + if only_intrinsic_case: + return set(graph.nodes) + return set(aligned_predicted_ids) + + +def _edge_relation_options(item: Mapping[str, Any]) -> set[str]: + values = [item.get("relation", ""), *_list_strings(item.get("accepted_relations"))] + return {normalize_relation(value) for value in values if normalize_relation(value)} + + +def _gold_edge_records( + case: Mapping[str, Any], + *, + directed: bool, +) -> list[tuple[str, str, str, set[str]]]: + records: list[tuple[str, str, str, set[str]]] = [] + for index, item in enumerate(case.get("gold_edges", [])): + if not isinstance(item, Mapping): + raise ValueError(f"{case['id']}: gold_edges[{index}] must be an object") + source = str(item.get("source", "")).strip() + target = str(item.get("target", "")).strip() + relations = _edge_relation_options(item) + if not source or not target or not relations: + raise ValueError( + f"{case['id']}: gold_edges[{index}] needs source, relation, and target" + ) + canonical_source, _, canonical_target = canonical_edge( + source, + "", + target, + directed=directed, + ) + primary = normalize_relation(item.get("relation", "")) + records.append((canonical_source, primary, canonical_target, relations)) + return records + + +def _intrinsic_case( + case: Mapping[str, Any], + graph: GraphData, + *, + only_intrinsic_case: bool, + semantic_model: str | None, +) -> dict[str, Any]: + gold_nodes = _gold_nodes(case) + alignment = align_entities(gold_nodes, graph) + reverse_alignment = {predicted: gold for gold, predicted in alignment.items()} + scope_ids = _scope_node_ids( + case, + graph, + alignment.values(), + only_intrinsic_case=only_intrinsic_case, + ) + + gold_entity_ids = {node.id for node in gold_nodes} + predicted_entity_keys = { + reverse_alignment.get(node_id, f"predicted::{node_id}") for node_id in scope_ids + } + entity_tp = len(gold_entity_ids.intersection(predicted_entity_keys)) + entity_prf = precision_recall_f1( + entity_tp, + len(predicted_entity_keys - gold_entity_ids), + len(gold_entity_ids - predicted_entity_keys), + ) + + directed = bool(case.get("directed", True)) + gold_edges = _gold_edge_records(case, directed=directed) + predicted_edges: set[tuple[str, str, str]] = set() + predicted_lexical_by_edge: dict[tuple[str, str, str], str] = {} + for edge in graph.edges: + if edge.source not in scope_ids or edge.target not in scope_ids: + continue + mapped_source = reverse_alignment.get(edge.source, f"predicted::{edge.source}") + mapped_target = reverse_alignment.get(edge.target, f"predicted::{edge.target}") + edge_key = canonical_edge( + mapped_source, + edge.relation, + mapped_target, + directed=directed, + ) + predicted_edges.add(edge_key) + predicted_lexical_by_edge.setdefault( + edge_key, + triple_sentence( + ( + graph.nodes[edge.source].label, + edge.relation, + graph.nodes[edge.target].label, + ) + ) + ) + + matched_predicted: set[tuple[str, str, str]] = set() + matched_gold_indices: set[int] = set() + for predicted_edge in sorted(predicted_edges): + for index, (source, _primary, target, relation_options) in enumerate(gold_edges): + if index in matched_gold_indices: + continue + if ( + predicted_edge[0] == source + and predicted_edge[2] == target + and predicted_edge[1] in relation_options + ): + matched_predicted.add(predicted_edge) + matched_gold_indices.add(index) + break + + edge_prf = precision_recall_f1( + len(matched_predicted), + len(predicted_edges) - len(matched_predicted), + len(gold_edges) - len(matched_gold_indices), + ) + + gold_by_id = {node.id: node for node in gold_nodes} + gold_lexical = [ + triple_sentence( + ( + gold_by_id[source].label, + primary, + gold_by_id[target].label, + ) + ) + for source, primary, target, _relations in gold_edges + if source in gold_by_id and target in gold_by_id + ] + + missing_entities = sorted(gold_entity_ids - set(alignment)) + spurious_entities = sorted(scope_ids - set(reverse_alignment)) + missing_edges = [ + { + "source": source, + "relation": primary, + "target": target, + } + for index, (source, primary, target, _relations) in enumerate(gold_edges) + if index not in matched_gold_indices + ] + spurious_edges = [ + {"source": source, "relation": relation, "target": target} + for source, relation, target in sorted(predicted_edges - matched_predicted) + ] + + predicted_lexical = [ + predicted_lexical_by_edge[key] for key in sorted(predicted_lexical_by_edge) + ] + result = { + "entity": entity_prf.as_dict(), + "strict_triple": edge_prf.as_dict(), + "g_bleu": lexical_graph_score(predicted_lexical, gold_lexical, metric="bleu"), + "g_rouge": lexical_graph_score(predicted_lexical, gold_lexical, metric="rouge"), + "alignment": alignment, + "missing_entities": missing_entities, + "spurious_entities": spurious_entities, + "missing_edges": missing_edges, + "spurious_edges": spurious_edges, + } + if semantic_model: + result["g_bertscore"] = bertscore_graph_score( + predicted_lexical, + gold_lexical, + model_type=semantic_model, + ) + return result + + +def _aliases_from_endpoint(value: object) -> list[str]: + if isinstance(value, Mapping): + values = [ + value.get("label", ""), + *_list_strings(value.get("aliases")), + ] + return [str(item) for item in values if str(item).strip()] + return _list_strings(value) + + +def _edge_is_present(graph: GraphData, item: Mapping[str, Any]) -> bool: + source_ids = set(graph.node_ids_for_aliases(_aliases_from_endpoint(item.get("source")))) + target_ids = set(graph.node_ids_for_aliases(_aliases_from_endpoint(item.get("target")))) + relations = _edge_relation_options(item) + for edge in graph.edges: + relation = normalize_relation(edge.relation) + if relation not in relations: + continue + if edge.source in source_ids and edge.target in target_ids: + return True + if not graph.directed and edge.target in source_ids and edge.source in target_ids: + return True + return False + + +def _prediction_metrics( + prediction: Mapping[str, Any] | None, + case: Mapping[str, Any], +) -> dict[str, float]: + if not prediction: + return {} + result: dict[str, float] = {} + answers = _list_strings(case.get("answers")) + if "answer" in prediction and answers: + result["answer_em"] = answer_exact_match(prediction["answer"], answers) + result["answer_f1"] = answer_token_f1(prediction["answer"], answers) + + ranked_nodes = _list_strings(prediction.get("retrieved_nodes")) + answer_entities = _list_strings(case.get("answer_entities")) or answers + if "retrieved_nodes" in prediction and answer_entities: + result["hit_at_1"] = hit_at_k(ranked_nodes, answer_entities, 1) + result["hit_at_5"] = hit_at_k(ranked_nodes, answer_entities, 5) + result["mrr"] = reciprocal_rank(ranked_nodes, answer_entities) + + retrieved_facts = _list_strings(prediction.get("retrieved_supporting_facts")) + supporting_facts = _list_strings(case.get("supporting_facts")) + if "retrieved_supporting_facts" in prediction and supporting_facts: + result["supporting_fact_recall_at_2"] = supporting_recall( + retrieved_facts, + supporting_facts, + k=2, + ) + result["supporting_fact_recall_at_5"] = supporting_recall( + retrieved_facts, + supporting_facts, + k=5, + ) + return result + + +def _downstream_case( + case: Mapping[str, Any], + graph: GraphData, + prediction: Mapping[str, Any] | None, +) -> dict[str, Any]: + evidence = case.get("evidence_edges", []) + if not isinstance(evidence, list): + raise ValueError(f"{case['id']}: evidence_edges must be an array") + evidence_hits = [ + bool(isinstance(item, Mapping) and _edge_is_present(graph, item)) + for item in evidence + ] + evidence_recall = ( + sum(evidence_hits) / len(evidence_hits) if evidence_hits else 1.0 + ) + complete_path = bool(all(evidence_hits)) + + seed_ids = graph.node_ids_for_aliases(_list_strings(case.get("seed_entities"))) + answer_ids = set( + graph.node_ids_for_aliases( + _list_strings(case.get("answer_entities")) + or _list_strings(case.get("answers")) + ) + ) + max_hops = int(case.get("max_hops", max(1, len(evidence)))) + reachable = bounded_bfs(graph, seed_ids, max_hops) + answer_reachable = bool(answer_ids.intersection(reachable)) + + result: dict[str, Any] = { + "evidence_edge_recall": evidence_recall, + "evidence_hits": evidence_hits, + "evidence_hit_count": sum(evidence_hits), + "evidence_edge_count": len(evidence_hits), + "complete_path": complete_path, + "answer_reachable": answer_reachable, + "max_hops": max_hops, + } + optional_metrics = _prediction_metrics(prediction, case) + if optional_metrics: + result["prediction"] = optional_metrics + return result + + +def _mean(values: Iterable[float]) -> float | None: + materialized = list(values) + return fmean(materialized) if materialized else None + + +def _aggregate_side( + cases: Sequence[Mapping[str, Any]], + graph: GraphData, + predictions: Mapping[str, Mapping[str, Any]], + *, + semantic_model: str | None, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + intrinsic_cases = [case for case in cases if case["task"] == "intrinsic"] + case_results: dict[str, dict[str, Any]] = {} + for case in cases: + if case["task"] == "intrinsic": + case_results[str(case["id"])] = _intrinsic_case( + case, + graph, + only_intrinsic_case=len(intrinsic_cases) == 1, + semantic_model=semantic_model, + ) + else: + case_results[str(case["id"])] = _downstream_case( + case, + graph, + predictions.get(str(case["id"])), + ) + + summary: dict[str, Any] = {"diagnostics": dict(graph.diagnostics)} + if intrinsic_cases: + intrinsic_results = [ + case_results[str(case["id"])] for case in intrinsic_cases + ] + entity_tp = sum(item["entity"]["tp"] for item in intrinsic_results) + entity_fp = sum(item["entity"]["fp"] for item in intrinsic_results) + entity_fn = sum(item["entity"]["fn"] for item in intrinsic_results) + edge_tp = sum(item["strict_triple"]["tp"] for item in intrinsic_results) + edge_fp = sum(item["strict_triple"]["fp"] for item in intrinsic_results) + edge_fn = sum(item["strict_triple"]["fn"] for item in intrinsic_results) + summary["intrinsic"] = { + "case_count": len(intrinsic_results), + "macro": { + "entity_f1": _mean(item["entity"]["f1"] for item in intrinsic_results), + "strict_triple_f1": _mean( + item["strict_triple"]["f1"] for item in intrinsic_results + ), + "g_bleu_f1": _mean(item["g_bleu"]["f1"] for item in intrinsic_results), + "g_rouge_f1": _mean( + item["g_rouge"]["f1"] for item in intrinsic_results + ), + }, + "micro": { + "entity": precision_recall_f1(entity_tp, entity_fp, entity_fn).as_dict(), + "strict_triple": precision_recall_f1( + edge_tp, + edge_fp, + edge_fn, + ).as_dict(), + }, + } + if semantic_model: + summary["intrinsic"]["macro"]["g_bertscore_f1"] = _mean( + item["g_bertscore"]["f1"] for item in intrinsic_results + ) + + downstream_cases = [case for case in cases if case["task"] == "downstream"] + if downstream_cases: + downstream_results = [ + case_results[str(case["id"])] for case in downstream_cases + ] + evidence_hits = sum(item["evidence_hit_count"] for item in downstream_results) + evidence_total = sum(item["evidence_edge_count"] for item in downstream_results) + prediction_names = sorted( + { + name + for item in downstream_results + for name in item.get("prediction", {}) + } + ) + prediction_summary: dict[str, Any] = {} + for name in prediction_names: + values = [ + item["prediction"][name] + for item in downstream_results + if name in item.get("prediction", {}) + ] + prediction_summary[name] = { + "value": _mean(values), + "evaluated_cases": len(values), + } + summary["downstream"] = { + "case_count": len(downstream_results), + "macro": { + "evidence_edge_recall": _mean( + item["evidence_edge_recall"] for item in downstream_results + ), + "complete_path_rate": _mean( + float(item["complete_path"]) for item in downstream_results + ), + "answer_reachability_rate": _mean( + float(item["answer_reachable"]) for item in downstream_results + ), + }, + "micro": { + "evidence_edge_recall": ( + evidence_hits / evidence_total if evidence_total else 1.0 + ), + "evidence_hit_count": evidence_hits, + "evidence_edge_count": evidence_total, + }, + "prediction": prediction_summary or None, + } + return summary, case_results + + +def _headline_delta( + baseline: Mapping[str, Any], + candidate: Mapping[str, Any], +) -> dict[str, Any]: + result: dict[str, Any] = {} + for category in ("intrinsic", "downstream"): + if category not in baseline or category not in candidate: + continue + base_macro = baseline[category].get("macro", {}) + candidate_macro = candidate[category].get("macro", {}) + result[category] = { + name: candidate_macro[name] - base_macro[name] + for name in sorted(set(base_macro).intersection(candidate_macro)) + if isinstance(base_macro[name], (int, float)) + and isinstance(candidate_macro[name], (int, float)) + } + return result + + +def _transitions( + cases: Sequence[Mapping[str, Any]], + baseline: Mapping[str, Mapping[str, Any]], + candidate: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + counts = {"0_to_1": 0, "1_to_0": 0, "1_to_1": 0, "0_to_0": 0} + for case in cases: + if case["task"] != "downstream": + continue + case_id = str(case["id"]) + before = int(bool(baseline[case_id]["complete_path"])) + after = int(bool(candidate[case_id]["complete_path"])) + counts[f"{before}_to_{after}"] += 1 + total = sum(counts.values()) + return { + "complete_path": { + "counts": counts, + "rates": { + name: count / total if total else 0.0 + for name, count in counts.items() + }, + } + } + + +def _case_delta( + task: str, + baseline: Mapping[str, Any], + candidate: Mapping[str, Any], +) -> dict[str, float]: + if task == "intrinsic": + return { + "entity_f1": candidate["entity"]["f1"] - baseline["entity"]["f1"], + "strict_triple_f1": ( + candidate["strict_triple"]["f1"] - baseline["strict_triple"]["f1"] + ), + "g_bleu_f1": candidate["g_bleu"]["f1"] - baseline["g_bleu"]["f1"], + "g_rouge_f1": candidate["g_rouge"]["f1"] - baseline["g_rouge"]["f1"], + } + return { + "evidence_edge_recall": ( + candidate["evidence_edge_recall"] - baseline["evidence_edge_recall"] + ), + "complete_path": float(candidate["complete_path"]) + - float(baseline["complete_path"]), + "answer_reachable": float(candidate["answer_reachable"]) + - float(baseline["answer_reachable"]), + } + + +def evaluate_suite( + suite_path: str | Path | Mapping[str, Any], + baseline_graph: str | Path, + candidate_graph: str | Path, + *, + baseline_predictions: str | Path | None = None, + candidate_predictions: str | Path | None = None, + baseline_wiki: str | Path | None = None, + candidate_wiki: str | Path | None = None, + semantic_model: str | None = None, + metadata: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Evaluate baseline and candidate graphs without modifying either input.""" + + started = time.perf_counter() + suite, _suite_directory = load_suite(suite_path) + baseline_path = Path(baseline_graph).resolve() + candidate_path = Path(candidate_graph).resolve() + before_hashes = { + "baseline": sha256_file(baseline_path), + "candidate": sha256_file(candidate_path), + } + baseline_prediction_hash = ( + sha256_file(baseline_predictions) if baseline_predictions else None + ) + candidate_prediction_hash = ( + sha256_file(candidate_predictions) if candidate_predictions else None + ) + + baseline_data = load_graphify_graph(baseline_path) + candidate_data = load_graphify_graph(candidate_path) + baseline_prediction_data = load_predictions(baseline_predictions) + candidate_prediction_data = load_predictions(candidate_predictions) + cases = suite["cases"] + baseline_summary, baseline_cases = _aggregate_side( + cases, + baseline_data, + baseline_prediction_data, + semantic_model=semantic_model, + ) + candidate_summary, candidate_cases = _aggregate_side( + cases, + candidate_data, + candidate_prediction_data, + semantic_model=semantic_model, + ) + if baseline_wiki: + baseline_summary["wiki"] = inspect_wiki_directory(baseline_wiki) + if candidate_wiki: + candidate_summary["wiki"] = inspect_wiki_directory(candidate_wiki) + + after_hashes = { + "baseline": sha256_file(baseline_path), + "candidate": sha256_file(candidate_path), + } + if before_hashes != after_hashes: + raise RuntimeError("Benchmark input graph changed during evaluation") + + case_results = [ + { + "case_id": str(case["id"]), + "task": case["task"], + "baseline": baseline_cases[str(case["id"])], + "candidate": candidate_cases[str(case["id"])], + "delta": _case_delta( + str(case["task"]), + baseline_cases[str(case["id"])], + candidate_cases[str(case["id"])], + ), + } + for case in cases + ] + source = suite.get("source") if isinstance(suite.get("source"), Mapping) else {} + run_metadata = dict(metadata or {}) + result = { + "schema_version": 1, + "suite": { + "suite_id": suite["suite_id"], + "suite_version": suite["suite_version"], + "profile": suite["profile"], + "source": source, + "case_count": len(cases), + "intrinsic_case_count": sum( + 1 for case in cases if case["task"] == "intrinsic" + ), + "downstream_case_count": sum( + 1 for case in cases if case["task"] == "downstream" + ), + }, + "inputs": { + "baseline_graph": { + "path": str(baseline_path), + "sha256": before_hashes["baseline"], + }, + "candidate_graph": { + "path": str(candidate_path), + "sha256": before_hashes["candidate"], + }, + "baseline_predictions": { + "path": str(Path(baseline_predictions).resolve()), + "sha256": baseline_prediction_hash, + } + if baseline_predictions + else None, + "candidate_predictions": { + "path": str(Path(candidate_predictions).resolve()), + "sha256": candidate_prediction_hash, + } + if candidate_predictions + else None, + "baseline_wiki": str(Path(baseline_wiki).resolve()) + if baseline_wiki + else None, + "candidate_wiki": str(Path(candidate_wiki).resolve()) + if candidate_wiki + else None, + }, + "environment": { + "deeprefine_skill_version": __version__, + "python_version": platform.python_version(), + "platform": sys.platform, + "graphify_version": run_metadata.get("graphify_version"), + "deeprefine_version": run_metadata.get("deeprefine_version"), + "model": run_metadata.get("model"), + "temperature": run_metadata.get("temperature"), + "prompt_config_hash": run_metadata.get("prompt_config_hash"), + "semantic_model": semantic_model, + }, + "cost": { + "llm_calls": int(run_metadata.get("llm_calls") or 0), + "input_tokens": int(run_metadata.get("input_tokens") or 0), + "output_tokens": int(run_metadata.get("output_tokens") or 0), + }, + "metrics": { + "baseline": baseline_summary, + "candidate": candidate_summary, + "delta": _headline_delta(baseline_summary, candidate_summary), + }, + "transitions": _transitions(cases, baseline_cases, candidate_cases), + "cases": case_results, + "runtime": { + "evaluation_seconds": time.perf_counter() - started, + }, + } + return result diff --git a/deeprefine_skill/benchmarking/graph.py b/deeprefine_skill/benchmarking/graph.py new file mode 100644 index 0000000..711cbe7 --- /dev/null +++ b/deeprefine_skill/benchmarking/graph.py @@ -0,0 +1,334 @@ +"""Graphify node-link JSON loading, normalization, and graph traversal.""" + +from __future__ import annotations + +import json +import re +import unicodedata +from collections import Counter, deque +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + + +_WHITESPACE_RE = re.compile(r"\s+") + + +def normalize_text(value: object) -> str: + """Return a stable, case-insensitive representation of a label.""" + + text = unicodedata.normalize("NFKC", str(value or "")) + return _WHITESPACE_RE.sub(" ", text).strip().casefold() + + +def normalize_relation(value: object) -> str: + """Normalize a relation while retaining its word-level meaning.""" + + text = normalize_text(value).replace("_", " ").replace("-", " ") + return _WHITESPACE_RE.sub(" ", text).strip() + + +def normalize_source(value: object) -> str: + """Normalize a source path without requiring it to exist.""" + + text = normalize_text(value).replace("\\", "/") + while "//" in text: + text = text.replace("//", "/") + return text.removeprefix("./") + + +def source_matches(left: str, right: str) -> bool: + """Return whether two normalized source paths identify the same file.""" + + if not left or not right: + return False + left_norm = normalize_source(left) + right_norm = normalize_source(right) + if left_norm == right_norm: + return True + return ( + left_norm.endswith("/" + right_norm) + or right_norm.endswith("/" + left_norm) + or left_norm.rsplit("/", 1)[-1] == right_norm.rsplit("/", 1)[-1] + ) + + +@dataclass(frozen=True) +class GraphNode: + """A normalized view of one Graphify node.""" + + id: str + label: str + aliases: tuple[str, ...] = () + source_file: str = "" + data: Mapping[str, Any] = field(default_factory=dict, compare=False) + + @property + def normalized_aliases(self) -> frozenset[str]: + values = {normalize_text(self.label), *(normalize_text(v) for v in self.aliases)} + values.discard("") + return frozenset(values) + + +@dataclass(frozen=True) +class GraphEdge: + """A normalized view of one Graphify edge.""" + + source: str + target: str + relation: str + data: Mapping[str, Any] = field(default_factory=dict, compare=False) + + +@dataclass +class GraphData: + """A dependency-free representation of Graphify's node-link graph.""" + + nodes: dict[str, GraphNode] + edges: list[GraphEdge] + directed: bool = True + multigraph: bool = True + diagnostics: dict[str, int] = field(default_factory=dict) + + def adjacency(self) -> dict[str, list[str]]: + """Build a deterministic adjacency list, respecting graph direction.""" + + result: dict[str, set[str]] = {node_id: set() for node_id in self.nodes} + for edge in self.edges: + if edge.source not in result or edge.target not in result: + continue + result[edge.source].add(edge.target) + if not self.directed: + result[edge.target].add(edge.source) + return {key: sorted(value) for key, value in result.items()} + + def node_ids_for_aliases(self, aliases: Iterable[object]) -> list[str]: + """Return node IDs matching any normalized alias.""" + + wanted = {normalize_text(alias) for alias in aliases} + wanted.discard("") + return sorted( + node.id + for node in self.nodes.values() + if node.normalized_aliases.intersection(wanted) + ) + + +def _coerce_aliases(value: object) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value,) + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + return tuple(str(item) for item in value if str(item).strip()) + return (str(value),) + + +def load_graphify_graph( + source: str | Path | Mapping[str, Any], + *, + strict: bool = False, +) -> GraphData: + """Load Graphify/NetworkX node-link JSON. + + In non-strict mode malformed edges are skipped and exposed through + diagnostics. Structural errors in the top-level document always raise + ``ValueError``. + """ + + if isinstance(source, Mapping): + raw = dict(source) + else: + path = Path(source) + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Cannot load graph JSON {path}: {exc}") from exc + + raw_nodes = raw.get("nodes") + raw_edges = raw.get("links", raw.get("edges")) + if not isinstance(raw_nodes, list): + raise ValueError("Graph JSON must contain a 'nodes' array") + if not isinstance(raw_edges, list): + raise ValueError("Graph JSON must contain a 'links' or 'edges' array") + + nodes: dict[str, GraphNode] = {} + duplicate_node_ids = 0 + for index, item in enumerate(raw_nodes): + if not isinstance(item, Mapping): + if strict: + raise ValueError(f"nodes[{index}] must be an object") + continue + node_id = str(item.get("id", "")).strip() + if not node_id: + if strict: + raise ValueError(f"nodes[{index}] is missing id") + continue + if node_id in nodes: + duplicate_node_ids += 1 + if strict: + raise ValueError(f"Duplicate node id: {node_id}") + continue + label = str(item.get("label", node_id)) + source_file = str( + item.get("source_file") + or item.get("file_id") + or item.get("source_url") + or "" + ) + nodes[node_id] = GraphNode( + id=node_id, + label=label, + aliases=_coerce_aliases(item.get("aliases")), + source_file=source_file, + data=dict(item), + ) + + edges: list[GraphEdge] = [] + dangling_edges = 0 + malformed_edges = 0 + edge_counter: Counter[tuple[str, str, str]] = Counter() + self_loops = 0 + directed = bool(raw.get("directed", True)) + + for index, item in enumerate(raw_edges): + if not isinstance(item, Mapping): + malformed_edges += 1 + if strict: + raise ValueError(f"edges[{index}] must be an object") + continue + edge_source = str(item.get("source", "")).strip() + edge_target = str(item.get("target", "")).strip() + relation = str(item.get("relation", item.get("label", ""))).strip() + if not edge_source or not edge_target: + malformed_edges += 1 + if strict: + raise ValueError(f"edges[{index}] is missing source or target") + continue + if edge_source not in nodes or edge_target not in nodes: + dangling_edges += 1 + if strict: + raise ValueError( + f"Dangling edge {edge_source!r} -> {edge_target!r}" + ) + continue + if edge_source == edge_target: + self_loops += 1 + + key_source, key_target = edge_source, edge_target + if not directed and key_target < key_source: + key_source, key_target = key_target, key_source + edge_counter[(key_source, normalize_relation(relation), key_target)] += 1 + edges.append( + GraphEdge( + source=edge_source, + target=edge_target, + relation=relation, + data=dict(item), + ) + ) + + duplicate_edges = sum(max(0, count - 1) for count in edge_counter.values()) + return GraphData( + nodes=nodes, + edges=edges, + directed=directed, + multigraph=bool(raw.get("multigraph", True)), + diagnostics={ + "duplicate_node_ids": duplicate_node_ids, + "duplicate_edges": duplicate_edges, + "dangling_edges": dangling_edges, + "malformed_edges": malformed_edges, + "self_loops": self_loops, + }, + ) + + +def _node_match_score(gold: GraphNode, predicted: GraphNode) -> int: + if not gold.normalized_aliases.intersection(predicted.normalized_aliases): + return 0 + if gold.source_file and predicted.source_file: + return 3 if source_matches(gold.source_file, predicted.source_file) else 1 + return 2 + + +def align_entities( + gold_nodes: Iterable[GraphNode], + predicted: GraphData, +) -> dict[str, str]: + """Greedily align gold nodes to predicted nodes with deterministic ties. + + Exact normalized aliases are required. Matching source paths receive the + highest score, and each predicted node can be used at most once. + """ + + candidates: list[tuple[int, str, str]] = [] + for gold in gold_nodes: + for node in predicted.nodes.values(): + score = _node_match_score(gold, node) + if score: + candidates.append((-score, gold.id, node.id)) + + result: dict[str, str] = {} + used_predicted: set[str] = set() + for _negative_score, gold_id, predicted_id in sorted(candidates): + if gold_id in result or predicted_id in used_predicted: + continue + result[gold_id] = predicted_id + used_predicted.add(predicted_id) + return result + + +def canonical_edge( + source: str, + relation: object, + target: str, + *, + directed: bool, +) -> tuple[str, str, str]: + """Return a comparable edge key.""" + + edge_source, edge_target = source, target + if not directed and edge_target < edge_source: + edge_source, edge_target = edge_target, edge_source + return edge_source, normalize_relation(relation), edge_target + + +def unique_edge_keys(graph: GraphData) -> set[tuple[str, str, str]]: + """Return unique normalized graph triples while preserving parallel relations.""" + + return { + canonical_edge( + edge.source, + edge.relation, + edge.target, + directed=graph.directed, + ) + for edge in graph.edges + } + + +def bounded_bfs( + graph: GraphData, + start_ids: Iterable[str], + max_hops: int, +) -> dict[str, int]: + """Return the shortest hop distance to nodes reachable within ``max_hops``.""" + + if max_hops < 0: + raise ValueError("max_hops must be non-negative") + starts = sorted({node_id for node_id in start_ids if node_id in graph.nodes}) + distances = {node_id: 0 for node_id in starts} + queue: deque[str] = deque(starts) + adjacency = graph.adjacency() + while queue: + current = queue.popleft() + distance = distances[current] + if distance >= max_hops: + continue + for neighbor in adjacency.get(current, ()): + if neighbor in distances: + continue + distances[neighbor] = distance + 1 + queue.append(neighbor) + return distances diff --git a/deeprefine_skill/benchmarking/metrics.py b/deeprefine_skill/benchmarking/metrics.py new file mode 100644 index 0000000..bb4cbd8 --- /dev/null +++ b/deeprefine_skill/benchmarking/metrics.py @@ -0,0 +1,358 @@ +"""Deterministic metrics used by the lightweight graph benchmark.""" + +from __future__ import annotations + +import math +import re +import string +from collections import Counter +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, Iterable, Sequence + +from .graph import normalize_text + + +@dataclass(frozen=True) +class PRF: + """Precision/recall/F1 plus the counts that produced them.""" + + precision: float + recall: float + f1: float + tp: int + fp: int + fn: int + + def as_dict(self) -> dict[str, float | int]: + return { + "precision": self.precision, + "recall": self.recall, + "f1": self.f1, + "tp": self.tp, + "fp": self.fp, + "fn": self.fn, + } + + +def precision_recall_f1(tp: int, fp: int, fn: int) -> PRF: + """Calculate PRF with documented empty-set behavior. + + Two empty sets score 1.0. A zero denominator otherwise scores 0.0. + """ + + if tp < 0 or fp < 0 or fn < 0: + raise ValueError("tp, fp, and fn must be non-negative") + if tp == fp == fn == 0: + return PRF(1.0, 1.0, 1.0, tp, fp, fn) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = ( + 2.0 * precision * recall / (precision + recall) + if precision + recall + else 0.0 + ) + return PRF(precision, recall, f1, tp, fp, fn) + + +def set_prf(predicted: set[object], gold: set[object]) -> PRF: + """Calculate exact set overlap PRF.""" + + true_positive = len(predicted.intersection(gold)) + return precision_recall_f1( + true_positive, + len(predicted - gold), + len(gold - predicted), + ) + + +def _tokens(value: object) -> list[str]: + return re.findall(r"\w+", normalize_text(value), flags=re.UNICODE) + + +def _ngrams(tokens: Sequence[str], size: int) -> Counter[tuple[str, ...]]: + if size <= 0: + raise ValueError("ngram size must be positive") + return Counter(tuple(tokens[index : index + size]) for index in range(len(tokens) - size + 1)) + + +def sentence_bleu4(predicted: str, gold: str, *, epsilon: float = 0.1) -> float: + """Small dependency-free BLEU-4 with method-1-style smoothing.""" + + candidate = _tokens(predicted) + reference = _tokens(gold) + if not candidate or not reference: + return 1.0 if candidate == reference else 0.0 + + log_precisions = 0.0 + for size in range(1, 5): + candidate_counts = _ngrams(candidate, size) + reference_counts = _ngrams(reference, size) + denominator = sum(candidate_counts.values()) + overlap = sum( + min(count, reference_counts[gram]) + for gram, count in candidate_counts.items() + ) + if denominator == 0: + precision = epsilon + elif overlap == 0: + precision = epsilon / denominator + else: + precision = overlap / denominator + log_precisions += 0.25 * math.log(max(precision, 1e-12)) + + brevity_penalty = ( + 1.0 + if len(candidate) >= len(reference) + else math.exp(1.0 - len(reference) / len(candidate)) + ) + return brevity_penalty * math.exp(log_precisions) + + +def rouge2_precision(predicted: str, gold: str) -> float: + """Return ROUGE-2 precision for one predicted/reference pair.""" + + candidate_counts = _ngrams(_tokens(predicted), 2) + reference_counts = _ngrams(_tokens(gold), 2) + denominator = sum(candidate_counts.values()) + if denominator == 0: + return 1.0 if candidate_counts == reference_counts else 0.0 + overlap = sum( + min(count, reference_counts[gram]) + for gram, count in candidate_counts.items() + ) + return overlap / denominator + + +def _maximum_assignment_sum(matrix: Sequence[Sequence[float]]) -> float: + """Return maximum one-to-one assignment weight using Hungarian O(n^3).""" + + if not matrix or not matrix[0]: + return 0.0 + rows = [list(row) for row in matrix] + if any(len(row) != len(rows[0]) for row in rows): + raise ValueError("Similarity matrix must be rectangular") + if len(rows) > len(rows[0]): + rows = [list(column) for column in zip(*rows)] + + row_count = len(rows) + column_count = len(rows[0]) + u = [0.0] * (row_count + 1) + v = [0.0] * (column_count + 1) + p = [0] * (column_count + 1) + way = [0] * (column_count + 1) + + for row_index in range(1, row_count + 1): + p[0] = row_index + minimum = [math.inf] * (column_count + 1) + used = [False] * (column_count + 1) + column_zero = 0 + while True: + used[column_zero] = True + active_row = p[column_zero] + delta = math.inf + next_column = 0 + for column_index in range(1, column_count + 1): + if used[column_index]: + continue + cost = -rows[active_row - 1][column_index - 1] + reduced = cost - u[active_row] - v[column_index] + if reduced < minimum[column_index]: + minimum[column_index] = reduced + way[column_index] = column_zero + if minimum[column_index] < delta: + delta = minimum[column_index] + next_column = column_index + for column_index in range(column_count + 1): + if used[column_index]: + u[p[column_index]] += delta + v[column_index] -= delta + else: + minimum[column_index] -= delta + column_zero = next_column + if p[column_zero] == 0: + break + while True: + previous = way[column_zero] + p[column_zero] = p[previous] + column_zero = previous + if column_zero == 0: + break + + return sum( + rows[p[column_index] - 1][column_index - 1] + for column_index in range(1, column_count + 1) + if p[column_index] + ) + + +def lexical_graph_score( + predicted: Sequence[str], + gold: Sequence[str], + *, + metric: str, +) -> dict[str, float]: + """GraphJudge-style one-to-one lexical triple matching. + + ``metric`` is ``"bleu"`` or ``"rouge"``. Scores are normalized by the + predicted/gold graph sizes to obtain precision, recall, and F1. + """ + + if not predicted and not gold: + return {"precision": 1.0, "recall": 1.0, "f1": 1.0} + if not predicted or not gold: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0} + scorer = sentence_bleu4 if metric == "bleu" else rouge2_precision + if metric not in {"bleu", "rouge"}: + raise ValueError("metric must be 'bleu' or 'rouge'") + matrix = [[scorer(candidate, reference) for reference in gold] for candidate in predicted] + score = _maximum_assignment_sum(matrix) + precision = score / len(predicted) + recall = score / len(gold) + f1 = ( + 2.0 * precision * recall / (precision + recall) + if precision + recall + else 0.0 + ) + return {"precision": precision, "recall": recall, "f1": f1} + + +def triple_sentence(triple: Sequence[object]) -> str: + """Render a triple in the same component order used for lexical scoring.""" + + if len(triple) != 3: + raise ValueError("A triple must have exactly three components") + from .graph import normalize_relation + + return "; ".join( + ( + normalize_text(triple[0]), + normalize_relation(triple[1]), + normalize_text(triple[2]), + ) + ) + + +@lru_cache(maxsize=4) +def _bert_scorer(model_type: str) -> Any: + try: + from bert_score import BERTScorer + except ImportError as exc: + raise RuntimeError( + "Semantic graph scoring requires " + "`pip install 'deeprefine-cli[benchmark-semantic]'`" + ) from exc + return BERTScorer( + model_type=model_type, + lang="en", + rescale_with_baseline=False, + ) + + +def bertscore_graph_score( + predicted: Sequence[str], + gold: Sequence[str], + *, + model_type: str = "roberta-large", +) -> dict[str, float]: + """Optional GraphJudge-style BERTScore with one-to-one graph matching. + + The heavy dependency is imported only when this function is requested. + Install ``deeprefine-cli[benchmark-semantic]`` to enable it. + """ + + if not predicted and not gold: + return {"precision": 1.0, "recall": 1.0, "f1": 1.0} + if not predicted or not gold: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0} + candidates = [candidate for candidate in predicted for _reference in gold] + references = [_reference for _candidate in predicted for _reference in gold] + scorer = _bert_scorer(model_type) + _precision, _recall, f1 = scorer.score(candidates, references) + scores = [float(value) for value in f1] + matrix = [ + scores[index * len(gold) : (index + 1) * len(gold)] + for index in range(len(predicted)) + ] + score = _maximum_assignment_sum(matrix) + graph_precision = score / len(predicted) + graph_recall = score / len(gold) + graph_f1 = ( + 2.0 * graph_precision * graph_recall / (graph_precision + graph_recall) + if graph_precision + graph_recall + else 0.0 + ) + return { + "precision": graph_precision, + "recall": graph_recall, + "f1": graph_f1, + } + + +_ARTICLES_RE = re.compile(r"\b(a|an|the)\b", flags=re.IGNORECASE) +_PUNCT_TRANSLATION = str.maketrans("", "", string.punctuation) + + +def normalize_answer(value: object) -> str: + """Normalize a short QA answer using the repository's EM/F1 conventions.""" + + text = normalize_text(value).replace("-", " ") + text = text.translate(_PUNCT_TRANSLATION) + text = _ARTICLES_RE.sub(" ", text) + return " ".join(text.split()) + + +def answer_exact_match(predicted: object, gold_answers: Iterable[object]) -> float: + prediction = normalize_answer(predicted) + answers = [normalize_answer(answer) for answer in gold_answers] + return float(any(prediction == answer for answer in answers)) + + +def _single_answer_f1(predicted: str, gold: str) -> float: + predicted_tokens = normalize_answer(predicted).split() + gold_tokens = normalize_answer(gold).split() + if not predicted_tokens or not gold_tokens: + return float(predicted_tokens == gold_tokens) + common = Counter(predicted_tokens) & Counter(gold_tokens) + overlap = sum(common.values()) + if not overlap: + return 0.0 + precision = overlap / len(predicted_tokens) + recall = overlap / len(gold_tokens) + return 2.0 * precision * recall / (precision + recall) + + +def answer_token_f1(predicted: object, gold_answers: Iterable[object]) -> float: + answers = [str(answer) for answer in gold_answers] + if not answers: + return 0.0 + return max(_single_answer_f1(str(predicted), answer) for answer in answers) + + +def hit_at_k(ranked_items: Sequence[object], gold_items: Iterable[object], k: int) -> float: + if k <= 0: + raise ValueError("k must be positive") + gold = {normalize_text(item) for item in gold_items} + return float(any(normalize_text(item) in gold for item in ranked_items[:k])) + + +def reciprocal_rank(ranked_items: Sequence[object], gold_items: Iterable[object]) -> float: + gold = {normalize_text(item) for item in gold_items} + for index, item in enumerate(ranked_items, start=1): + if normalize_text(item) in gold: + return 1.0 / index + return 0.0 + + +def supporting_recall( + retrieved_items: Sequence[object], + gold_items: Iterable[object], + *, + k: int | None = None, +) -> float: + gold = {normalize_text(item) for item in gold_items} + if not gold: + return 1.0 + selected = retrieved_items if k is None else retrieved_items[:k] + retrieved = {normalize_text(item) for item in selected} + return len(gold.intersection(retrieved)) / len(gold) diff --git a/deeprefine_skill/benchmarking/prepare.py b/deeprefine_skill/benchmarking/prepare.py new file mode 100644 index 0000000..757f0a1 --- /dev/null +++ b/deeprefine_skill/benchmarking/prepare.py @@ -0,0 +1,562 @@ +"""Prepare deterministic Re-DocRED, 2Wiki, and synthetic benchmark suites.""" + +from __future__ import annotations + +import hashlib +import json +import re +import shutil +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +from .suite import builtin_suite_path, sha256_file + + +SELECTION_SEED = "deeprefine-benchmark-v1" +SUPPORTED_PROFILES = {"quick", "readme"} +SUPPORTED_SUITES = { + "synthetic-smoke-v1", + "redocred-mini-v1", + "2wiki-mini-v1", +} +_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") + + +def _json_text(value: object) -> str: + return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + + +def _stable_digest(value: object) -> str: + return hashlib.sha256(f"{SELECTION_SEED}:{value}".encode("utf-8")).hexdigest() + + +def _safe_file_name(value: object, *, suffix: str = ".txt") -> str: + stem = _SAFE_NAME_RE.sub("-", str(value)).strip("-._")[:48] or "document" + return f"{stem}-{_stable_digest(value)[:8]}{suffix}" + + +def _ensure_empty_output(path: Path) -> None: + if path.exists() and not path.is_dir(): + raise ValueError(f"Output path is not a directory: {path}") + if path.exists() and any(path.iterdir()): + raise ValueError(f"Output directory is not empty: {path}") + path.mkdir(parents=True, exist_ok=True) + + +def _load_json_list(path: Path) -> list[dict[str, Any]]: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Cannot load dataset {path}: {exc}") from exc + if isinstance(raw, Mapping): + for key in ("data", "documents", "examples"): + if isinstance(raw.get(key), list): + raw = raw[key] + break + if not isinstance(raw, list) or not all(isinstance(item, Mapping) for item in raw): + raise ValueError(f"Dataset {path} must contain an array of objects") + return [dict(item) for item in raw] + + +def _hashed_take( + items: Iterable[dict[str, Any]], + count: int, + *, + key: Callable[[dict[str, Any]], object], +) -> list[dict[str, Any]]: + ranked = sorted(items, key=lambda item: (_stable_digest(key(item)), str(key(item)))) + if len(ranked) < count: + raise ValueError(f"Dataset has {len(ranked)} eligible items; need {count}") + return ranked[:count] + + +def _write_jsonl(path: Path, items: Iterable[Mapping[str, Any]]) -> None: + lines = [json.dumps(item, ensure_ascii=False, sort_keys=True) for item in items] + path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + + +def _write_lock( + output_dir: Path, + *, + suite_id: str, + profile: str, + source: Mapping[str, Any], +) -> None: + files = { + str(path.relative_to(output_dir)).replace("\\", "/"): sha256_file(path) + for path in sorted(output_dir.rglob("*")) + if path.is_file() and path.name != "suite.lock.json" + } + lock = { + "schema_version": 1, + "suite_id": suite_id, + "profile": profile, + "selection_seed": SELECTION_SEED, + "source": dict(source), + "files": files, + } + (output_dir / "suite.lock.json").write_text(_json_text(lock), encoding="utf-8") + + +def _relation_mapping(source_path: Path) -> dict[str, str]: + candidates = ( + source_path.parent / "rel_info.json", + source_path.parent / "rel_info_full.json", + source_path.parent / "relation_map.json", + ) + for candidate in candidates: + if not candidate.is_file(): + continue + try: + raw = json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if not isinstance(raw, Mapping): + continue + result: dict[str, str] = {} + for relation_id, value in raw.items(): + if isinstance(value, str): + result[str(relation_id)] = value + elif isinstance(value, Sequence) and value: + result[str(relation_id)] = str(value[0]) + elif isinstance(value, Mapping): + result[str(relation_id)] = str( + value.get("name") or value.get("label") or relation_id + ) + return result + return {} + + +def _redocred_triple_count(document: Mapping[str, Any]) -> int: + labels = document.get("labels", []) + return len(labels) if isinstance(labels, list) else 0 + + +def _redocred_cross_sentence_ratio(document: Mapping[str, Any]) -> float: + labels = document.get("labels", []) + vertex_set = document.get("vertexSet", []) + if not isinstance(labels, list) or not labels or not isinstance(vertex_set, list): + return 0.0 + cross_sentence = 0 + valid = 0 + for label in labels: + if not isinstance(label, Mapping): + continue + head = label.get("h") + tail = label.get("t") + if ( + not isinstance(head, int) + or not isinstance(tail, int) + or head >= len(vertex_set) + or tail >= len(vertex_set) + ): + continue + head_mentions = vertex_set[head] if isinstance(vertex_set[head], list) else [] + tail_mentions = vertex_set[tail] if isinstance(vertex_set[tail], list) else [] + head_sentences = { + mention.get("sent_id") + for mention in head_mentions + if isinstance(mention, Mapping) and isinstance(mention.get("sent_id"), int) + } + tail_sentences = { + mention.get("sent_id") + for mention in tail_mentions + if isinstance(mention, Mapping) and isinstance(mention.get("sent_id"), int) + } + if not head_sentences or not tail_sentences: + continue + valid += 1 + if head_sentences.isdisjoint(tail_sentences): + cross_sentence += 1 + return cross_sentence / valid if valid else 0.0 + + +def _redocred_key(document: Mapping[str, Any]) -> str: + title = str(document.get("title", "")).strip() + if title: + return title + return json.dumps(document, ensure_ascii=False, sort_keys=True) + + +def _stratified_redocred( + documents: list[dict[str, Any]], + count: int, +) -> list[dict[str, Any]]: + eligible = [ + document + for document in documents + if isinstance(document.get("vertexSet"), list) + and isinstance(document.get("labels"), list) + and _redocred_triple_count(document) > 0 + ] + if len(eligible) < count: + raise ValueError( + f"Re-DocRED source has {len(eligible)} eligible documents; need {count}" + ) + ordered = sorted( + eligible, + key=lambda item: ( + _redocred_triple_count(item), + _redocred_cross_sentence_ratio(item), + str(item.get("title", "")), + ), + ) + bucket_count = min(5, count) + selected: list[dict[str, Any]] = [] + for bucket_index in range(bucket_count): + start = bucket_index * len(ordered) // bucket_count + end = (bucket_index + 1) * len(ordered) // bucket_count + quota = count // bucket_count + int(bucket_index < count % bucket_count) + bucket = sorted( + ordered[start:end], + key=lambda item: (_redocred_cross_sentence_ratio(item), _redocred_key(item)), + ) + midpoint = len(bucket) // 2 + low = bucket[:midpoint] + high = bucket[midpoint:] + low_quota = quota // 2 + high_quota = quota - low_quota + chosen: list[dict[str, Any]] = [] + if low_quota: + chosen.extend(_hashed_take(low, low_quota, key=_redocred_key)) + if high_quota: + chosen.extend(_hashed_take(high, high_quota, key=_redocred_key)) + selected.extend(chosen) + return sorted(selected, key=lambda item: str(item.get("title", ""))) + + +def _mention_aliases(entity: object) -> tuple[str, list[str]]: + mentions = entity if isinstance(entity, list) else [] + aliases: list[str] = [] + for mention in mentions: + if isinstance(mention, Mapping) and str(mention.get("name", "")).strip(): + aliases.append(str(mention["name"])) + if not aliases: + return "unknown entity", [] + return aliases[0], sorted(set(aliases[1:])) + + +def _prepare_redocred( + source_path: Path, + output_dir: Path, + profile: str, +) -> dict[str, Any]: + documents = _load_json_list(source_path) + selected_count = 10 if profile == "quick" else 50 + selected = _stratified_redocred(documents, selected_count) + relation_names = _relation_mapping(source_path) + corpus_dir = output_dir / "corpus" + corpus_dir.mkdir() + cases: list[dict[str, Any]] = [] + corpus_records: list[dict[str, Any]] = [] + + for index, document in enumerate(selected): + title = str(document.get("title") or f"document-{index}") + case_id = f"redocred-{_stable_digest(title)[:12]}" + file_name = _safe_file_name(title) + sentences = document.get("sents", []) + text = "\n\n".join( + " ".join(str(token) for token in sentence) + for sentence in sentences + if isinstance(sentence, list) + ) + (corpus_dir / file_name).write_text(text.strip() + "\n", encoding="utf-8") + corpus_records.append( + { + "id": case_id, + "title": title, + "text_file": f"corpus/{file_name}", + } + ) + + entities: list[dict[str, Any]] = [] + vertex_set = document.get("vertexSet", []) + for entity_index, entity in enumerate(vertex_set): + label, aliases = _mention_aliases(entity) + entities.append( + { + "id": f"e{entity_index}", + "label": label, + "aliases": aliases, + "source_file": file_name, + } + ) + edges: list[dict[str, Any]] = [] + for label in document.get("labels", []): + if not isinstance(label, Mapping): + continue + head = label.get("h") + tail = label.get("t") + relation_id = str(label.get("r", "")).strip() + if not isinstance(head, int) or not isinstance(tail, int) or not relation_id: + continue + if head >= len(entities) or tail >= len(entities): + continue + relation = relation_names.get(relation_id, relation_id) + edge: dict[str, Any] = { + "source": f"e{head}", + "relation": relation, + "target": f"e{tail}", + } + if relation != relation_id: + edge["accepted_relations"] = [relation_id] + edges.append(edge) + + cases.append( + { + "id": case_id, + "task": "intrinsic", + "title": title, + "source_files": [file_name], + "directed": True, + "gold_entities": entities, + "gold_edges": edges, + } + ) + + source = { + "dataset": "Re-DocRED", + "split": "dev", + "path": str(source_path.resolve()), + "sha256": sha256_file(source_path), + "paper": "https://arxiv.org/abs/2205.12696", + } + suite = { + "schema_version": 1, + "suite_id": "redocred-mini-v1", + "suite_version": "1", + "profile": profile, + "selection_seed": SELECTION_SEED, + "source": source, + "cases": cases, + } + (output_dir / "suite.json").write_text(_json_text(suite), encoding="utf-8") + _write_jsonl(output_dir / "corpus.jsonl", corpus_records) + _write_jsonl(output_dir / "queries.jsonl", []) + _write_lock( + output_dir, + suite_id=suite["suite_id"], + profile=profile, + source=source, + ) + return suite + + +_2WIKI_TYPES = ("compositional", "comparison", "inference", "bridge_comparison") + + +def _context_items(value: object) -> list[tuple[str, list[str]]]: + result: list[tuple[str, list[str]]] = [] + if not isinstance(value, list): + return result + for item in value: + if ( + isinstance(item, Sequence) + and not isinstance(item, (str, bytes, bytearray)) + and len(item) >= 2 + ): + title = str(item[0]) + sentences = item[1] + if isinstance(sentences, list): + result.append((title, [str(sentence) for sentence in sentences])) + return result + + +def _evidence_edges(value: object) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + if not isinstance(value, list): + return result + for item in value: + if ( + isinstance(item, Sequence) + and not isinstance(item, (str, bytes, bytearray)) + and len(item) >= 3 + ): + result.append( + { + "source": str(item[0]), + "relation": str(item[1]), + "target": str(item[2]), + } + ) + return result + + +def _supporting_facts(value: object) -> list[str]: + result: list[str] = [] + if not isinstance(value, list): + return result + for item in value: + if ( + isinstance(item, Sequence) + and not isinstance(item, (str, bytes, bytearray)) + and len(item) >= 2 + ): + result.append(f"{item[0]}#{item[1]}") + return result + + +def _seed_entities(edges: Sequence[Mapping[str, Any]]) -> list[str]: + heads = [str(edge["source"]) for edge in edges] + tails = {str(edge["target"]) for edge in edges} + roots = [head for head in heads if head not in tails] + return list(dict.fromkeys(roots or heads[:1])) + + +def _prepare_2wiki( + source_path: Path, + output_dir: Path, + profile: str, +) -> dict[str, Any]: + examples = _load_json_list(source_path) + per_type = 4 if profile == "quick" else 16 + selected: list[dict[str, Any]] = [] + for question_type in _2WIKI_TYPES: + typed = [ + item + for item in examples + if str(item.get("type", "")).replace("-", "_") == question_type + and _evidence_edges(item.get("evidences")) + ] + selected.extend( + _hashed_take( + typed, + per_type, + key=lambda item: item.get("_id") or item.get("id") or item.get("question"), + ) + ) + selected.sort( + key=lambda item: ( + str(item.get("type", "")), + str(item.get("_id") or item.get("id") or item.get("question")), + ) + ) + + corpus_dir = output_dir / "corpus" + corpus_dir.mkdir() + corpus_by_title: dict[str, tuple[str, list[str]]] = {} + for example in selected: + for title, sentences in _context_items(example.get("context")): + corpus_by_title.setdefault(title, (_safe_file_name(title), sentences)) + corpus_records: list[dict[str, Any]] = [] + for title in sorted(corpus_by_title): + file_name, sentences = corpus_by_title[title] + (corpus_dir / file_name).write_text( + "\n".join(sentence.strip() for sentence in sentences if sentence.strip()) + + "\n", + encoding="utf-8", + ) + corpus_records.append( + { + "id": _stable_digest(title)[:12], + "title": title, + "text_file": f"corpus/{file_name}", + } + ) + + cases: list[dict[str, Any]] = [] + queries: list[dict[str, Any]] = [] + for index, example in enumerate(selected): + raw_id = example.get("_id") or example.get("id") or f"question-{index}" + case_id = f"2wiki-{raw_id}" + evidence = _evidence_edges(example.get("evidences")) + answer = str(example.get("answer", "")).strip() + answers = [answer] if answer else [] + endpoints = { + str(edge["source"]) + for edge in evidence + }.union(str(edge["target"]) for edge in evidence) + answer_entities = [ + endpoint for endpoint in endpoints if endpoint.casefold() == answer.casefold() + ] or answers + cases.append( + { + "id": case_id, + "task": "downstream", + "question_type": str(example.get("type", "")).replace("-", "_"), + "question": str(example.get("question", "")), + "answers": answers, + "seed_entities": _seed_entities(evidence), + "answer_entities": answer_entities, + "evidence_edges": evidence, + "supporting_facts": _supporting_facts(example.get("supporting_facts")), + "max_hops": min(4, max(1, len(evidence))), + } + ) + queries.append( + { + "case_id": case_id, + "question": str(example.get("question", "")), + } + ) + + source = { + "dataset": "2WikiMultiHopQA", + "split": "provided source", + "path": str(source_path.resolve()), + "sha256": sha256_file(source_path), + "paper": "https://arxiv.org/abs/2011.01060", + } + suite = { + "schema_version": 1, + "suite_id": "2wiki-mini-v1", + "suite_version": "1", + "profile": profile, + "selection_seed": SELECTION_SEED, + "source": source, + "cases": cases, + } + (output_dir / "suite.json").write_text(_json_text(suite), encoding="utf-8") + _write_jsonl(output_dir / "corpus.jsonl", corpus_records) + _write_jsonl(output_dir / "queries.jsonl", queries) + _write_lock( + output_dir, + suite_id=suite["suite_id"], + profile=profile, + source=source, + ) + return suite + + +def prepare_suite( + suite_id: str, + profile: str, + output_dir: str | Path, + *, + source: str | Path | None = None, +) -> Path: + """Prepare one deterministic suite and return its output directory.""" + + if suite_id not in SUPPORTED_SUITES: + raise ValueError( + f"Unsupported suite {suite_id!r}; choose from {sorted(SUPPORTED_SUITES)}" + ) + output_path = Path(output_dir).resolve() + _ensure_empty_output(output_path) + + if suite_id == "synthetic-smoke-v1": + if profile != "smoke": + raise ValueError("synthetic-smoke-v1 only supports the smoke profile") + source_dir = builtin_suite_path(suite_id) + for item in source_dir.iterdir(): + destination = output_path / item.name + if item.is_dir(): + shutil.copytree(item, destination) + else: + shutil.copy2(item, destination) + return output_path + + if profile not in SUPPORTED_PROFILES: + raise ValueError( + f"Unsupported profile {profile!r}; choose from {sorted(SUPPORTED_PROFILES)}" + ) + if source is None: + raise ValueError(f"--source is required for {suite_id}") + source_path = Path(source).resolve() + if not source_path.is_file(): + raise ValueError(f"Dataset source does not exist: {source_path}") + if suite_id == "redocred-mini-v1": + _prepare_redocred(source_path, output_path, profile) + else: + _prepare_2wiki(source_path, output_path, profile) + return output_path diff --git a/deeprefine_skill/benchmarking/report.py b/deeprefine_skill/benchmarking/report.py new file mode 100644 index 0000000..d65bc5d --- /dev/null +++ b/deeprefine_skill/benchmarking/report.py @@ -0,0 +1,119 @@ +"""Markdown rendering for benchmark result JSON.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def _percent(value: object) -> str: + return "—" if not isinstance(value, (int, float)) else f"{100.0 * value:.2f}%" + + +def _delta(value: object) -> str: + if not isinstance(value, (int, float)): + return "—" + return f"{100.0 * value:+.2f} pp" + + +def _rows(result: Mapping[str, Any]) -> list[tuple[str, str, object, object, object]]: + suite_label = ( + f"{result['suite']['suite_id']} " + f"({result['suite']['profile']}, n={result['suite']['case_count']})" + ) + metrics = result["metrics"] + rows: list[tuple[str, str, object, object, object]] = [] + definitions = ( + ("intrinsic", "entity_f1", "Entity F1"), + ("intrinsic", "strict_triple_f1", "Strict Triple F1"), + ("intrinsic", "g_bleu_f1", "G-BLEU F1"), + ("intrinsic", "g_rouge_f1", "G-ROUGE F1"), + ("intrinsic", "g_bertscore_f1", "G-BERTScore F1"), + ("downstream", "evidence_edge_recall", "Evidence Edge Recall"), + ("downstream", "complete_path_rate", "Complete Path Rate"), + ("downstream", "answer_reachability_rate", "Answer Reachability"), + ) + for category, key, label in definitions: + baseline_category = metrics["baseline"].get(category) + candidate_category = metrics["candidate"].get(category) + if not baseline_category or not candidate_category: + continue + baseline = baseline_category["macro"].get(key) + candidate = candidate_category["macro"].get(key) + if not isinstance(baseline, (int, float)) or not isinstance( + candidate, + (int, float), + ): + continue + delta = metrics.get("delta", {}).get(category, {}).get(key) + rows.append((suite_label, label, baseline, candidate, delta)) + + base_prediction = metrics["baseline"].get("downstream", {}).get("prediction") or {} + candidate_prediction = ( + metrics["candidate"].get("downstream", {}).get("prediction") or {} + ) + if "answer_f1" in base_prediction and "answer_f1" in candidate_prediction: + before = base_prediction["answer_f1"]["value"] + after = candidate_prediction["answer_f1"]["value"] + rows.append((suite_label, "Answer F1", before, after, after - before)) + return rows + + +def render_markdown( + results: Mapping[str, Any] | Sequence[Mapping[str, Any]], +) -> str: + """Render one or more benchmark results as a README-ready report.""" + + materialized = [results] if isinstance(results, Mapping) else list(results) + lines = [ + "# Lightweight Graph Quality Benchmark", + "", + "| Suite | Metric | Graphify Before | DeepRefine After | Δ |", + "|---|---|---:|---:|---:|", + ] + for result in materialized: + for suite, metric, before, after, delta in _rows(result): + lines.append( + f"| {suite} | {metric} | {_percent(before)} | " + f"{_percent(after)} | {_delta(delta)} |" + ) + + lines.extend(["", "## Reproducibility", ""]) + for result in materialized: + environment = result.get("environment", {}) + cost = result.get("cost", {}) + lines.extend( + [ + f"- `{result['suite']['suite_id']}` v{result['suite']['suite_version']}: " + f"baseline `{result['inputs']['baseline_graph']['sha256'][:12]}`, " + f"candidate `{result['inputs']['candidate_graph']['sha256'][:12]}`; " + f"Graphify `{environment.get('graphify_version') or 'not recorded'}`, " + f"DeepRefine `{environment.get('deeprefine_version') or 'not recorded'}`, " + f"model `{environment.get('model') or 'not run'}`; " + f"{cost.get('llm_calls', 0)} LLM calls.", + ] + ) + + lines.extend( + [ + "", + "> This is a deterministic micro-benchmark for regression checks and " + "README demonstrations. It does not replace a full GraphRAG evaluation. " + "2Wiki evidence metrics measure query-relevant path coverage, not " + "full-graph precision.", + "", + ] + ) + return "\n".join(lines) + + +def write_markdown( + results: Mapping[str, Any] | Sequence[Mapping[str, Any]], + output: str | Path, +) -> Path: + """Render and write a UTF-8 Markdown report.""" + + output_path = Path(output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(render_markdown(results), encoding="utf-8") + return output_path diff --git a/deeprefine_skill/benchmarking/suite.py b/deeprefine_skill/benchmarking/suite.py new file mode 100644 index 0000000..bacf2f5 --- /dev/null +++ b/deeprefine_skill/benchmarking/suite.py @@ -0,0 +1,150 @@ +"""Suite schema loading and reproducibility helpers.""" + +from __future__ import annotations + +import hashlib +import json +from importlib import resources +from pathlib import Path +from typing import Any, Mapping + + +SUPPORTED_SCHEMA_VERSION = 1 + + +def sha256_file(path: str | Path) -> str: + """Return the SHA-256 digest of a file.""" + + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def builtin_suite_path(suite_id: str) -> Path: + """Resolve an unpacked built-in suite directory.""" + + root = resources.files("deeprefine_skill").joinpath("benchmark_suites", suite_id) + path = Path(str(root)) + if not (path / "suite.json").is_file(): + raise ValueError(f"Unknown built-in benchmark suite: {suite_id}") + return path + + +def resolve_suite_path(value: str | Path) -> Path: + """Resolve a directory, suite.json path, or built-in suite ID.""" + + path = Path(value) + if path.is_dir(): + path = path / "suite.json" + if path.is_file(): + return path.resolve() + return (builtin_suite_path(str(value)) / "suite.json").resolve() + + +def verify_suite_lock(suite_directory: str | Path) -> None: + """Verify files listed by an optional ``suite.lock.json``.""" + + root = Path(suite_directory).resolve() + lock_path = root / "suite.lock.json" + if not lock_path.is_file(): + return + try: + lock = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Cannot load suite lock {lock_path}: {exc}") from exc + files = lock.get("files") + if not isinstance(files, Mapping): + raise ValueError(f"Suite lock has no files mapping: {lock_path}") + for relative, expected in files.items(): + target = (root / str(relative)).resolve() + try: + target.relative_to(root) + except ValueError as exc: + raise ValueError(f"Suite lock path escapes suite directory: {relative}") from exc + if not target.is_file(): + raise ValueError(f"Suite lock file is missing: {relative}") + actual = sha256_file(target) + if actual != str(expected).casefold(): + raise ValueError( + f"Suite lock checksum mismatch for {relative}: " + f"expected {expected}, got {actual}" + ) + + +def load_suite(value: str | Path | Mapping[str, Any]) -> tuple[dict[str, Any], Path | None]: + """Load and minimally validate a benchmark suite. + + Returns ``(suite, suite_directory)``. Inline mappings have no directory. + """ + + if isinstance(value, Mapping): + suite = dict(value) + suite_dir = None + else: + suite_path = resolve_suite_path(value) + verify_suite_lock(suite_path.parent) + try: + suite = json.loads(suite_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"Cannot load suite {suite_path}: {exc}") from exc + suite_dir = suite_path.parent + + if suite.get("schema_version") != SUPPORTED_SCHEMA_VERSION: + raise ValueError( + "Unsupported suite schema_version " + f"{suite.get('schema_version')!r}; expected {SUPPORTED_SCHEMA_VERSION}" + ) + for field_name in ("suite_id", "suite_version", "profile", "cases"): + if field_name not in suite: + raise ValueError(f"Suite is missing required field: {field_name}") + if not isinstance(suite["cases"], list): + raise ValueError("Suite 'cases' must be an array") + + seen_ids: set[str] = set() + for index, case in enumerate(suite["cases"]): + if not isinstance(case, Mapping): + raise ValueError(f"cases[{index}] must be an object") + case_id = str(case.get("id", "")).strip() + if not case_id: + raise ValueError(f"cases[{index}] is missing id") + if case_id in seen_ids: + raise ValueError(f"Duplicate case id: {case_id}") + seen_ids.add(case_id) + if case.get("task") not in {"intrinsic", "downstream"}: + raise ValueError( + f"cases[{index}].task must be 'intrinsic' or 'downstream'" + ) + return suite, suite_dir + + +def load_predictions(path: str | Path | None) -> dict[str, dict[str, Any]]: + """Load case-keyed prediction JSONL.""" + + if path is None: + return {} + prediction_path = Path(path) + result: dict[str, dict[str, Any]] = {} + try: + lines = prediction_path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + raise ValueError(f"Cannot load predictions {prediction_path}: {exc}") from exc + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"Invalid prediction JSON on line {line_number}: {exc}" + ) from exc + if not isinstance(item, dict) or not str(item.get("case_id", "")).strip(): + raise ValueError( + f"Prediction line {line_number} must contain a case_id" + ) + case_id = str(item["case_id"]) + if case_id in result: + raise ValueError(f"Duplicate prediction case_id: {case_id}") + result[case_id] = item + return result diff --git a/deeprefine_skill/benchmarking/wiki.py b/deeprefine_skill/benchmarking/wiki.py new file mode 100644 index 0000000..0dfe791 --- /dev/null +++ b/deeprefine_skill/benchmarking/wiki.py @@ -0,0 +1,67 @@ +"""Non-semantic integrity checks for a generated Graphify Wiki directory.""" + +from __future__ import annotations + +import re +from pathlib import Path +from urllib.parse import unquote + + +_MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*]\(([^)\s]+)(?:\s+['\"][^)]*['\"])?\)") + + +def inspect_wiki_directory(value: str | Path) -> dict[str, object]: + """Check index presence and local Markdown links without judging content.""" + + root = Path(value).resolve() + if not root.is_dir(): + raise ValueError(f"Wiki directory does not exist: {root}") + pages = sorted(root.rglob("*.md")) + page_set = {page.resolve() for page in pages} + inbound = {page: 0 for page in page_set} + broken: list[dict[str, str]] = [] + local_link_count = 0 + + for page in pages: + try: + text = page.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"Cannot read Wiki page {page}: {exc}") from exc + for raw_target in _MARKDOWN_LINK_RE.findall(text): + target = unquote(raw_target).split("#", 1)[0] + if ( + not target + or "://" in target + or target.startswith(("mailto:", "data:")) + ): + continue + local_link_count += 1 + target_path = (page.parent / target).resolve() + if target_path.is_dir(): + target_path = target_path / "index.md" + if not target_path.exists(): + broken.append( + { + "source": str(page.relative_to(root)).replace("\\", "/"), + "target": raw_target, + } + ) + elif target_path in inbound: + inbound[target_path] += 1 + + index_path = (root / "index.md").resolve() + orphan_pages = sorted( + str(page.relative_to(root)).replace("\\", "/") + for page, count in inbound.items() + if page != index_path and count == 0 + ) + return { + "path": str(root), + "index_exists": index_path.is_file(), + "page_count": len(pages), + "local_link_count": local_link_count, + "broken_link_count": len(broken), + "broken_links": broken, + "orphan_page_count": len(orphan_pages), + "orphan_pages": orphan_pages, + } diff --git a/deeprefine_skill/cli.py b/deeprefine_skill/cli.py index 1418a52..abb774b 100644 --- a/deeprefine_skill/cli.py +++ b/deeprefine_skill/cli.py @@ -887,6 +887,11 @@ def main(argv: list[str] | None = None) -> int: ) p_sync.set_defaults(func=cmd_history_sync_memory) + # deeprefine benchmark prepare | evaluate | report + from deeprefine_skill.benchmarking.cli import register_benchmark_commands + + register_benchmark_commands(sub) + p_index = sub.add_parser("index", help="Rebuild FAISS cache from graph.json") p_index.add_argument("--rebuild", action="store_true", default=True) p_index.set_defaults(func=cmd_index) diff --git a/docs/benchmark-data.md b/docs/benchmark-data.md new file mode 100644 index 0000000..02e7fce --- /dev/null +++ b/docs/benchmark-data.md @@ -0,0 +1,54 @@ +# Benchmark data attribution and licensing + +DeepRefine-Skill's benchmark commands support small, reproducible subsets of +third-party datasets. The project does not relicense those datasets. Keep the +original notices when downloading, preparing, sharing, or publishing results. + +The package itself includes only `synthetic-smoke-v1`, a generated fixture used +to test the evaluator and report pipeline. It is covered by this repository's +MIT license and is not evidence of real-world graph quality. + +## Recommended suites + +### Re-DocRED + +- Purpose in DeepRefine-Skill: intrinsic entity and relation quality. +- Source: [tonytan48/Re-DocRED](https://github.com/tonytan48/Re-DocRED) +- Paper: [Revisiting DocRED – Addressing the False Negative Problem in + Relation Extraction](https://arxiv.org/abs/2205.12696) +- Upstream repository license: MIT. +- Provenance: the corpus derives from DocRED, Wikipedia, and Wikidata. Cite the + upstream paper and preserve any source attribution that accompanies the + downloaded data. + +### 2WikiMultiHopQA + +- Purpose in DeepRefine-Skill: downstream multi-hop evidence retrieval, + reasoning-path coverage, and answer quality. +- Source: + [Alab-NII/2wikimultihop](https://github.com/Alab-NII/2wikimultihop) +- Paper: [Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of + Reasoning Steps](https://arxiv.org/abs/2011.01060) +- Upstream repository license: Apache-2.0. The paper is published under + CC BY 4.0. +- Provenance: passages and evidence also derive from Wikipedia and Wikidata. + Preserve their attribution when redistributing prepared data. + +### MultiHop-RAG + +- Purpose in DeepRefine-Skill: optional, slower cross-document RAG evaluation. +- Source: [yixuantt/MultiHop-RAG](https://github.com/yixuantt/MultiHop-RAG) +- Paper: [MultiHop-RAG: Benchmarking Retrieval-Augmented Generation for + Multi-Hop Queries](https://arxiv.org/abs/2401.15391) +- Upstream dataset license: ODC-BY. +- Provenance: its knowledge base contains news articles from third-party + publishers. Review the original URLs and applicable content rights before + redistributing article text. + +## Safe distribution pattern + +Prefer committing a preparation manifest containing upstream IDs, suite +version, profile, selection rules, and checksums. Download the source data from +its official location at preparation time. Do not copy a third-party subset +into a wheel or source distribution unless its license and required notices +have been reviewed. diff --git a/pyproject.toml b/pyproject.toml index d3b9efd..9d6b26c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,11 @@ dependencies = [ "openai>=1.0", ] +[project.optional-dependencies] +benchmark-semantic = [ + "bert-score>=0.3.13", +] + [project.urls] Homepage = "https://github.com/HKUST-KnowComp/DeepRefine-Skill" Documentation = "https://github.com/HKUST-KnowComp/DeepRefine-Skill#readme" @@ -57,4 +62,9 @@ deeprefine_skill = [ "gemini_extension/commands/*.toml", "gemini_extension/commands/deeprefine/*.toml", "gemini_extension/skills/deeprefine/SKILL.md", + "benchmark_suites/*.md", + "benchmark_suites/*/*.json", + "benchmark_suites/*/*.jsonl", + "benchmark_suites/*/*.md", + "benchmark_suites/*/corpus/*.txt", ] diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 0000000..25236e0 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,552 @@ +"""Public-contract tests for the lightweight benchmark CLI. + +The bundled synthetic suite is deliberately tiny. It validates the evaluator +and report plumbing without downloading or redistributing third-party data. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import deeprefine_skill +import pytest + +from deeprefine_skill.benchmarking.graph import ( + GraphNode, + align_entities, + bounded_bfs, + load_graphify_graph, + normalize_source, +) +from deeprefine_skill.benchmarking.evaluator import evaluate_suite +from deeprefine_skill.benchmarking.metrics import ( + answer_exact_match, + answer_token_f1, + lexical_graph_score, + precision_recall_f1, + reciprocal_rank, +) +from deeprefine_skill.benchmarking.prepare import prepare_suite +from deeprefine_skill.benchmarking.suite import sha256_file, verify_suite_lock +from deeprefine_skill.benchmarking.wiki import inspect_wiki_directory +from deeprefine_skill.cli import main + + +PACKAGE_ROOT = Path(deeprefine_skill.__file__).resolve().parent +SMOKE_SUITE_DIR = ( + PACKAGE_ROOT / "benchmark_suites" / "synthetic-smoke-v1" +) +SMOKE_SUITE = SMOKE_SUITE_DIR / "suite.json" +BASELINE_GRAPH = SMOKE_SUITE_DIR / "baseline_graph.json" +CANDIDATE_GRAPH = SMOKE_SUITE_DIR / "candidate_graph.json" +BASELINE_PREDICTIONS = SMOKE_SUITE_DIR / "baseline_predictions.jsonl" +CANDIDATE_PREDICTIONS = SMOKE_SUITE_DIR / "candidate_predictions.jsonl" + + +def _run_smoke_evaluation(output_dir: Path, *, predictions: bool = False) -> int: + args = [ + "benchmark", + "evaluate", + "--suite", + str(SMOKE_SUITE), + "--baseline-graph", + str(BASELINE_GRAPH), + "--candidate-graph", + str(CANDIDATE_GRAPH), + "--output-dir", + str(output_dir), + ] + if predictions: + args.extend( + [ + "--baseline-predictions", + str(BASELINE_PREDICTIONS), + "--candidate-predictions", + str(CANDIDATE_PREDICTIONS), + ] + ) + return main(args) + + +def test_benchmark_help_lists_public_subcommands( + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as exc_info: + main(["benchmark", "--help"]) + + assert exc_info.value.code == 0 + output = capsys.readouterr().out + assert "prepare" in output + assert "evaluate" in output + assert "report" in output + + +def test_packaged_synthetic_suite_uses_versioned_public_schema() -> None: + assert SMOKE_SUITE.is_file() + assert BASELINE_GRAPH.is_file() + assert CANDIDATE_GRAPH.is_file() + + suite = json.loads(SMOKE_SUITE.read_text(encoding="utf-8")) + assert suite["schema_version"] == 1 + assert suite["suite_id"] == "synthetic-smoke-v1" + assert suite["suite_version"] + assert suite["profile"] == "smoke" + assert suite["source"] + assert suite["cases"] + + tasks = {case["task"] for case in suite["cases"]} + assert tasks == {"intrinsic", "downstream"} + for case in suite["cases"]: + assert case["id"] + if case["task"] == "intrinsic": + assert case["source_files"] + assert case["gold_entities"] + assert case["gold_edges"] + else: + assert case["question"] + assert case["answers"] + assert case["seed_entities"] + assert case["answer_entities"] + assert case["evidence_edges"] + assert case["max_hops"] >= 1 + + +def test_benchmark_evaluate_writes_json_and_markdown(tmp_path: Path) -> None: + output_dir = tmp_path / "evaluation" + + exit_code = _run_smoke_evaluation(output_dir) + + assert exit_code == 0 + result_path = output_dir / "result.json" + report_path = output_dir / "report.md" + assert result_path.is_file() + assert report_path.is_file() + + result = json.loads(result_path.read_text(encoding="utf-8")) + assert result["schema_version"] == 1 + before = result["metrics"]["baseline"] + after = result["metrics"]["candidate"] + assert before["intrinsic"]["macro"]["strict_triple_f1"] < 1.0 + assert after["intrinsic"]["macro"]["strict_triple_f1"] == pytest.approx(1.0) + assert before["downstream"]["macro"]["complete_path_rate"] == pytest.approx(0.25) + assert after["downstream"]["macro"]["complete_path_rate"] == pytest.approx(1.0) + + report = report_path.read_text(encoding="utf-8") + assert "synthetic" in report.lower() + assert "baseline" in report.lower() + assert "candidate" in report.lower() + + +def test_benchmark_evaluate_accepts_optional_predictions(tmp_path: Path) -> None: + assert BASELINE_PREDICTIONS.is_file() + assert CANDIDATE_PREDICTIONS.is_file() + output_dir = tmp_path / "evaluation-with-predictions" + + exit_code = _run_smoke_evaluation(output_dir, predictions=True) + + assert exit_code == 0 + result = json.loads( + (output_dir / "result.json").read_text(encoding="utf-8") + ) + serialized = json.dumps(result, sort_keys=True).lower() + assert "answer" in serialized + assert ( + result["metrics"]["candidate"]["downstream"]["prediction"]["answer_f1"]["value"] + == pytest.approx(1.0) + ) + + +def test_benchmark_report_renders_an_existing_result(tmp_path: Path) -> None: + evaluation_dir = tmp_path / "evaluation" + assert _run_smoke_evaluation(evaluation_dir, predictions=True) == 0 + output = tmp_path / "combined-report.md" + + exit_code = main( + [ + "benchmark", + "report", + "--result", + str(evaluation_dir / "result.json"), + "--format", + "markdown", + "--output", + str(output), + ] + ) + + assert exit_code == 0 + assert output.is_file() + rendered = output.read_text(encoding="utf-8") + assert "synthetic" in rendered.lower() + assert "baseline" in rendered.lower() + assert "candidate" in rendered.lower() + + +def test_graph_loader_preserves_parallel_relations_and_diagnostics() -> None: + graph = load_graphify_graph( + { + "directed": False, + "multigraph": True, + "nodes": [ + {"id": "a", "label": "Ａlpha", "aliases": ["Alpha"]}, + {"id": "b", "label": "Beta"}, + ], + "edges": [ + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "a", "target": "b", "relation": "imports"}, + {"source": "b", "target": "a", "relation": "calls"}, + {"source": "a", "target": "missing", "relation": "calls"}, + {"source": "a", "target": "a", "relation": "self"}, + ], + } + ) + + assert len(graph.edges) == 4 + assert graph.diagnostics["duplicate_edges"] == 1 + assert graph.diagnostics["dangling_edges"] == 1 + assert graph.diagnostics["self_loops"] == 1 + assert graph.node_ids_for_aliases(["alpha"]) == ["a"] + assert bounded_bfs(graph, ["b"], 1)["a"] == 1 + + +def test_entity_alignment_uses_alias_and_source_path() -> None: + predicted = load_graphify_graph( + { + "nodes": [ + { + "id": "one", + "label": "main()", + "source_file": "src/first.py", + }, + { + "id": "two", + "label": "main()", + "source_file": "src/second.py", + }, + ], + "links": [], + } + ) + gold = [ + GraphNode("g1", "MAIN()", source_file="first.py"), + GraphNode("g2", "main()", source_file="second.py"), + ] + + assert align_entities(gold, predicted) == {"g1": "one", "g2": "two"} + assert normalize_source(r".\SRC\first.py") == "src/first.py" + + +def test_metric_primitives_are_hand_checkable() -> None: + score = precision_recall_f1(2, 1, 2) + assert score.precision == pytest.approx(2 / 3) + assert score.recall == pytest.approx(1 / 2) + assert score.f1 == pytest.approx(4 / 7) + assert precision_recall_f1(0, 0, 0).f1 == 1.0 + + identical = lexical_graph_score( + ["alpha; located in; beta"], + ["alpha; located in; beta"], + metric="rouge", + ) + mismatch = lexical_graph_score( + ["alpha; located in; beta"], + ["gamma; created; delta"], + metric="rouge", + ) + assert identical["f1"] == pytest.approx(1.0) + assert mismatch["f1"] < identical["f1"] + + assert answer_exact_match("The Orion-Database!", ["orion database"]) == 1.0 + assert answer_token_f1("Orion Database", ["the Orion Database"]) == 1.0 + assert reciprocal_rank(["wrong", "answer"], ["answer"]) == 0.5 + + +def test_suite_lock_and_input_graphs_are_unchanged(tmp_path: Path) -> None: + verify_suite_lock(SMOKE_SUITE_DIR) + baseline_before = sha256_file(BASELINE_GRAPH) + candidate_before = sha256_file(CANDIDATE_GRAPH) + + assert _run_smoke_evaluation(tmp_path / "read-only") == 0 + + assert sha256_file(BASELINE_GRAPH) == baseline_before + assert sha256_file(CANDIDATE_GRAPH) == candidate_before + + +def test_repeated_evaluation_is_deterministic_except_runtime() -> None: + first = evaluate_suite( + SMOKE_SUITE, + BASELINE_GRAPH, + CANDIDATE_GRAPH, + baseline_predictions=BASELINE_PREDICTIONS, + candidate_predictions=CANDIDATE_PREDICTIONS, + ) + second = evaluate_suite( + SMOKE_SUITE, + BASELINE_GRAPH, + CANDIDATE_GRAPH, + baseline_predictions=BASELINE_PREDICTIONS, + candidate_predictions=CANDIDATE_PREDICTIONS, + ) + first.pop("runtime") + second.pop("runtime") + + assert first == second + + +def test_empty_ranked_predictions_are_scored_as_zero(tmp_path: Path) -> None: + predictions = tmp_path / "empty.jsonl" + predictions.write_text( + "\n".join( + json.dumps( + { + "case_id": f"qa-{index:02d}", + "answer": "", + "retrieved_nodes": [], + "retrieved_supporting_facts": [], + } + ) + for index in range(1, 9) + ) + + "\n", + encoding="utf-8", + ) + + result = evaluate_suite( + SMOKE_SUITE, + BASELINE_GRAPH, + CANDIDATE_GRAPH, + baseline_predictions=predictions, + candidate_predictions=predictions, + ) + prediction = result["metrics"]["baseline"]["downstream"]["prediction"] + + assert prediction["hit_at_5"]["value"] == 0.0 + assert prediction["mrr"]["value"] == 0.0 + assert prediction["supporting_fact_recall_at_5"]["value"] == 0.0 + + +def test_prepare_copies_builtin_smoke_suite(tmp_path: Path) -> None: + destination = prepare_suite( + "synthetic-smoke-v1", + "smoke", + tmp_path / "prepared", + ) + + assert (destination / "suite.json").is_file() + assert (destination / "corpus" / "doc-alpha.txt").is_file() + verify_suite_lock(destination) + + +def test_cli_prepare_defaults_synthetic_to_smoke(tmp_path: Path) -> None: + destination = tmp_path / "cli-prepared" + + assert ( + main( + [ + "benchmark", + "prepare", + "--suite", + "synthetic-smoke-v1", + "--output-dir", + str(destination), + ] + ) + == 0 + ) + suite = json.loads((destination / "suite.json").read_text(encoding="utf-8")) + assert suite["profile"] == "smoke" + + +def test_suite_lock_rejects_tampered_fixture(tmp_path: Path) -> None: + destination = prepare_suite( + "synthetic-smoke-v1", + "smoke", + tmp_path / "tampered", + ) + (destination / "corpus" / "doc-alpha.txt").write_text( + "tampered\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="checksum mismatch"): + verify_suite_lock(destination) + + +def test_prepare_redocred_quick_is_deterministic(tmp_path: Path) -> None: + source = tmp_path / "dev_revised.json" + documents = [ + { + "title": f"Document {index}", + "sents": [["Entity", str(index), "links", "Target"]], + "vertexSet": [ + [{"name": f"Entity {index}", "sent_id": 0}], + [{"name": f"Target {index}", "sent_id": 0}], + ], + "labels": [{"h": 0, "t": 1, "r": "P17"}], + } + for index in range(10) + ] + source.write_text(json.dumps(documents), encoding="utf-8") + + first = prepare_suite( + "redocred-mini-v1", + "quick", + tmp_path / "first", + source=source, + ) + second = prepare_suite( + "redocred-mini-v1", + "quick", + tmp_path / "second", + source=source, + ) + + assert (first / "suite.json").read_bytes() == (second / "suite.json").read_bytes() + suite = json.loads((first / "suite.json").read_text(encoding="utf-8")) + assert len(suite["cases"]) == 10 + assert all(case["task"] == "intrinsic" for case in suite["cases"]) + verify_suite_lock(first) + + +def test_prepare_2wiki_quick_is_balanced_and_keeps_distractors( + tmp_path: Path, +) -> None: + source = tmp_path / "2wiki.json" + examples = [] + question_types = ( + "compositional", + "comparison", + "inference", + "bridge_comparison", + ) + context = [ + [f"Passage {index}", [f"Distractor or evidence sentence {index}."]] + for index in range(10) + ] + for question_type in question_types: + for index in range(4): + examples.append( + { + "_id": f"{question_type}-{index}", + "type": question_type, + "question": f"Question {question_type} {index}?", + "answer": "Answer Entity", + "context": context, + "supporting_facts": [["Passage 0", 0], ["Passage 1", 0]], + "evidences": [ + ["Seed Entity", "links to", "Bridge Entity"], + ["Bridge Entity", "answers with", "Answer Entity"], + ], + } + ) + source.write_text(json.dumps(examples), encoding="utf-8") + + destination = prepare_suite( + "2wiki-mini-v1", + "quick", + tmp_path / "prepared-2wiki", + source=source, + ) + + suite = json.loads((destination / "suite.json").read_text(encoding="utf-8")) + assert len(suite["cases"]) == 16 + assert { + question_type: sum( + case["question_type"] == question_type for case in suite["cases"] + ) + for question_type in question_types + } == {question_type: 4 for question_type in question_types} + assert len(list((destination / "corpus").glob("*.txt"))) == 10 + assert all(len(case["evidence_edges"]) == 2 for case in suite["cases"]) + verify_suite_lock(destination) + + +def test_wiki_integrity_check_distinguishes_broken_and_orphan_pages( + tmp_path: Path, +) -> None: + wiki = tmp_path / "wiki" + wiki.mkdir() + (wiki / "index.md").write_text( + "[Entity](entity.md)\n[Missing](missing.md)\n", + encoding="utf-8", + ) + (wiki / "entity.md").write_text("[Home](index.md)\n", encoding="utf-8") + (wiki / "orphan.md").write_text("No incoming links.\n", encoding="utf-8") + + result = inspect_wiki_directory(wiki) + + assert result["index_exists"] is True + assert result["broken_link_count"] == 1 + assert result["orphan_pages"] == ["orphan.md"] + + +def test_invalid_graph_returns_nonzero_without_partial_result( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + invalid = tmp_path / "invalid.json" + invalid.write_text('{"nodes": []}', encoding="utf-8") + output = tmp_path / "invalid-output" + + exit_code = main( + [ + "benchmark", + "evaluate", + "--suite", + str(SMOKE_SUITE), + "--baseline-graph", + str(invalid), + "--candidate-graph", + str(CANDIDATE_GRAPH), + "--output-dir", + str(output), + ] + ) + + captured = capsys.readouterr() + assert exit_code == 2 + assert "links" in captured.err or "edges" in captured.err + assert not (output / "result.json").exists() + + +def test_readme_smoke_table_matches_measured_values() -> None: + result = evaluate_suite( + SMOKE_SUITE, + BASELINE_GRAPH, + CANDIDATE_GRAPH, + baseline_predictions=BASELINE_PREDICTIONS, + candidate_predictions=CANDIDATE_PREDICTIONS, + ) + readme = (PACKAGE_ROOT.parent / "README.md").read_text(encoding="utf-8") + before = result["metrics"]["baseline"] + after = result["metrics"]["candidate"] + expected_rows = ( + ( + "Entity F1", + before["intrinsic"]["macro"]["entity_f1"], + after["intrinsic"]["macro"]["entity_f1"], + ), + ( + "Strict Triple F1", + before["intrinsic"]["macro"]["strict_triple_f1"], + after["intrinsic"]["macro"]["strict_triple_f1"], + ), + ( + "Evidence Edge Recall", + before["downstream"]["macro"]["evidence_edge_recall"], + after["downstream"]["macro"]["evidence_edge_recall"], + ), + ( + "Complete Path Rate", + before["downstream"]["macro"]["complete_path_rate"], + after["downstream"]["macro"]["complete_path_rate"], + ), + ) + for label, baseline, candidate in expected_rows: + delta = candidate - baseline + row = ( + f"| synthetic-smoke-v1 | {label} | {baseline * 100:.2f}% | " + f"{candidate * 100:.2f}% | {delta * 100:+.2f} pp |" + ) + assert row in readme