From eeb1018d2b9349694f257b7a177107babd3d5c4a Mon Sep 17 00:00:00 2001 From: carlos-alm Date: Tue, 18 Aug 2026 17:09:36 -0600 Subject: [PATCH] test(hcl): mark resolution-benchmark fixture's module-reference edges as aspirational expected-edges.json declares 2 calls edges (module.user_service -> module.repository, -> module.validators), but building the fixture yields 31 nodes, 27 edges, all contains -- verified empirically, 0 of 2 expected edges materialize. The benchmark currently passes only because HCL's threshold is 0.0/0.0, so the fixture and its manifest were encoding an aspiration no test actually enforces. Unlike cuda/groovy/verilog's own 0.0/0.0 entries (call resolution not implemented yet, for the same kind of edge every other language has), HCL has no functions/calls at all -- these edges describe a genuinely different, unimplemented notion of module-source-reference linkage. Documents this explicitly in both the fixture's own description and an inline comment next to the threshold, rather than leaving a reader to mistake it for a passing, enforced capability. Closes #2524 --- .../benchmarks/resolution/fixtures/hcl/expected-edges.json | 2 +- tests/benchmarks/resolution/resolution-benchmark.test.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/resolution/fixtures/hcl/expected-edges.json b/tests/benchmarks/resolution/fixtures/hcl/expected-edges.json index 3fa4f9fbe..e9891da82 100644 --- a/tests/benchmarks/resolution/fixtures/hcl/expected-edges.json +++ b/tests/benchmarks/resolution/fixtures/hcl/expected-edges.json @@ -1,7 +1,7 @@ { "$schema": "../../expected-edges.schema.json", "language": "hcl", - "description": "Hand-annotated call edges for HCL/Terraform resolution benchmark. HCL uses module references rather than function calls — edges represent module source references.", + "description": "ASPIRATIONAL, not currently enforced (see resolution-benchmark.test.ts's hcl threshold, pinned at 0.0/0.0, and issue #2524): hand-annotated module-reference edges for a HCL/Terraform resolution capability that does not exist yet. HCL has no functions/calls to resolve in the traditional sense — these edges describe a genuinely different, unimplemented notion of module-source-reference linkage ('module.user_service references module.repository'), not a gap in the same call-resolution machinery other 0.0/0.0 languages (cuda/groovy/verilog) are missing. Building this fixture today yields 0 of these 2 edges; every edge that does materialize is a structural 'contains' edge.", "edges": [ { "source": { "name": "module.user_service", "file": "main.tf" }, diff --git a/tests/benchmarks/resolution/resolution-benchmark.test.ts b/tests/benchmarks/resolution/resolution-benchmark.test.ts index 25dd707fc..23c4641ff 100644 --- a/tests/benchmarks/resolution/resolution-benchmark.test.ts +++ b/tests/benchmarks/resolution/resolution-benchmark.test.ts @@ -245,6 +245,12 @@ const THRESHOLDS: Record = { cuda: { precision: 0.0, recall: 0.0 }, groovy: { precision: 0.0, recall: 0.0 }, verilog: { precision: 0.0, recall: 0.0 }, + // hcl's fixture (unlike cuda/groovy/verilog above) isn't just "call + // resolution not implemented yet" for the same kind of edge every other + // language has — HCL has no functions/calls at all. Its expected-edges.json + // aspirationally describes a genuinely different, unimplemented capability + // (module-source-reference linkage) that 0 of its own edges currently + // materialize (see that file's own description and issue #2524). hcl: { precision: 0.0, recall: 0.0 }, };