From 9f7d084cb38f0c1436905c69ea87678fc282c988 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:18:17 -0700 Subject: [PATCH 01/14] fix(python): an operator or a truth test runs its operand's dunder `a == b`, `a | b`, `x in c`, `-a`, `if x:`, `not x`, `x or y` compile to no call, so the client __eq__, __or__, __contains__, __bool__ or __len__ they run had no caller: impact listed nothing that compares, composes or truth-tests the object, and a test doing exactly that was not selected. Two new protocol edges on the existing channel, each its own kind: OPERATOR_PROTOCOL the left operand's slot and the right operand's reflected one (a comparison's mirror, `a < b` -> b.__gt__), `!=` falling back to __eq__, `in` running the right operand's __contains__ (or __iter__), and the unary -, +, ~. A chained comparison matches no catalogue row and emits nothing. TRUTH_PROTOCOL if / while / assert conditions, a conditional or comprehension condition, `not x`, and the operands of `and`/`or` -- the right one only when the whole expression is tested (`return a or b` hands b back untested). __bool__, or __len__ when the class has none. Operators and dunders are catalogued in builtins.dl (the data model's tables). Mutation oracle, sixteen repositories: +5 failing test files selected (one tuning, four held-out), 9 extra, none lost; empty answers 96 -> 92; impact's one-hop callers src recall 0.707 -> 0.714 (tuning 0.726 -> 0.736); a failing test's path to the change found 0.528 -> 0.541. Edges +0.8%, index time flat. Case operator-runs-the-dunder fails 6 of 8 on the base; controls: a class with its own __bool__ is never tested through __len__, and the untested right operand of `or` reaches nothing. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- graph/bundle/SCHEMA.md | 4 +- graph/bundle/schema.ts | 4 +- .../engine/call-edge-generation/call_chain.dl | 78 +++++++++++++++++++ graph/python/engine/resolution/builtins.dl | 49 ++++++++++++ graph/python/souffle/decls_all.dl | 16 ++++ .../operator-runs-the-dunder/app/__init__.py | 0 .../operator-runs-the-dunder/app/shapes.py | 71 +++++++++++++++++ .../python/operator-runs-the-dunder/case.json | 30 +++++++ .../operator-runs-the-dunder/tests/test_eq.py | 5 ++ .../tests/test_fallback.py | 6 ++ .../tests/test_gated.py | 6 ++ .../operator-runs-the-dunder/tests/test_in.py | 5 ++ .../operator-runs-the-dunder/tests/test_or.py | 5 ++ .../tests/test_truth.py | 11 +++ 14 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 tests/cases/python/operator-runs-the-dunder/app/__init__.py create mode 100644 tests/cases/python/operator-runs-the-dunder/app/shapes.py create mode 100644 tests/cases/python/operator-runs-the-dunder/case.json create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_eq.py create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_fallback.py create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_gated.py create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_in.py create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_or.py create mode 100644 tests/cases/python/operator-runs-the-dunder/tests/test_truth.py diff --git a/graph/bundle/SCHEMA.md b/graph/bundle/SCHEMA.md index 8a5c923b..a3f36c76 100644 --- a/graph/bundle/SCHEMA.md +++ b/graph/bundle/SCHEMA.md @@ -442,7 +442,7 @@ One row per place a call is written (or, for a synthesised edge, the construct t - **typescript** — end_line / end_column come from the expression row; the call-site row itself records only the start. - **javascript** — caller_id is the parser's enclosing method, or the module initializer for top-level code. end_line / end_column come from the expression row. `require()` is a module edge, not a call site. - **typescript** — PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor's name is on the callee's methods row. Filter them out with kind NOT IN (…) when counting calls. -- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. +- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. - **python** — The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class's TYPE hash for METACLASS_CREATION, positioned at the class declaration. ### `call_edges` @@ -537,6 +537,8 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `ITERATION_PROTOCOL` | python | `for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression. | | `SUBSCRIPT_PROTOCOL` | python | `x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver's class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call. | | `BUILTIN_PROTOCOL` | python | `repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument's class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call. | +| `OPERATOR_PROTOCOL` | python | An operator runs a dunder of its operands: `a + b` the left operand's `__add__` and the right one's `__radd__`, a comparison the left slot and the right one's mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand's `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression. | +| `TRUTH_PROTOCOL` | python | A truth test runs the tested value's `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression. | **`call_edges.tier` values** diff --git a/graph/bundle/schema.ts b/graph/bundle/schema.ts index 306817ac..3ff97670 100644 --- a/graph/bundle/schema.ts +++ b/graph/bundle/schema.ts @@ -673,6 +673,8 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'ITERATION_PROTOCOL', languages: P, meaning: '`for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression.' }, { table: 'call_edges', column: 'kind', value: 'SUBSCRIPT_PROTOCOL', languages: P, meaning: '`x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver\'s class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call.' }, { table: 'call_edges', column: 'kind', value: 'BUILTIN_PROTOCOL', languages: P, meaning: '`repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument\'s class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call.' }, + { table: 'call_edges', column: 'kind', value: 'OPERATOR_PROTOCOL', languages: P, meaning: 'An operator runs a dunder of its operands: `a + b` the left operand\'s `__add__` and the right one\'s `__radd__`, a comparison the left slot and the right one\'s mirror (`a < b` -> `b.__gt__`, `!=` falls back to `__eq__`), `x in c` the right operand\'s `__contains__` (or `__iter__`), `-a`/`+a`/`~a` its `__neg__`/`__pos__`/`__invert__`. No written call; the site is the operator expression.' }, + { table: 'call_edges', column: 'kind', value: 'TRUTH_PROTOCOL', languages: P, meaning: 'A truth test runs the tested value\'s `__bool__`, or `__len__` when its class has none: `if x:`, `while x:`, `assert x`, `not x`, a conditional or comprehension condition, and an operand of `and`/`or` (the right one only when the whole expression is tested). No written call; the site is the tested expression.' }, // entry_points.reason { table: 'entry_points', column: 'reason', value: 'main', languages: ['java', 'csharp'], meaning: 'A static `main`. C#: a static `Main`, or the method top-level statements compile to.' }, @@ -751,7 +753,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'typescript', table: 'call_sites', note: 'PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor\'s name is on the callee\'s methods row. Filter them out with kind NOT IN (…) when counting calls.' }, - { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, + { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, OPERATOR_PROTOCOL, TRUTH_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'java', table: 'call_edges', note: 'A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform\'s own subclasses, are the other possible targets. Stage the library to replace the label with the real method.' }, diff --git a/graph/python/engine/call-edge-generation/call_chain.dl b/graph/python/engine/call-edge-generation/call_chain.dl index c527a151..ade21045 100644 --- a/graph/python/engine/call-edge-generation/call_chain.dl +++ b/graph/python/engine/call-edge-generation/call_chain.dl @@ -709,6 +709,72 @@ builtin_protocol_edge(x, caller, m) :- expr_ultimate_method("client", x, caller). type_has_member(t, d) :- builtin_protocol_fallback(_, d, _), mro_lookup("client", t, d, _). +// ── OPERATOR PROTOCOL — `a + b`, `a == b`, `x in c`, `-a` run a dunder ──────── +// An operator compiles to BINARY_OP / COMPARE_OP / CONTAINS_OP / UNARY_*, never to a CALL, +// so there is no written site, and the dunder it runs -- a client `__eq__`, `__or__`, +// `__contains__` -- had no caller at all: a test asserting `a == b` or composing two +// strategies with `|` reached nothing of theirs. Same channel as the protocol edges above, +// its own kind. The site is the operator expression. Both halves of the protocol are +// emitted, because both can run: the left operand's slot, and the right operand's +// reflected one when the left is missing it or returns NotImplemented (a comparison's +// mirror, `a < b` -> b.__gt__). The operators and their dunders are catalogued in +// resolution/builtins.dl. A chained comparison (`a < b < c`) carries several operators in +// one node and matches no catalogue row, so it emits nothing rather than a guess. +operator_protocol_edge(e, caller, m) :- + expr_operator("client", op, e), operator_protocol_slot(op, d, _), + expr_parent("client", e, "OPERAND_LEFT", _, l), expr_type("client", l, t), + mro_lookup("client", t, d, m), expr_ultimate_method("client", e, caller). +operator_protocol_edge(e, caller, m) :- + expr_operator("client", op, e), operator_protocol_slot(op, _, d), + expr_parent("client", e, "OPERAND_RIGHT", _, r), expr_type("client", r, t), + mro_lookup("client", t, d, m), expr_ultimate_method("client", e, caller). +// `a != b` on a class with no __ne__ of its own runs __eq__ through object.__ne__. +operator_protocol_edge(e, caller, m) :- + expr_operator("client", op, e), operator_protocol_slot(op, d, _), operator_protocol_fallback(d, d2), + expr_parent("client", e, "OPERAND_LEFT", _, l), expr_type("client", l, t), + !operator_type_has(t, d), mro_lookup("client", t, d2, m), expr_ultimate_method("client", e, caller). +operator_type_has(t, d) :- operator_protocol_fallback(d, _), mro_lookup("client", t, d, _). +// `x in c` runs the RIGHT operand's __contains__, or iterates it when it has none. +operator_protocol_edge(e, caller, m) :- + expr_operator("client", op, e), membership_operator(op), membership_protocol_slot(d), + expr_parent("client", e, "OPERAND_RIGHT", _, r), expr_type("client", r, t), + mro_lookup("client", t, d, m), expr_ultimate_method("client", e, caller). +operator_protocol_edge(e, caller, m) :- + expr_operator("client", op, e), membership_operator(op), membership_protocol_fallback(d, d2), + expr_parent("client", e, "OPERAND_RIGHT", _, r), expr_type("client", r, t), + !membership_type_has(t, d), mro_lookup("client", t, d2, m), expr_ultimate_method("client", e, caller). +membership_type_has(t, d) :- membership_protocol_fallback(d, _), mro_lookup("client", t, d, _). +// `-a`, `+a`, `~a` +operator_protocol_edge(e, caller, m) :- + expr_node("client", "UNARY_OPERATION", _, _, e), expr_operator("client", op, e), unary_protocol_slot(op, d), + expr_parent("client", e, "UNARY_OPERAND", _, x), expr_type("client", x, t), + mro_lookup("client", t, d, m), expr_ultimate_method("client", e, caller). + +// ── TRUTH PROTOCOL — `if x:` runs type(x).__bool__, or __len__ ─────────────── +// Truth testing has no written call either: `if x:`, `while x:`, `assert x`, `not x`, the +// condition of `a if x else b` and of a comprehension, and an operand of `and` / `or` all +// run x's __bool__ -- or __len__ when the class defines no __bool__ (data model, +// object.__bool__). The left operand of `and`/`or` is always tested; the right one only +// when the whole expression is (`return a or b` hands b back untested). The site is the +// tested expression. +truth_tested(x) :- + truth_test_root(rc), expr_root_context("client", rc, x), !expr_has_parent("client", x). +truth_tested(x) :- truth_test_role(role), expr_parent("client", _, role, _, x). +truth_tested(x) :- + expr_node("client", "UNARY_OPERATION", _, _, e), expr_operator("client", op, e), truth_negation_operator(op), + expr_parent("client", e, "UNARY_OPERAND", _, x). +truth_tested(x) :- expr_node("client", "BOOLEAN_OPERATION", _, _, e), expr_parent("client", e, "OPERAND_LEFT", _, x). +truth_tested(x) :- + expr_node("client", "BOOLEAN_OPERATION", _, _, e), truth_tested(e), expr_parent("client", e, "OPERAND_RIGHT", _, x). + +truth_protocol_edge(x, caller, m) :- + truth_tested(x), truth_protocol_fallback(d, _), expr_type("client", x, t), + mro_lookup("client", t, d, m), expr_ultimate_method("client", x, caller). +truth_protocol_edge(x, caller, m) :- + truth_tested(x), truth_protocol_fallback(d, d2), expr_type("client", x, t), + !truth_type_has(t, d), mro_lookup("client", t, d2, m), expr_ultimate_method("client", x, caller). +truth_type_has(t, d) :- truth_protocol_fallback(d, _), mro_lookup("client", t, d, _). + // ── THE THREE PROTOCOL EDGES ARE TIERED BY TARGET COUNT, like every other edge ── // // A property read, a context-manager entry and an iteration all reach their target @@ -753,6 +819,8 @@ protocol_edge(e, d, caller, m) :- with_protocol_edge(e, caller, m), method_decl( protocol_edge(e, d, caller, m) :- iter_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- subscript_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- builtin_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). +protocol_edge(e, d, caller, m) :- operator_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). +protocol_edge(e, d, caller, m) :- truth_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). // Aggregate in its own stratum, mirroring site_client_target_count above. protocol_edge_target_count(e, d, n) :- @@ -843,6 +911,16 @@ call_chain_edge(x, caller, "-", m, "client", cls, "BUILTIN_PROTOCOL") :- builtin_protocol_edge(x, caller, m), method_decl(_, d, _, _, _, m), protocol_edge_class(x, d, cls). +// OPERATOR PROTOCOL — an operator's dunder. Its own kind: there is no written call. +call_chain_edge(e, caller, "-", m, "client", cls, "OPERATOR_PROTOCOL") :- + operator_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m), + protocol_edge_class(e, d, cls). + +// TRUTH PROTOCOL — the __bool__ (or __len__) a truth test runs. Its own kind likewise. +call_chain_edge(x, caller, "-", m, "client", cls, "TRUTH_PROTOCOL") :- + truth_protocol_edge(x, caller, m), method_decl(_, d, _, _, _, m), + protocol_edge_class(x, d, cls). + // A site whose caller could not be determined AT ALL would vanish from every rule // above. That must be impossible (the parser guarantees a non-empty owner), so it is // asserted rather than assumed: any such site is emitted with caller "-" so the diff --git a/graph/python/engine/resolution/builtins.dl b/graph/python/engine/resolution/builtins.dl index f530f72d..e08902e1 100644 --- a/graph/python/engine/resolution/builtins.dl +++ b/graph/python/engine/resolution/builtins.dl @@ -388,6 +388,55 @@ builtin_protocol_slot("reversed", "__reversed__"). builtin_protocol_fallback("str", "__str__", "__repr__"). builtin_protocol_fallback("bool", "__bool__", "__len__"). +// ── operator_protocol_slot(Operator, Dunder, ReflectedDunder) ──────────────── +// `a + b` runs type(a).__add__, and type(b).__radd__ when that is missing or returns +// NotImplemented; a comparison runs the left operand's slot and the right operand's mirror +// (`a < b` -> b.__gt__, `a == b` -> b.__eq__). The language reference's data model, 3.3.8 +// (numeric) and 3.3.1 (rich comparison). Read by call-edge-generation/call_chain.dl's +// OPERATOR PROTOCOL edges. +operator_protocol_slot("+", "__add__", "__radd__"). +operator_protocol_slot("-", "__sub__", "__rsub__"). +operator_protocol_slot("*", "__mul__", "__rmul__"). +operator_protocol_slot("@", "__matmul__", "__rmatmul__"). +operator_protocol_slot("/", "__truediv__", "__rtruediv__"). +operator_protocol_slot("//", "__floordiv__", "__rfloordiv__"). +operator_protocol_slot("%", "__mod__", "__rmod__"). +operator_protocol_slot("**", "__pow__", "__rpow__"). +operator_protocol_slot("<<", "__lshift__", "__rlshift__"). +operator_protocol_slot(">>", "__rshift__", "__rrshift__"). +operator_protocol_slot("&", "__and__", "__rand__"). +operator_protocol_slot("|", "__or__", "__ror__"). +operator_protocol_slot("^", "__xor__", "__rxor__"). +operator_protocol_slot("==", "__eq__", "__eq__"). +operator_protocol_slot("!=", "__ne__", "__ne__"). +operator_protocol_slot("<", "__lt__", "__gt__"). +operator_protocol_slot("<=", "__le__", "__ge__"). +operator_protocol_slot(">", "__gt__", "__lt__"). +operator_protocol_slot(">=", "__ge__", "__le__"). +// `a != b` on a class with no __ne__ of its own runs object.__ne__, which calls __eq__. +operator_protocol_fallback("__ne__", "__eq__"). +// ── membership_operator(Operator) — `x in c` runs type(c).__contains__ ─────── +// and, for a class with no __contains__, iterates it (__iter__). Data model 3.3.7. +membership_operator("in"). +membership_operator("not in"). +membership_protocol_slot("__contains__"). +membership_protocol_fallback("__contains__", "__iter__"). +// ── unary_protocol_slot(Operator, Dunder) ──────────────────────────────────── +unary_protocol_slot("-", "__neg__"). +unary_protocol_slot("+", "__pos__"). +unary_protocol_slot("~", "__invert__"). +// ── truth testing — `if x:`, `while x:`, `not x`, `x and y`, `assert x` ─────── +// runs type(x).__bool__, or __len__ when the class has no __bool__ (data model 3.3.1, +// object.__bool__). truth_negation_operator is the one unary operator that truth-tests its +// operand; truth_test_root the statement contexts whose expression is truth-tested. +truth_negation_operator("not"). +truth_protocol_fallback("__bool__", "__len__"). +truth_test_root("IF_CONDITION"). +truth_test_root("WHILE_CONDITION"). +truth_test_root("ASSERT_CONDITION"). +truth_test_role("CONDITION"). +truth_test_role("COMPREHENSION_CONDITION"). + // ── dict_view_name / dict_view_slot / dict_items_view — a mapping's views ──── // `keys()` yields the key (generic position 0), `values()` the value (position 1), and // `items()` a (key, value) pair. Read by resolution/iteration.dl. diff --git a/graph/python/souffle/decls_all.dl b/graph/python/souffle/decls_all.dl index c221749a..08cc0035 100644 --- a/graph/python/souffle/decls_all.dl +++ b/graph/python/souffle/decls_all.dl @@ -690,6 +690,22 @@ .decl builtin_protocol_fallback(c0:symbol,c1:symbol,c2:symbol) .decl builtin_protocol_arg(c0:symbol,c1:symbol,c2:symbol) .decl builtin_protocol_edge(c0:symbol,c1:symbol,c2:symbol) +.decl operator_protocol_slot(c0:symbol,c1:symbol,c2:symbol) +.decl operator_protocol_fallback(c0:symbol,c1:symbol) +.decl membership_operator(c0:symbol) +.decl membership_protocol_slot(c0:symbol) +.decl membership_protocol_fallback(c0:symbol,c1:symbol) +.decl unary_protocol_slot(c0:symbol,c1:symbol) +.decl truth_negation_operator(c0:symbol) +.decl truth_protocol_fallback(c0:symbol,c1:symbol) +.decl truth_test_root(c0:symbol) +.decl truth_test_role(c0:symbol) +.decl operator_protocol_edge(c0:symbol,c1:symbol,c2:symbol) +.decl operator_type_has(c0:symbol,c1:symbol) +.decl membership_type_has(c0:symbol,c1:symbol) +.decl truth_tested(c0:symbol) +.decl truth_protocol_edge(c0:symbol,c1:symbol,c2:symbol) +.decl truth_type_has(c0:symbol,c1:symbol) .decl type_has_member(c0:symbol,c1:symbol) // staged beside the client IR by graph/python/libsum/libsum.py (framework-behavior/library-callbacks.dl) .decl py_lib_callback(callee:symbol, root:symbol, pos:symbol, path:symbol, member:symbol, via:symbol, type:symbol) diff --git a/tests/cases/python/operator-runs-the-dunder/app/__init__.py b/tests/cases/python/operator-runs-the-dunder/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/operator-runs-the-dunder/app/shapes.py b/tests/cases/python/operator-runs-the-dunder/app/shapes.py new file mode 100644 index 00000000..bec0c6d3 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/app/shapes.py @@ -0,0 +1,71 @@ +def compare_keys(): + return True + + +def combine_ids(): + return 7 + + +def member_count(): + return 1 + + +def has_items(): + return True + + +def plain_size(): + return 2 + + +def gated_flag(): + return True + + +def gated_size(): + return 0 + + +def fallback_flag(): + return True + + +class Key: + def __init__(self, value): + self.value = value + + def __eq__(self, other): + return compare_keys() + + +class Strategy: + def __or__(self, other): + return combine_ids() + + +class Bucket: + def __contains__(self, item): + return item == member_count() + + +class Batch: + def __bool__(self): + return has_items() + + +class Sized: + def __len__(self): + return plain_size() + + +class Gated: + def __bool__(self): + return gated_flag() + + def __len__(self): + return gated_size() + + +class Fallback: + def __bool__(self): + return fallback_flag() diff --git a/tests/cases/python/operator-runs-the-dunder/case.json b/tests/cases/python/operator-runs-the-dunder/case.json new file mode 100644 index 00000000..2f33be49 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/case.json @@ -0,0 +1,30 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a == b runs the left operand's __eq__: a test comparing two keys covers what __eq__ calls", + "run": ["impact", "compare_keys", "--tests"], + "want": ["test_eq.py"], + "avoid": ["test_or.py", "test_in.py"]}, + {"why": "a | b runs __or__: composing two strategies covers what __or__ calls", + "run": ["impact", "combine_ids", "--tests"], + "want": ["test_or.py"], + "avoid": ["test_eq.py"]}, + {"why": "x in c runs the RIGHT operand's __contains__", + "run": ["impact", "member_count", "--tests"], + "want": ["test_in.py"]}, + {"why": "if x: runs __bool__, and assert x on a class with no __bool__ runs __len__", + "run": ["impact", "has_items", "--tests"], + "want": ["test_truth.py"]}, + {"why": "truth testing falls back to __len__ when the class defines no __bool__", + "run": ["impact", "plain_size", "--tests"], + "want": ["test_truth.py"]}, + {"why": "not x truth-tests x through its own __bool__", + "run": ["impact", "gated_flag", "--tests"], + "want": ["test_gated.py"]}, + {"why": "CONTROL: a class that defines __bool__ is never truth-tested through its __len__", + "run": ["impact", "gated_size", "--tests"], + "want": ["tests: 0 of"], + "avoid": ["test_gated.py"]}, + {"why": "CONTROL: `None or Fallback()` returns the right operand untested, and `is not` runs no dunder", + "run": ["impact", "fallback_flag", "--tests"], + "want": ["tests: 0 of"], + "avoid": ["test_fallback.py"]}]} diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_eq.py b/tests/cases/python/operator-runs-the-dunder/tests/test_eq.py new file mode 100644 index 00000000..4c83263b --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_eq.py @@ -0,0 +1,5 @@ +from app.shapes import Key + + +def test_equal(): + assert Key(1) == Key(1) diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_fallback.py b/tests/cases/python/operator-runs-the-dunder/tests/test_fallback.py new file mode 100644 index 00000000..9bc1015a --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_fallback.py @@ -0,0 +1,6 @@ +from app.shapes import Fallback + + +def test_fallback(): + value = None or Fallback() + assert value is not None diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_gated.py b/tests/cases/python/operator-runs-the-dunder/tests/test_gated.py new file mode 100644 index 00000000..b82c5628 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_gated.py @@ -0,0 +1,6 @@ +from app.shapes import Gated + + +def test_gated(): + if not Gated(): + raise AssertionError diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_in.py b/tests/cases/python/operator-runs-the-dunder/tests/test_in.py new file mode 100644 index 00000000..c26ed9f0 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_in.py @@ -0,0 +1,5 @@ +from app.shapes import Bucket + + +def test_member(): + assert 1 in Bucket() diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_or.py b/tests/cases/python/operator-runs-the-dunder/tests/test_or.py new file mode 100644 index 00000000..a69e3ce0 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_or.py @@ -0,0 +1,5 @@ +from app.shapes import Strategy + + +def test_compose(): + assert Strategy() | Strategy() == 7 diff --git a/tests/cases/python/operator-runs-the-dunder/tests/test_truth.py b/tests/cases/python/operator-runs-the-dunder/tests/test_truth.py new file mode 100644 index 00000000..385ee990 --- /dev/null +++ b/tests/cases/python/operator-runs-the-dunder/tests/test_truth.py @@ -0,0 +1,11 @@ +from app.shapes import Batch, Sized + + +def test_batch(): + if Batch(): + return + raise AssertionError + + +def test_sized(): + assert Sized() From a4119efde6845052d2330e522840dce70b8a6045 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:58:08 -0700 Subject: [PATCH 02/14] fix(python): a library class named through a dotted base, a star re-export or a generic alias is that class A client class whose base is a library class was recognised only when the base was a bare name or a subscript. Three spellings of the same base left it a plain unresolved class, so the library's own methods never called back the overrides it defines: * a DOTTED base, `class Command(clikit.core.Command)` / `class Store(abc.Mapping)`: the base list's DOTTED_NAME slot is read like NAME (138 such classes across the sixteen repositories, every one of them); * a name a library re-exports with `from M import *` (`collections.abc.MutableMapping` is `_collections_abc.MutableMapping`): libsum now loads every module first and expands star imports (M's __all__ when it declares one, else its public names) before analysing any body, so the alias table has the row; * a module-level generic alias built around a class, `Mapping = _alias(collections.abc. Mapping, 2)`: followed only when the first argument resolves to a library CLASS, so a `partial(f, x)` wrapper around a function is not taken for f. Mutation oracle, sixteen repositories: tests recall 0.786 -> 0.851 (tuning 0.884 flat, +1 file; held-out 0.713 -> 0.826, +536 files on a CLI subject whose commands subclass the library's by dotted name), precision 0.431 -> 0.417, empty answers 92 -> 83; a failing test's path to the change found 0.541 -> 0.559. Two tuning files lost to a by-name match the new library site suppressed are restored by the next commit. Case library-star-reexport fails 4 of 5 on the base; control: a library base whose method hands nothing back never reaches the client's __getitem__. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../framework-behavior/library-callbacks.dl | 3 + graph/python/libsum/libsum.py | 47 ++++++++++-- .../site-packages/mixkit/__init__.py | 0 .../python3.12/site-packages/mixkit/_base.py | 23 ++++++ .../python3.12/site-packages/mixkit/abc.py | 1 + .../site-packages/mixkit/aliases.py | 14 ++++ .../library-star-reexport/.venv/pyvenv.cfg | 2 + .../library-star-reexport/app/__init__.py | 0 .../library-star-reexport/app/dotted.py | 10 +++ .../python/library-star-reexport/app/store.py | 26 +++++++ .../python/library-star-reexport/app/typed.py | 10 +++ .../python/library-star-reexport/case.json | 71 +++++++++++++++++++ .../library-star-reexport/tests/__init__.py | 0 .../tests/test_dotted.py | 5 ++ .../library-star-reexport/tests/test_flat.py | 5 ++ .../library-star-reexport/tests/test_get.py | 5 ++ .../library-star-reexport/tests/test_typed.py | 5 ++ .../tests/test_update.py | 5 ++ 18 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/__init__.py create mode 100644 tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/_base.py create mode 100644 tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/abc.py create mode 100644 tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/aliases.py create mode 100644 tests/cases/python/library-star-reexport/.venv/pyvenv.cfg create mode 100644 tests/cases/python/library-star-reexport/app/__init__.py create mode 100644 tests/cases/python/library-star-reexport/app/dotted.py create mode 100644 tests/cases/python/library-star-reexport/app/store.py create mode 100644 tests/cases/python/library-star-reexport/app/typed.py create mode 100644 tests/cases/python/library-star-reexport/case.json create mode 100644 tests/cases/python/library-star-reexport/tests/__init__.py create mode 100644 tests/cases/python/library-star-reexport/tests/test_dotted.py create mode 100644 tests/cases/python/library-star-reexport/tests/test_flat.py create mode 100644 tests/cases/python/library-star-reexport/tests/test_get.py create mode 100644 tests/cases/python/library-star-reexport/tests/test_typed.py create mode 100644 tests/cases/python/library-star-reexport/tests/test_update.py diff --git a/graph/python/engine/framework-behavior/library-callbacks.dl b/graph/python/engine/framework-behavior/library-callbacks.dl index e17d9043..eef361c8 100644 --- a/graph/python/engine/framework-behavior/library-callbacks.dl +++ b/graph/python/engine/framework-behavior/library-callbacks.dl @@ -76,6 +76,9 @@ lcb_expr_names(e, c) :- lcb_path_of(e, p), lcb_canon(p, c). .decl lcb_direct_ext_base(t:symbol, c:symbol) lcb_direct_ext_base(t, c) :- type_base_slot("client", t, _, "NAME", _, _, bh), base_expr("client", bh, e), lcb_expr_names(e, c). +// a dotted base, `class Store(abc.Mapping)` after `from collections import abc`: the attribute chain names it +lcb_direct_ext_base(t, c) :- + type_base_slot("client", t, _, "DOTTED_NAME", _, _, bh), base_expr("client", bh, e), lcb_expr_names(e, c). lcb_direct_ext_base(t, c) :- type_base_slot("client", t, _, "SUBSCRIPT", _, _, bh), base_expr("client", bh, e), expr_parent("client", e, "SUBSCRIPT_OBJECT", _, obj), lcb_expr_names(obj, c). diff --git a/graph/python/libsum/libsum.py b/graph/python/libsum/libsum.py index e910d123..304a8cf5 100644 --- a/graph/python/libsum/libsum.py +++ b/graph/python/libsum/libsum.py @@ -208,6 +208,8 @@ def __init__(self, name, path, is_pkg): self.ns = {} # local name -> qualified name (a def, a class, an imported thing) self.fns, self.classes = {}, {} self.imports = set() + self.stars, self.all_names = [], None # `from M import *` sources; a literal __all__, when declared + self.call_alias = {} # `Name = wrap(pkg.Class, ...)`: the dotted first argument, an alias when it is a class def load(self): try: @@ -229,6 +231,8 @@ def _ns_stmt(self, st): for a in st.names: if a.name != '*': self.ns[a.asname or a.name] = f'{base}.{a.name}' if base else a.name + elif base: + self.stars.append(base) elif isinstance(st, ast.Import): for a in st.names: self.imports.add(a.name) @@ -236,6 +240,14 @@ def _ns_stmt(self, st): self.ns[a.asname] = a.name else: self.ns.setdefault(a.name.split('.')[0], a.name.split('.')[0]) + elif isinstance(st, ast.Assign) and len(st.targets) == 1 and isinstance(st.targets[0], ast.Name) \ + and st.targets[0].id == '__all__' and isinstance(st.value, (ast.List, ast.Tuple)): + self.all_names = [e.value for e in st.value.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)] + elif isinstance(st, ast.Assign) and len(st.targets) == 1 and isinstance(st.targets[0], ast.Name) \ + and isinstance(st.value, ast.Call) and st.value.args and dotted(st.value.args[0]): + v = dotted(st.value.args[0]) + if v.split('.')[0] in self.ns: + self.call_alias.setdefault(st.targets[0].id, self.ns[v.split('.')[0]] + v[len(v.split('.')[0]):]) elif isinstance(st, ast.Assign) and len(st.targets) == 1 and isinstance(st.targets[0], ast.Name): v = dotted(st.value) if v and v.split('.')[0] in self.ns: @@ -608,6 +620,13 @@ def canon(self, q, seen=None): nxt = m.qual(parts[i]) if nxt and nxt != '.'.join(parts[:i + 1]): return self.canon('.'.join([nxt] + parts[i + 1:]), seen) + if not nxt and parts[i] in m.call_alias: + # a generic alias built around a class (`Mapping = _alias(collections.abc.Mapping, 2)`) is that class; + # a wrapper around anything else (`f = partial(g, x)`) is not followed + c = self.canon(m.call_alias[parts[i]], seen) + if c in self.classes: + return self.canon('.'.join([c] + parts[i + 1:]), seen) + return None if i + 1 < len(parts): # Class.method c = self.canon('.'.join(parts[:i + 1]), seen) if c in self.classes: @@ -963,7 +982,7 @@ def emit(callee_name, fq, kind): if p == 0: de.add((c, 'DIRECT', c, fld)) # `@C def f`: C(f) holds f for mname, m in self.mods.items(): - for local, q in m.ns.items(): + for local, q in list(m.ns.items()) + [(k, v) for k, v in m.call_alias.items() if k not in m.ns]: a = f'{mname}.{local}' if a == q: continue @@ -1038,7 +1057,7 @@ def main(): for n in os.listdir(d): if os.path.isfile(os.path.join(d, n, '__init__.py')): client_pkgs.add(n) - nfiles = 0 + nfiles, loaded = 0, {} while todo: r = todo.pop() if r in seen or r in client_pkgs: @@ -1052,13 +1071,31 @@ def main(): if not m.load(): continue nfiles += 1 - m.analyse() - del m.tree - prog.add(m) + loaded[m.name] = m for imp in m.imports: t = imp.split('.')[0] if t and t not in seen: todo.append(t) + # `from M import *` re-exports M's public names (its __all__ when it declares one), so a name the client imports + # through the star module (`collections.abc.MutableMapping`) is the declaration in M. Expanded before any body is + # analysed, to a fixpoint for a star of a star. + for _ in range(4): + grew = False + for m in loaded.values(): + for b in m.stars: + bm = loaded.get(b) + if bm is None: + continue + names = bm.all_names if bm.all_names is not None else [n for n in bm.ns if not n.startswith('_')] + for n in names: + if n in bm.ns and n not in m.ns: + m.ns[n] = bm.ns[n]; grew = True + if not grew: + break + for m in loaded.values(): + m.analyse() + del m.tree + prog.add(m) rounds = prog.solve() prog.returns() lib_pkgs = {p for p in seen if p not in client_pkgs} diff --git a/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/__init__.py b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/_base.py b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/_base.py new file mode 100644 index 00000000..7138da3b --- /dev/null +++ b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/_base.py @@ -0,0 +1,23 @@ +__all__ = ["Table", "Plain"] + + +class Table: + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def update(self, other): + for k in other: + self[k] = other[k] + + +class Plain: + def get(self, key): + return key + + +class _Hidden: + def get(self, key): + return self[key] diff --git a/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/abc.py b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/abc.py new file mode 100644 index 00000000..4a720caa --- /dev/null +++ b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/abc.py @@ -0,0 +1 @@ +from mixkit._base import * diff --git a/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/aliases.py b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/aliases.py new file mode 100644 index 00000000..1b2c7e89 --- /dev/null +++ b/tests/cases/python/library-star-reexport/.venv/lib/python3.12/site-packages/mixkit/aliases.py @@ -0,0 +1,14 @@ +import mixkit.abc + + +class _Alias: + def __init__(self, origin, nparams): + self.origin = origin + + +def _wrap(fn, arg): + return fn + + +Table = _Alias(mixkit.abc.Table, 2) +helper = _wrap(len, 1) diff --git a/tests/cases/python/library-star-reexport/.venv/pyvenv.cfg b/tests/cases/python/library-star-reexport/.venv/pyvenv.cfg new file mode 100644 index 00000000..5c46911c --- /dev/null +++ b/tests/cases/python/library-star-reexport/.venv/pyvenv.cfg @@ -0,0 +1,2 @@ +home = /nonexistent/bin +version_info = 3.12.0 diff --git a/tests/cases/python/library-star-reexport/app/__init__.py b/tests/cases/python/library-star-reexport/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-star-reexport/app/dotted.py b/tests/cases/python/library-star-reexport/app/dotted.py new file mode 100644 index 00000000..1b7ffe27 --- /dev/null +++ b/tests/cases/python/library-star-reexport/app/dotted.py @@ -0,0 +1,10 @@ +import mixkit._base as mb + + +def dotted_lookup(key): + return key + + +class DottedStore(mb.Table): + def __getitem__(self, key): + return dotted_lookup(key) diff --git a/tests/cases/python/library-star-reexport/app/store.py b/tests/cases/python/library-star-reexport/app/store.py new file mode 100644 index 00000000..6b15f0ca --- /dev/null +++ b/tests/cases/python/library-star-reexport/app/store.py @@ -0,0 +1,26 @@ +from mixkit.abc import Table, Plain + + +def lookup_key(key): + return key.upper() + + +def store_key(key): + return key.lower() + + +def plain_lookup(key): + return key + + +class Store(Table): + def __getitem__(self, key): + return lookup_key(key) + + def __setitem__(self, key, value): + store_key(key) + + +class Flat(Plain): + def __getitem__(self, key): + return plain_lookup(key) diff --git a/tests/cases/python/library-star-reexport/app/typed.py b/tests/cases/python/library-star-reexport/app/typed.py new file mode 100644 index 00000000..b4ec7b80 --- /dev/null +++ b/tests/cases/python/library-star-reexport/app/typed.py @@ -0,0 +1,10 @@ +import mixkit.aliases as mt + + +def typed_lookup(key): + return key + + +class TypedStore(mt.Table): + def __getitem__(self, key): + return typed_lookup(key) diff --git a/tests/cases/python/library-star-reexport/case.json b/tests/cases/python/library-star-reexport/case.json new file mode 100644 index 00000000..f8d287c9 --- /dev/null +++ b/tests/cases/python/library-star-reexport/case.json @@ -0,0 +1,71 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a name a library re-exports with `from M import *` is M's class: Table.get, inherited through the star module, calls back the client's __getitem__", + "run": [ + "impact", + "lookup_key", + "--tests" + ], + "want": [ + "tests/test_get.py", + "library callback" + ], + "avoid": [ + "tests/test_flat.py" + ] + }, + { + "why": "Table.update calls back the client's __setitem__", + "run": [ + "impact", + "store_key", + "--tests" + ], + "want": [ + "tests/test_update.py" + ] + }, + { + "why": "a dotted base (`class DottedStore(mb.Table)`) names the library class through its attribute chain", + "run": [ + "impact", + "dotted_lookup", + "--tests" + ], + "want": [ + "tests/test_dotted.py" + ], + "avoid": [ + "tests/test_get.py" + ] + }, + { + "why": "a generic alias a library builds around a class (`Table = _Alias(mixkit.abc.Table, 2)`) is that class", + "run": [ + "impact", + "typed_lookup", + "--tests" + ], + "want": [ + "tests/test_typed.py" + ] + }, + { + "why": "CONTROL: a library base whose get() hands nothing back never reaches the client's __getitem__", + "run": [ + "impact", + "plain_lookup", + "--tests" + ], + "want": [ + "tests: 0 of" + ], + "avoid": [ + "tests/test_flat.py" + ] + } + ] +} diff --git a/tests/cases/python/library-star-reexport/tests/__init__.py b/tests/cases/python/library-star-reexport/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-star-reexport/tests/test_dotted.py b/tests/cases/python/library-star-reexport/tests/test_dotted.py new file mode 100644 index 00000000..7dba1cf0 --- /dev/null +++ b/tests/cases/python/library-star-reexport/tests/test_dotted.py @@ -0,0 +1,5 @@ +from app.dotted import DottedStore + + +def test_dotted_get(): + assert DottedStore().get("a") == "a" diff --git a/tests/cases/python/library-star-reexport/tests/test_flat.py b/tests/cases/python/library-star-reexport/tests/test_flat.py new file mode 100644 index 00000000..ed2a4cd6 --- /dev/null +++ b/tests/cases/python/library-star-reexport/tests/test_flat.py @@ -0,0 +1,5 @@ +from app.store import Flat + + +def test_flat(): + assert Flat().get("a") == "a" diff --git a/tests/cases/python/library-star-reexport/tests/test_get.py b/tests/cases/python/library-star-reexport/tests/test_get.py new file mode 100644 index 00000000..6de6ddcd --- /dev/null +++ b/tests/cases/python/library-star-reexport/tests/test_get.py @@ -0,0 +1,5 @@ +from app.store import Store + + +def test_get(): + assert Store().get("a") == "A" diff --git a/tests/cases/python/library-star-reexport/tests/test_typed.py b/tests/cases/python/library-star-reexport/tests/test_typed.py new file mode 100644 index 00000000..cb3a0079 --- /dev/null +++ b/tests/cases/python/library-star-reexport/tests/test_typed.py @@ -0,0 +1,5 @@ +from app.typed import TypedStore + + +def test_typed_get(): + assert TypedStore().get("a") == "a" diff --git a/tests/cases/python/library-star-reexport/tests/test_update.py b/tests/cases/python/library-star-reexport/tests/test_update.py new file mode 100644 index 00000000..7662b3fd --- /dev/null +++ b/tests/cases/python/library-star-reexport/tests/test_update.py @@ -0,0 +1,5 @@ +from app.store import Store + + +def test_update(): + Store().update({"A": 1}) From 0e934b6877193c1be0dc8a71bdafe6982bb4320f Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:58:21 -0700 Subject: [PATCH 03/14] fix(impact, path): a library-callback site keeps its name match unless a hand-back from it is summarised A call the engine knows enters a summarised library callable (ext_lib_callback_site) was made a library receiver outright, which drops its by-name callers. Where the summary does say what the library hands back from that call (a lib_callback_edge from the same caller through the same callable), that is right: the hand-back is the answer and a project method merely named like the call is not. Where it says nothing -- the library reaches the project through a method's return, a registry, anything the summary cannot follow -- knowing the call enters a library is no evidence that nothing in the project runs behind it, and the name match it had before the library was read is kept. Found when naming more library bases (previous commit) turned a test client's `super().__init__(...)` into such a site and two failing test files lost the only route the graph had to them. Mutation oracle, sixteen repositories: both restored, +17 extra files, 0 lost; empty answers 83 -> 80. IMPACT_VERSION 69 and path EXPORT_VERSION 5, since the staged facts change. Case library-site-without-summary: the base fails the name-match check; control: a call whose hand-back is summarised still reaches no project method merely named like it. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_edges.py | 18 ++++++-- .../skills/axiomcode/scripts/axiomcode-impact | 2 +- .../skills/axiomcode/scripts/axiomcode-path | 5 ++- .../site-packages/netkit/__init__.py | 13 ++++++ .../.venv/pyvenv.cfg | 2 + .../app/__init__.py | 0 .../app/client.py | 21 +++++++++ .../library-site-without-summary/case.json | 44 +++++++++++++++++++ .../tests/__init__.py | 0 .../tests/test_session.py | 5 +++ 10 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tests/cases/python/library-site-without-summary/.venv/lib/python3.12/site-packages/netkit/__init__.py create mode 100644 tests/cases/python/library-site-without-summary/.venv/pyvenv.cfg create mode 100644 tests/cases/python/library-site-without-summary/app/__init__.py create mode 100644 tests/cases/python/library-site-without-summary/app/client.py create mode 100644 tests/cases/python/library-site-without-summary/case.json create mode 100644 tests/cases/python/library-site-without-summary/tests/__init__.py create mode 100644 tests/cases/python/library-site-without-summary/tests/test_session.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py index 8bdcf20c..b85276e6 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py @@ -513,9 +513,21 @@ def library_receiver_sites(q): library callable (Python ext_lib_callback_site: a member a client type inherits from its library base, whose source declares it). q(sql, params) -> rows.""" out = set() - for t in ('ext_library_receiver', 'ext_lib_callback_site'): - if list(q("SELECT 1 FROM sqlite_master WHERE name = ?", (t,))): - out |= {r[0] for r in q(f"SELECT DISTINCT c0 FROM {t}", ())} + has = lambda t: bool(list(q("SELECT 1 FROM sqlite_master WHERE name = ?", (t,)))) + if has('ext_library_receiver'): + out |= {r[0] for r in q("SELECT DISTINCT c0 FROM ext_library_receiver", ())} + if has('ext_lib_callback_site'): + # A site entering a summarised library callable is a library receiver only where the summary says what the + # library hands back from it (a lib_callback_edge from the same caller through the same callable). Where it + # says nothing -- the hand-back runs through a value the summary cannot follow, a method's return, a + # registry -- the site stays the untyped name match it was before the library was read: knowing a call + # ENTERS a library is no evidence that nothing in the project runs behind it, and dropping the match lost + # failing tests a library reaches that way. + edges = has('ext_lib_callback_edge') and has('call_sites') + sql = ("SELECT DISTINCT s.c0 FROM ext_lib_callback_site s JOIN call_sites cs ON cs.id = s.c0 " + "WHERE EXISTS (SELECT 1 FROM ext_lib_callback_edge e WHERE e.c0 = cs.caller_id AND e.c2 = s.c1)" + if edges else "SELECT DISTINCT c0 FROM ext_lib_callback_site") + out |= {r[0] for r in q(sql, ())} return out diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index dfc15d5b..9f3a4100 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -1343,7 +1343,7 @@ class Impact: W('cs_fixture_type', sorted(x for x in fixt if not x[0].startswith('collection:'))) # ── facts: the graph, exported once (reused while graph.sqlite is unchanged) ──────────────────────────────── - IMPACT_VERSION = '68' # 68: both 67s below, from two branches; 67: spawns_fact also links the module a `python -m pkg.mod` spawn runs; 67: libcb, a library summarised to call a client member back (Python library-callbacks.dl), walked as fw_edge "library callback"; 66: imports_fact names a module from its package root (a src layout) and has each test file import the conftest.py files above it; 65: guard_only also reads a conjunctive guard (`cond and __name__ == "__main__"`); 64: guard_only, a module body's call made only under `if __name__ == "__main__":`, which the import walk does not take; 63: imports_fact resolves an import written with its extension and a workspace package's specifier, and test_method takes no JavaScript / TypeScript helper by its name; 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key + IMPACT_VERSION = '69' # 69: a library-callback site is a library receiver only where a hand-back from it is summarised (ax_edges.library_receiver_sites); 68: both 67s below, from two branches; 67: spawns_fact also links the module a `python -m pkg.mod` spawn runs; 67: libcb, a library summarised to call a client member back (Python library-callbacks.dl), walked as fw_edge "library callback"; 66: imports_fact names a module from its package root (a src layout) and has each test file import the conftest.py files above it; 65: guard_only also reads a conjunctive guard (`cond and __name__ == "__main__"`); 64: guard_only, a module body's call made only under `if __name__ == "__main__":`, which the import walk does not take; 63: imports_fact resolves an import written with its extension and a workspace package's specifier, and test_method takes no JavaScript / TypeScript helper by its name; 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key # layer; 15: the registration facts (two 14s landed independently, which is exactly the collision this # guards); 16: regsite folded into ax_registration's reg_key_fact; 20: implements_pair (#1011); 17/18: the tagged-template test registrar # (it.each`…`) and its table span diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 4fcb51ae..9561e57d 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -989,11 +989,12 @@ class G: # ── facts: exported once per graph, reused while graph.sqlite is unchanged ─────────────────────────────── # EXPORT_VERSION is in the stamp beside the graph's mtime: bump it whenever the edges this writes change, or an # upgraded plugin keeps answering from the edges the old code exported (the graph is unchanged, so `index` says it - # is up to date and nothing rebuilds). 4: both 3s below, which landed on two branches under one number; + # is up to date and nothing rebuilds). 5: a library-callback site stays a by-name caller unless a hand-back from it + # is summarised (ax_edges.library_receiver_sites); 4: both 3s below, which landed on two branches under one number; # 3 (0.1.5): on one line a JavaScript anonymous callable nests only where its name does (#1598); # 3 (0.1.6): a call written inside a mock's stub or verification is not an edge (ax_edges.stub_sites); # 2: defines needs a written, strictly wider span (#1402, #1399); 1 was the bare mtime. - EXPORT_VERSION = '4' + EXPORT_VERSION = '5' def export(self): stamp = os.path.join(self.facts, 'stamp'); want = f"{self.db_mtime}:{self.EXPORT_VERSION}" def fresh(): diff --git a/tests/cases/python/library-site-without-summary/.venv/lib/python3.12/site-packages/netkit/__init__.py b/tests/cases/python/library-site-without-summary/.venv/lib/python3.12/site-packages/netkit/__init__.py new file mode 100644 index 00000000..a5713f3f --- /dev/null +++ b/tests/cases/python/library-site-without-summary/.venv/lib/python3.12/site-packages/netkit/__init__.py @@ -0,0 +1,13 @@ +_REGISTRY = [] + + +class Session: + def __init__(self, base_url="", on_ready=None): + self.base_url = base_url + for factory in _REGISTRY: + factory() + if on_ready is not None: + on_ready() + + def fetch(self, key): + return self[key] diff --git a/tests/cases/python/library-site-without-summary/.venv/pyvenv.cfg b/tests/cases/python/library-site-without-summary/.venv/pyvenv.cfg new file mode 100644 index 00000000..5c46911c --- /dev/null +++ b/tests/cases/python/library-site-without-summary/.venv/pyvenv.cfg @@ -0,0 +1,2 @@ +home = /nonexistent/bin +version_info = 3.12.0 diff --git a/tests/cases/python/library-site-without-summary/app/__init__.py b/tests/cases/python/library-site-without-summary/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-site-without-summary/app/client.py b/tests/cases/python/library-site-without-summary/app/client.py new file mode 100644 index 00000000..83a99bcd --- /dev/null +++ b/tests/cases/python/library-site-without-summary/app/client.py @@ -0,0 +1,21 @@ +from netkit import Session + + +def record_lookup(key): + return key + + +class AppSession(Session): + def __init__(self): + super().__init__(base_url="http://app") + + def __getitem__(self, key): + return record_lookup(key) + + +class Pool: + def __init__(self): + self.size = 4 + + def fetch(self, key): + return key diff --git a/tests/cases/python/library-site-without-summary/case.json b/tests/cases/python/library-site-without-summary/case.json new file mode 100644 index 00000000..00a87897 --- /dev/null +++ b/tests/cases/python/library-site-without-summary/case.json @@ -0,0 +1,44 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a call into a library callable whose summary names no hand-back from what this call passes (super().__init__ runs whatever a library registry holds) keeps the name match it had before the library was read", + "run": [ + "impact", + "Pool.__init__", + "--tests" + ], + "want": [ + "tests/test_session.py", + "by name" + ] + }, + { + "why": "CONTROL: a call the summary explains (fetch hands back __getitem__) is a library receiver: it reaches the client's __getitem__, and no project method merely named fetch", + "run": [ + "impact", + "Pool.fetch", + "--tests" + ], + "want": [ + "tests: 0 of" + ], + "avoid": [ + "tests/test_session.py" + ] + }, + { + "why": "the summarised hand-back itself", + "run": [ + "impact", + "record_lookup", + "--tests" + ], + "want": [ + "tests/test_session.py", + "library callback" + ] + } + ] +} diff --git a/tests/cases/python/library-site-without-summary/tests/__init__.py b/tests/cases/python/library-site-without-summary/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-site-without-summary/tests/test_session.py b/tests/cases/python/library-site-without-summary/tests/test_session.py new file mode 100644 index 00000000..a3975a8e --- /dev/null +++ b/tests/cases/python/library-site-without-summary/tests/test_session.py @@ -0,0 +1,5 @@ +from app.client import AppSession + + +def test_session(): + assert AppSession().fetch("k") == "k" From ec6e30d269aa2ee804a0b012836a3b25784a4707 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:58:21 -0700 Subject: [PATCH 04/14] fix(context): a report whose headline names no code chooses its scope from the body A scope-less context call ranks directories by where the HEADLINE's words land, and refuses with a menu when they land nowhere. A report titled in prose ("Release 8.4.2", "Disable when developing?", "Make unimported utilities private") lands nowhere by its headline while its body names the code outright (`@retry(wait=wait_fixed(1))`, `LazyFile`, `make_short_help`), and the refusal answered NOTHING. The whole report's words now rank the directories when the headline's have no evidence; the pick stays an offered scope (it neither filters nor weights), and a report whose words land nowhere at all still refuses. Issue-to-fix oracle (92 tasks, sixteen repositories): empty answers 6 -> 0 (three repositories), recall@5 0.846 -> 0.891, recall@10 0.881 -> 0.933, MRR 0.703 -> 0.744; tuning 0.799 -> 0.837 recall@5, held-out 0.894 -> 0.946. No task lost a gold file. Case context-body-names-the-code: the base refuses ("--in is required"); control: a headline that lands still chooses the place by itself. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/axiomcode-context | 6 ++++++ .../context-body-names-the-code/billing/__init__.py | 0 .../context-body-names-the-code/billing/invoice.py | 6 ++++++ .../cases/python/context-body-names-the-code/case.json | 10 ++++++++++ .../context-body-names-the-code/shipping/__init__.py | 0 .../context-body-names-the-code/shipping/route.py | 6 ++++++ .../context-body-names-the-code/tests/__init__.py | 0 .../context-body-names-the-code/tests/test_invoice.py | 5 +++++ 8 files changed, 33 insertions(+) create mode 100644 tests/cases/python/context-body-names-the-code/billing/__init__.py create mode 100644 tests/cases/python/context-body-names-the-code/billing/invoice.py create mode 100644 tests/cases/python/context-body-names-the-code/case.json create mode 100644 tests/cases/python/context-body-names-the-code/shipping/__init__.py create mode 100644 tests/cases/python/context-body-names-the-code/shipping/route.py create mode 100644 tests/cases/python/context-body-names-the-code/tests/__init__.py create mode 100644 tests/cases/python/context-body-names-the-code/tests/test_invoice.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context index 6c4207b3..aabee9fc 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context @@ -1282,6 +1282,12 @@ def main(argv): # it was swept over 57 real commits and was worth at most 0.012 recall@10 against not doing it, which # is noise at that sample size. The menu is worth printing for the reader; it is not worth ranking on. rank = rank_dirs(g, scope_terms(g, task)) if not scopes else None; flat = False + if rank and best_scope(rank) is None and terms: + # The headline chooses the place because its words are the subject -- when they land somewhere. A report + # titled in prose ("Release 8.4.2", "Disable when developing?", "Make unimported utilities private") + # lands nowhere, and the refusal below then answered NOTHING while the body named the code outright + # (`@retry(wait=wait_fixed(1))`, `LazyFile`, `make_short_help`). The whole report's words choose it then. + rank = rank_dirs(g, terms) if not scopes: # One candidate is not a choice: take it and SAY SO, rather than refusing and being handed back # the only path there was. This is NOT the offered-scope case above, so it is not marked diff --git a/tests/cases/python/context-body-names-the-code/billing/__init__.py b/tests/cases/python/context-body-names-the-code/billing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/context-body-names-the-code/billing/invoice.py b/tests/cases/python/context-body-names-the-code/billing/invoice.py new file mode 100644 index 00000000..ac04f1e0 --- /dev/null +++ b/tests/cases/python/context-body-names-the-code/billing/invoice.py @@ -0,0 +1,6 @@ +def round_cents(amount): + return round(amount, 2) + + +def compute_invoice_total(lines): + return round_cents(sum(lines)) diff --git a/tests/cases/python/context-body-names-the-code/case.json b/tests/cases/python/context-body-names-the-code/case.json new file mode 100644 index 00000000..19a08717 --- /dev/null +++ b/tests/cases/python/context-body-names-the-code/case.json @@ -0,0 +1,10 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a report titled in prose lands nowhere by its headline; its body names the code, and the answer comes from there instead of a refusal", + "run": ["context", "Disable when developing?\n\nIs there a way to make compute_invoice_total skip round_cents while testing?"], + "want": ["billing/invoice.py"], + "avoid": ["--in is required"]}, + {"why": "CONTROL: a headline that lands still chooses the place by itself", + "run": ["context", "plan route estimate distance"], + "want": ["where this task's words land: shipping", "shipping/route.py"], + "avoid": ["--in is required"]}]} diff --git a/tests/cases/python/context-body-names-the-code/shipping/__init__.py b/tests/cases/python/context-body-names-the-code/shipping/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/context-body-names-the-code/shipping/route.py b/tests/cases/python/context-body-names-the-code/shipping/route.py new file mode 100644 index 00000000..46921168 --- /dev/null +++ b/tests/cases/python/context-body-names-the-code/shipping/route.py @@ -0,0 +1,6 @@ +def plan_route(stops): + return sorted(stops) + + +def estimate_distance(stops): + return len(plan_route(stops)) diff --git a/tests/cases/python/context-body-names-the-code/tests/__init__.py b/tests/cases/python/context-body-names-the-code/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/context-body-names-the-code/tests/test_invoice.py b/tests/cases/python/context-body-names-the-code/tests/test_invoice.py new file mode 100644 index 00000000..70b6064c --- /dev/null +++ b/tests/cases/python/context-body-names-the-code/tests/test_invoice.py @@ -0,0 +1,5 @@ +from billing.invoice import compute_invoice_total + + +def test_total(): + assert compute_invoice_total([1, 2]) == 3 From 6ff9a1315767335a813df3430185f44b9cfd53a6 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:54:54 -0700 Subject: [PATCH 05/14] docs(skill): explain --library: one entry per dependency, comma-separated, and what staging buys Closes #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 8b9bc79b..9900ee47 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -69,10 +69,21 @@ identifier as written in the code. ## index -`axiomcode index` builds the graph explicitly; `--lang`, `--src` and `--library` narrow it. Never re-run it on an +`axiomcode index` builds the graph explicitly; `--lang` and `--src` narrow it. Never re-run it on an existing graph: the graph rebuilds itself after edits, and an answer given before that finishes says so on a `graph refresh:` line. +`--library` stages the project's dependencies, one entry per dependency, comma-separated with no spaces: + + axiomcode index --library .venv/lib/python3.12/site-packages/requests,.venv/lib/python3.12/site-packages/click + +Each entry is a dependency's source directory (its package directory under `site-packages` or `node_modules`) or a +pre-built library IR, relative to the repository or absolute (`~` is not expanded). `AXIOMCODE_LIBRARY` takes the same +list. Without it every call into a dependency is *unknown*; with it the call resolves to the dependency's declaration, +and a chain is typed through the dependency's declared return types (`client.post(...).json()`). Library bodies are +not walked, so a dependency calling back into the project is not found this way. The roots are kept with the graph: +a later bare `index` reuses them. + ## What it cannot see — say so instead of guessing Reflection, string dispatch, event buses; receivers the engine could not type; callbacks invoked by a library; what a From a159869c64d91dd34a61da042ffeeb61aae4030a Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:57:18 -0700 Subject: [PATCH 06/14] docs(skill): a --library entry is a library IR (the parser's CSV tables), made once per dependency The entry the skill led with was a dependency's source directory, which index re-parses; the reusable form is the CSV IR `axiomcode parser --library` writes. Lead with that, keep the source directory as the accepted fallback. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 9900ee47..4342eeec 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -73,16 +73,19 @@ identifier as written in the code. existing graph: the graph rebuilds itself after edits, and an answer given before that finishes says so on a `graph refresh:` line. -`--library` stages the project's dependencies, one entry per dependency, comma-separated with no spaces: - - axiomcode index --library .venv/lib/python3.12/site-packages/requests,.venv/lib/python3.12/site-packages/click - -Each entry is a dependency's source directory (its package directory under `site-packages` or `node_modules`) or a -pre-built library IR, relative to the repository or absolute (`~` is not expanded). `AXIOMCODE_LIBRARY` takes the same -list. Without it every call into a dependency is *unknown*; with it the call resolves to the dependency's declaration, -and a chain is typed through the dependency's declared return types (`client.post(...).json()`). Library bodies are -not walked, so a dependency calling back into the project is not found this way. The roots are kept with the graph: -a later bare `index` reuses them. +`--library` stages the project's dependencies, one entry per dependency, comma-separated with no spaces. Each entry +is a library IR: a directory of the parser's CSV tables (`all-python-modules.csv`, `all-types.csv`, …), produced once +per dependency version and reused by every build: + + axiomcode parser --library # once per dependency, with the engine CLI + axiomcode index --library /deps/ir/requests,/deps/ir/click,/deps/ir/jdk + +A dependency's source directory (its package under `site-packages` or `node_modules`) is accepted too: `index` parses +it into the same CSVs under `.axiomcode/out/.intermediate/lib/` on its first build. Entries are relative to the +repository or absolute (`~` is not expanded); `AXIOMCODE_LIBRARY` takes the same list. Without it every call into a +dependency is *unknown*; with it the call resolves to the dependency's declaration, and a chain is typed through the +dependency's declared return types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling +back into the project is not found this way. The entries are kept with the graph: a later bare `index` reuses them. ## What it cannot see — say so instead of guessing From 145d8a8c04ecd7f6b5bfb3d82b34e6e6c7ae6546 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:01:20 -0700 Subject: [PATCH 07/14] docs(skill): --library wording matches what was checked: `~` after a comma, unresolved vs external Every claim in the --library section was exercised: two IR entries stage two roots; a space after a comma is refused as "not a directory"; a relative entry resolves against the repository; a source directory is parsed under .axiomcode/out/.intermediate/lib/; AXIOMCODE_LIBRARY works the same; a bare index keeps the library graph. A `~` after a comma is refused (a leading one is the shell's), and without libraries a dependency call is unresolved or an external boundary, not always unknown. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 4342eeec..35a03881 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -82,10 +82,11 @@ per dependency version and reused by every build: A dependency's source directory (its package under `site-packages` or `node_modules`) is accepted too: `index` parses it into the same CSVs under `.axiomcode/out/.intermediate/lib/` on its first build. Entries are relative to the -repository or absolute (`~` is not expanded); `AXIOMCODE_LIBRARY` takes the same list. Without it every call into a -dependency is *unknown*; with it the call resolves to the dependency's declaration, and a chain is typed through the -dependency's declared return types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling -back into the project is not found this way. The entries are kept with the graph: a later bare `index` reuses them. +repository or absolute; write `~` out, since one after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same +list. Without it a call into a dependency is unresolved or named only as an external boundary; with it the call resolves +to the dependency's declaration, and a chain is typed through the dependency's declared return types +(`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is not found +this way. The entries are kept with the graph: a later bare `index` reuses them. ## What it cannot see — say so instead of guessing From a9e41b6fd0da426bd0d5cc19d234509b4065d377 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:19:18 -0700 Subject: [PATCH 08/14] feat(index): `--library auto` stages the dependencies the project imports; a source entry is compiled once Libraries are opt-in: the graph is built without them unless --library is given. `--library auto` finds the packages the project's source imports, in its virtual environment or under node_modules, compiles each once into ~/.cache/axiomcode/libir with `bin/axiomcode parser --library`, and stages the compiled tables. A named source directory goes through the same cache, so a rebuild the refresher starts after an edit no longer parses it again (a large dependency was re-parsed on every edit). A library IR entry passes through unchanged. The skill says when to do this: for better coverage, compile the dependencies and enable them. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 31 ++-- .../skills/axiomcode/scripts/ax_libs.py | 154 ++++++++++++++++++ .../skills/axiomcode/scripts/axiomcode-build | 9 +- .../site-packages/greeter/__init__.py | 11 ++ .../site-packages/unused/__init__.py | 2 + .../.venv/pyvenv.cfg | 2 + .../app/__init__.py | 0 .../app/main.py | 5 + .../case.json | 9 + .../pyproject.toml | 3 + 10 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/greeter/__init__.py create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/unused/__init__.py create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/.venv/pyvenv.cfg create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/app/__init__.py create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/app/main.py create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/case.json create mode 100644 tests/cases/python/library-auto-discovers-imported-dependencies/pyproject.toml diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 35a03881..42f67b0f 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -73,20 +73,23 @@ identifier as written in the code. existing graph: the graph rebuilds itself after edits, and an answer given before that finishes says so on a `graph refresh:` line. -`--library` stages the project's dependencies, one entry per dependency, comma-separated with no spaces. Each entry -is a library IR: a directory of the parser's CSV tables (`all-python-modules.csv`, `all-types.csv`, …), produced once -per dependency version and reused by every build: - - axiomcode parser --library # once per dependency, with the engine CLI - axiomcode index --library /deps/ir/requests,/deps/ir/click,/deps/ir/jdk - -A dependency's source directory (its package under `site-packages` or `node_modules`) is accepted too: `index` parses -it into the same CSVs under `.axiomcode/out/.intermediate/lib/` on its first build. Entries are relative to the -repository or absolute; write `~` out, since one after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same -list. Without it a call into a dependency is unresolved or named only as an external boundary; with it the call resolves -to the dependency's declaration, and a chain is typed through the dependency's declared return types -(`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is not found -this way. The entries are kept with the graph: a later bare `index` reuses them. +By default the graph is built without the project's dependencies: a call into one is unresolved or named only as an +external boundary. When better coverage is needed, compile the dependencies and enable them with `--library`, given +once on `index` and kept by every rebuild after it: + + axiomcode index --library auto # the dependencies the project imports + axiomcode index --library .venv/lib/python3.12/site-packages/requests,/deps/ir/jdk # or name them + +`auto` finds the packages the project's source imports in its virtual environment (`.venv`, `venv`, `$VIRTUAL_ENV`) +or under `node_modules`. A named entry, comma-separated with no spaces, is a dependency's source directory or a +library IR (a directory of the parser's CSV tables, as `axiomcode parser --library` writes it). Every +source directory is compiled once into `~/.cache/axiomcode/libir/` and reused until its files change. Java and C# +dependencies are not discovered: name their IR or source directories. Entries are relative to the repository or +absolute; write `~` out, since one after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. + +With them, a call into a dependency resolves to its declaration and a chain is typed through its declared return +types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is +not found this way. ## What it cannot see — say so instead of guessing diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py new file mode 100644 index 00000000..d9c25f85 --- /dev/null +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""ax_libs.py — the --library list as compiled library roots, comma-separated on stdout. + +An entry is one of + auto the project's dependencies, found where it installed them: a Python package in its virtual environment + (.venv, venv, env, .env, $VIRTUAL_ENV) or a JavaScript / TypeScript package under node_modules, kept only + when the project's own source imports it + an IR root a directory already holding the parser's CSV tables (flat, or one level down), passed through + a source a dependency's source directory, compiled once with `bin/axiomcode parser --library` into + ~/.cache/axiomcode/libir/ and reused until a file in it changes + +Without the cache a source entry was parsed again on every rebuild the graph's refresher started after an edit. +Notes go to stderr; a failure to compile one entry drops that entry and says so, it never fails the build. +""" +import hashlib, json, os, re, subprocess, sys + +MARKERS = ('all-types.csv', 'all-typescript-modules.csv', 'all-python-modules.csv', 'all-javascript-modules.csv', + 'all-csharp-modules.csv') +SKIP = {'.git', 'node_modules', '.venv', 'venv', 'env', '.env', '__pycache__', '.axiomcode', 'dist', 'build', '.tox', + '.mypy_cache', '.pytest_cache', 'site-packages'} +PY_IMPORT = re.compile(r'^\s*(?:from\s+([A-Za-z_][\w]*)[\w.]*\s+import|import\s+([A-Za-z_][\w]*))', re.M) +JS_IMPORT = re.compile(r'''(?:from\s+|require\(\s*|import\(\s*|import\s+)['"]((?:@[\w.-]+/)?[\w.-]+)''') + + +def note(msg): + print(f'▶ library: {msg}', file=sys.stderr) + + +def is_ir(d): + for m in MARKERS: + if os.path.isfile(os.path.join(d, m)): + return True + try: + if any(os.path.isfile(os.path.join(d, s, m)) for s in os.listdir(d)): + return True + except OSError: + return False + return False + + +def source_files(repo, exts): + for root, dirs, files in os.walk(repo): + dirs[:] = [x for x in dirs if x not in SKIP and not x.startswith('.')] + for f in files: + if f.endswith(exts): + yield os.path.join(root, f) + + +def python_sites(repo): + envs = [os.path.join(repo, n) for n in ('.venv', 'venv', 'env', '.env')] + if os.environ.get('VIRTUAL_ENV'): + envs.append(os.environ['VIRTUAL_ENV']) + for e in envs: + if not os.path.isfile(os.path.join(e, 'pyvenv.cfg')): + continue + sites = [] + for root in (os.path.join(e, 'lib'), os.path.join(e, 'Lib')): + if os.path.isdir(os.path.join(root, 'site-packages')): + sites.append(os.path.join(root, 'site-packages')) + if os.path.isdir(root): + sites += [os.path.join(root, d, 'site-packages') for d in sorted(os.listdir(root)) + if d.startswith('python') and os.path.isdir(os.path.join(root, d, 'site-packages'))] + if sites: + return sites + return [] + + +def discover(repo): + found, own = [], {d for d in os.listdir(repo) if os.path.isdir(os.path.join(repo, d))} + src = os.path.join(repo, 'src') + if os.path.isdir(src): + own |= {d for d in os.listdir(src) if os.path.isdir(os.path.join(src, d))} + sites = python_sites(repo) + if sites: + names = set() + for f in source_files(repo, ('.py',)): + try: + for a, b in PY_IMPORT.findall(open(f, encoding='utf-8', errors='replace').read()): + names.add(a or b) + except OSError: + pass + for n in sorted(names - own): + for s in sites: + if os.path.isfile(os.path.join(s, n, '__init__.py')) or os.path.isdir(os.path.join(s, n)) and \ + any(x.endswith('.py') for x in os.listdir(os.path.join(s, n))): + found.append(os.path.join(s, n)); break + note(f'auto: {len(found)} Python package(s) the project imports, from {sites[0]}') + nm = os.path.join(repo, 'node_modules') + if os.path.isdir(nm): + names, before = set(), len(found) + for f in source_files(repo, ('.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts')): + try: + names.update(JS_IMPORT.findall(open(f, encoding='utf-8', errors='replace').read())) + except OSError: + pass + for n in sorted(names): + if n.startswith('.') or n.startswith('node:'): + continue + d = os.path.join(nm, n) + if os.path.isdir(d) and not os.path.islink(d): + found.append(d) + note(f'auto: {len(found) - before} JavaScript / TypeScript package(s) the project imports, from node_modules') + if not found: + note('auto: no dependency found (no virtual environment or node_modules in the repository); Java and C# ' + 'dependencies are not discovered — name their library IR or source directories instead') + return found + + +def stamp(d): + """A dependency's identity: its path, file count and newest modification time.""" + n, newest = 0, 0.0 + for root, dirs, files in os.walk(d): + dirs[:] = [x for x in dirs if x != '__pycache__'] + for f in files: + try: + newest = max(newest, os.stat(os.path.join(root, f)).st_mtime); n += 1 + except OSError: + pass + return hashlib.sha1(f'{os.path.realpath(d)}|{n}|{newest:.0f}'.encode()).hexdigest()[:12] + + +def compiled(src, engine, cache): + out = os.path.join(cache, f"{os.path.basename(os.path.normpath(src)) or 'lib'}-{stamp(src)}") + if os.path.isfile(os.path.join(out, '.complete')): + return out + log = out + '.log' + os.makedirs(cache, exist_ok=True) + rc = subprocess.run([os.path.join(engine, 'bin', 'axiomcode'), 'parser', src, out, '--library'], + stdout=open(log, 'w'), stderr=subprocess.STDOUT).returncode + if rc != 0 or not is_ir(out): + note(f'{src} did not compile (rc {rc}, see {log}); not staged') + return None + open(os.path.join(out, '.complete'), 'w').close() + note(f'compiled {src} -> {out}') + return out + + +def main(): + repo, engine, spec = sys.argv[1], sys.argv[2], sys.argv[3] + cache = os.path.join(os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'), 'axiomcode', 'libir') + out, seen = [], set() + for e in [x for x in spec.split(',') if x]: + entries = discover(repo) if e == 'auto' else [e if os.path.isabs(e) else os.path.join(repo, e)] + for d in entries: + if not os.path.isdir(d): + note(f'not a directory: {d}; not staged'); continue + r = d if is_ir(d) else compiled(d, engine, cache) + if r and os.path.realpath(r) not in seen: + seen.add(os.path.realpath(r)); out.append(r) + print(','.join(out)) + + +if __name__ == '__main__': + main() diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index abd5d098..fb6f15bb 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -261,7 +261,7 @@ build_head_graph(){ [ -d "$REPO/node_modules" ] && ln -s "$REPO/node_modules" "$T/node_modules" # library roots are given relative to the repository or absolute; from the scratch directory only absolute reach local LIBS="" r; if [ -n "${AXIOMCODE_LIBRARY:-}" ]; then - local IFS=','; for r in $AXIOMCODE_LIBRARY; do case "$r" in /*|[A-Za-z]:*) ;; *) r="$REPO/$r" ;; esac; LIBS="${LIBS:+$LIBS,}$r"; done; unset IFS + local IFS=','; for r in $AXIOMCODE_LIBRARY; do case "$r" in /*|[A-Za-z]:*|auto) ;; *) r="$REPO/$r" ;; esac; LIBS="${LIBS:+$LIBS,}$r"; done; unset IFS fi if env -u AXIOMCODE_BACKGROUND -u AXIOMCODE_KEEP_BASE AXIOMCODE_NO_REFRESH=1 AXIOMCODE_LIBRARY="$LIBS" bash "$H/axiomcode-build" "$T" > "$REPO/.axiomcode/base-build.log" 2>&1; then # graph.sqlite is a symlink, or (Windows without the symlink privilege) the copy Git Bash made instead of one @@ -376,7 +376,12 @@ done # left graph.sqlite dangling — every later query rebuilt from scratch, and the refresher, finding "no graph", stopped restore(){ rm -rf "$OUT/$LANG_ARG"; if [ -d "$PREV" ]; then mv "$PREV" "$OUT/$LANG_ARG"; [ -f "$OUT/$LANG_ARG/graph.sqlite" ] && point "$OUT/$LANG_ARG/graph.sqlite"; fi [ -f "$OUT/.live.sqlite" ] && [ "$(real "$OUT/graph.sqlite")" != "$(real "$OUT/.live.sqlite")" ] && rm -f "$OUT/.live.sqlite"; return 0; } -LIB=(); [ -n "${AXIOMCODE_LIBRARY:-}" ] && LIB=(--library "$AXIOMCODE_LIBRARY") +# --library entries reach the engine compiled: `auto` becomes the dependencies the project imports, and a source +# directory is parsed once into ~/.cache/axiomcode/libir and reused (ax_libs.py) instead of on every rebuild +LIB=(); if [ -n "${AXIOMCODE_LIBRARY:-}" ]; then + LIBS_C="$(python3 "$H/ax_libs.py" "$REPO" "$ENGINE" "$AXIOMCODE_LIBRARY")" || LIBS_C="$AXIOMCODE_LIBRARY" + [ -n "$LIBS_C" ] && LIB=(--library "$LIBS_C") +fi ENGINE_RC=0 REASON="$(printf %s "${AXIOMCODE_REFRESH_REASON:-axiomcode index}" | tr -d "'")" # WHEN AND WHY THIS GRAPH WAS BUILT, in the graph itself (index_meta): refreshed_at (UTC), refresh_reason (`axiomcode diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/greeter/__init__.py b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/greeter/__init__.py new file mode 100644 index 00000000..27bef95d --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/greeter/__init__.py @@ -0,0 +1,11 @@ +class Greeter: + def hello(self, name: str) -> "Greeting": + return Greeting(f"hello {name}") + + +class Greeting: + def __init__(self, text: str) -> None: + self.text = text + + def shout(self) -> str: + return self.text.upper() diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/unused/__init__.py b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/unused/__init__.py new file mode 100644 index 00000000..0e5ceeae --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/lib/python3.12/site-packages/unused/__init__.py @@ -0,0 +1,2 @@ +def never_imported(): + return 1 diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/pyvenv.cfg b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/pyvenv.cfg new file mode 100644 index 00000000..302343a8 --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/.venv/pyvenv.cfg @@ -0,0 +1,2 @@ +home = /usr/bin +version_info = 3.12.0 diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/app/__init__.py b/tests/cases/python/library-auto-discovers-imported-dependencies/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/app/main.py b/tests/cases/python/library-auto-discovers-imported-dependencies/app/main.py new file mode 100644 index 00000000..9ca5d6dc --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/app/main.py @@ -0,0 +1,5 @@ +from greeter import Greeter + + +def welcome(name): + return Greeter().hello(name).shout() diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/case.json b/tests/cases/python/library-auto-discovers-imported-dependencies/case.json new file mode 100644 index 00000000..50482e06 --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/case.json @@ -0,0 +1,9 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "`--library auto` stages the packages the project imports from its virtual environment, and only those", + "run": ["index", "{repo}", "--library", "auto"], + "want": ["auto: 1 Python package(s) the project imports"]}, + {"why": "a chain is typed through the staged package's declared return type: hello() returns Greeting, so shout() resolves into the library", + "run": ["path", "welcome", "Greeting.shout"], + "want": ["[library · call @ app/main.py:5] greeter.Greeting.shout"], + "avoid": ["as written, unresolved"]}]} diff --git a/tests/cases/python/library-auto-discovers-imported-dependencies/pyproject.toml b/tests/cases/python/library-auto-discovers-imported-dependencies/pyproject.toml new file mode 100644 index 00000000..469c98de --- /dev/null +++ b/tests/cases/python/library-auto-discovers-imported-dependencies/pyproject.toml @@ -0,0 +1,3 @@ +[project] +name = "app" +version = "0.0.0" From 491209cddbf8328533fb7e5117c15d46dbda7b4f Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:21:48 -0700 Subject: [PATCH 09/14] fix(path): a TypeScript library method is found by its member name; cases for `--library auto` in JS and TS TypeScript names a library method module#Member (`index#Greeting.shout`); the library endpoint lookup matched only a `.`-separated tail, so `path welcome Greeting.shout` said nothing of that name was declared although the call resolved into the library. Cases cover discovery from node_modules and a call typed through the package in both languages; each fails without the build change. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../axiomcode/skills/axiomcode/scripts/axiomcode-path | 7 ++++--- .../case.json | 9 +++++++++ .../node_modules/greeter/index.js | 8 ++++++++ .../node_modules/greeter/package.json | 1 + .../node_modules/unused/index.js | 1 + .../node_modules/unused/package.json | 1 + .../package.json | 1 + .../src/main.js | 7 +++++++ .../case.json | 9 +++++++++ .../node_modules/greeter/index.d.ts | 7 +++++++ .../node_modules/greeter/package.json | 1 + .../node_modules/unused/index.d.ts | 1 + .../node_modules/unused/package.json | 1 + .../package.json | 1 + .../src/main.ts | 5 +++++ .../tsconfig.json | 1 + 16 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/case.json create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.js create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/index.js create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/package.json create mode 100644 tests/cases/javascript/library-auto-discovers-imported-dependencies/src/main.js create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/case.json create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.d.ts create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/index.d.ts create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/package.json create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/src/main.ts create mode 100644 tests/cases/typescript/library-auto-discovers-imported-dependencies/tsconfig.json diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 9561e57d..dfdf5441 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -797,11 +797,12 @@ class G: if m: s = m.group(1) + '.' + m.group(1).split('.')[-1] star = s.endswith('.*'); base = s[:-2] if star else s rows = self.q("""SELECT DISTINCT m.id, m.qualified_name q FROM call_edges e JOIN methods m ON m.id = e.callee_method_id - WHERE e.tier = 'boundary_lib' AND m.provenance <> 'client' AND (m.qualified_name = ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ?)""", - base, f"%.{base}", f"{base}.%" if star else base, f"%.{base}.%" if star else base) + WHERE e.tier = 'boundary_lib' AND m.provenance <> 'client' AND (m.qualified_name = ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ? OR m.qualified_name LIKE ?)""", + base, f"%.{base}", f"{base}.%" if star else base, f"%.{base}.%" if star else base, + f"%#{base}", f"%#{base}.%" if star else base) if not rows: return None if not star: # exact tail only: Files.readAllBytes, not Files.readAllBytesX - rows = [r for r in rows if r['q'] == base or r['q'].endswith('.' + base)] + rows = [r for r in rows if r['q'] == base or r['q'].endswith('.' + base) or r['q'].endswith('#' + base)] # TypeScript: module#Member if not rows: return None for r in rows: self.sym.setdefault(r['id'], {'id': r['id'], 'display': r['q'], 'file': '', 'line': None, 'end_line': None, 'kind': 'library', 'owner': r['q'].rsplit('.', 1)[0], 'is_test': 0, 'name': r['q'].rsplit('.', 1)[-1], 'method_id': r['id']}) return (f"{s} (library" + (f": {len(rows)} methods" if len(rows) > 1 else '') + ")"), [r['id'] for r in rows] diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/case.json b/tests/cases/javascript/library-auto-discovers-imported-dependencies/case.json new file mode 100644 index 00000000..f29623fc --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/case.json @@ -0,0 +1,9 @@ +{"lang": "javascript", "src": ".", + "checks": [ + {"why": "`--library auto` stages the packages the project imports from node_modules, and only those", + "run": ["index", "{repo}", "--library", "auto"], + "want": ["auto: 1 JavaScript / TypeScript package(s) the project imports"]}, + {"why": "a call on what the package's hello() returns resolves to the package's Greeting.shout", + "run": ["path", "welcome", "Greeting.shout"], + "want": ["[library · call @ src/main.js:4]", "Greeting.shout library"], + "avoid": ["as written, unresolved"]}]} diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.js b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.js new file mode 100644 index 00000000..4fa6a57b --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.js @@ -0,0 +1,8 @@ +class Greeting { + constructor(text) { this.text = text; } + shout() { return this.text.toUpperCase(); } +} +class Greeter { + hello(name) { return new Greeting(`hello ${name}`); } +} +module.exports = { Greeter, Greeting }; diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json new file mode 100644 index 00000000..554d4b35 --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json @@ -0,0 +1 @@ +{"name": "greeter", "version": "1.0.0", "main": "index.js"} diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/index.js b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/index.js new file mode 100644 index 00000000..c0013b93 --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/index.js @@ -0,0 +1 @@ +module.exports = { neverImported() { return 1; } }; diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json new file mode 100644 index 00000000..18ff1ee7 --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json @@ -0,0 +1 @@ +{"name": "unused", "version": "1.0.0", "main": "index.js"} diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/package.json b/tests/cases/javascript/library-auto-discovers-imported-dependencies/package.json new file mode 100644 index 00000000..db9172cc --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/package.json @@ -0,0 +1 @@ +{"name": "app", "version": "0.0.0", "dependencies": {"greeter": "1.0.0"}} diff --git a/tests/cases/javascript/library-auto-discovers-imported-dependencies/src/main.js b/tests/cases/javascript/library-auto-discovers-imported-dependencies/src/main.js new file mode 100644 index 00000000..d79a0afb --- /dev/null +++ b/tests/cases/javascript/library-auto-discovers-imported-dependencies/src/main.js @@ -0,0 +1,7 @@ +const { Greeter } = require("greeter"); + +function welcome(name) { + return new Greeter().hello(name).shout(); +} + +module.exports = { welcome }; diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/case.json b/tests/cases/typescript/library-auto-discovers-imported-dependencies/case.json new file mode 100644 index 00000000..d1499907 --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/case.json @@ -0,0 +1,9 @@ +{"lang": "typescript", "src": ".", + "checks": [ + {"why": "`--library auto` stages the packages the project imports from node_modules, and only those", + "run": ["index", "{repo}", "--library", "auto"], + "want": ["auto: 1 JavaScript / TypeScript package(s) the project imports"]}, + {"why": "a chain is typed through the package's declaration: hello() returns Greeting, so shout() resolves into the library, and the library method is found by its member name (the module part is written module#Member)", + "run": ["path", "welcome", "Greeting.shout"], + "want": ["[library · call @ src/main.ts:4] index#Greeting.shout"], + "avoid": ["nothing named 'Greeting.shout'"]}]} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.d.ts b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.d.ts new file mode 100644 index 00000000..3486e42d --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/index.d.ts @@ -0,0 +1,7 @@ +export declare class Greeting { + constructor(text: string); + shout(): string; +} +export declare class Greeter { + hello(name: string): Greeting; +} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json new file mode 100644 index 00000000..1a1090df --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/greeter/package.json @@ -0,0 +1 @@ +{"name": "greeter", "version": "1.0.0", "types": "index.d.ts", "main": "index.js"} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/index.d.ts b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/index.d.ts new file mode 100644 index 00000000..dd9dae05 --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/index.d.ts @@ -0,0 +1 @@ +export declare function neverImported(): number; diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json new file mode 100644 index 00000000..a761b308 --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/node_modules/unused/package.json @@ -0,0 +1 @@ +{"name": "unused", "version": "1.0.0", "types": "index.d.ts"} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/package.json b/tests/cases/typescript/library-auto-discovers-imported-dependencies/package.json new file mode 100644 index 00000000..db9172cc --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/package.json @@ -0,0 +1 @@ +{"name": "app", "version": "0.0.0", "dependencies": {"greeter": "1.0.0"}} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/src/main.ts b/tests/cases/typescript/library-auto-discovers-imported-dependencies/src/main.ts new file mode 100644 index 00000000..06cffb2f --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/src/main.ts @@ -0,0 +1,5 @@ +import { Greeter } from "greeter"; + +export function welcome(name: string): string { + return new Greeter().hello(name).shout(); +} diff --git a/tests/cases/typescript/library-auto-discovers-imported-dependencies/tsconfig.json b/tests/cases/typescript/library-auto-discovers-imported-dependencies/tsconfig.json new file mode 100644 index 00000000..75fe4f44 --- /dev/null +++ b/tests/cases/typescript/library-auto-discovers-imported-dependencies/tsconfig.json @@ -0,0 +1 @@ +{"compilerOptions": {"strict": true, "module": "commonjs", "target": "es2020"}} From 0e043b3c33ad2e375f9bbc135bb26a92fe2e8f75 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:23:14 -0700 Subject: [PATCH 10/14] feat(index): `--library auto` finds a Java project's dependencies through their sources jars auto reads the dependencies pom.xml (a ${property} version resolved) and build.gradle declare, finds each one's -sources.jar in the Maven repository (.mvn/maven.config's maven.repo.local, else ~/.m2) or Gradle's module cache, unpacks it once and compiles it like any source library. A dependency with no sources jar is named, with the command that fetches it. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_libs.py | 80 +++++++++++++++++- .../.mvn/maven.config | 1 + .../case.json | 8 ++ .../greeter/1.0/greeter-1.0-sources.jar | Bin 0 -> 677 bytes .../pom.xml | 11 +++ .../src/main/java/app/Main.java | 9 ++ 6 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/.mvn/maven.config create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/case.json create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/greeter/1.0/greeter-1.0-sources.jar create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/pom.xml create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index d9c25f85..1db6d83d 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -12,7 +12,8 @@ Without the cache a source entry was parsed again on every rebuild the graph's refresher started after an edit. Notes go to stderr; a failure to compile one entry drops that entry and says so, it never fails the build. """ -import hashlib, json, os, re, subprocess, sys +import glob, hashlib, json, os, re, subprocess, sys, zipfile +import xml.etree.ElementTree as ET MARKERS = ('all-types.csv', 'all-typescript-modules.csv', 'all-python-modules.csv', 'all-javascript-modules.csv', 'all-csharp-modules.csv') @@ -22,6 +23,9 @@ JS_IMPORT = re.compile(r'''(?:from\s+|require\(\s*|import\(\s*|import\s+)['"]((?:@[\w.-]+/)?[\w.-]+)''') +CACHE = os.path.join(os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'), 'axiomcode', 'libir') + + def note(msg): print(f'▶ library: {msg}', file=sys.stderr) @@ -100,9 +104,77 @@ def discover(repo): if os.path.isdir(d) and not os.path.islink(d): found.append(d) note(f'auto: {len(found) - before} JavaScript / TypeScript package(s) the project imports, from node_modules') + found += java_sources(repo) if not found: - note('auto: no dependency found (no virtual environment or node_modules in the repository); Java and C# ' - 'dependencies are not discovered — name their library IR or source directories instead') + note('auto: no dependency found (no virtual environment, node_modules, Maven or Gradle sources jar, or NuGet ' + 'package for this repository)') + return found + + +def maven_repo(repo): + cfg = os.path.join(repo, '.mvn', 'maven.config') + if os.path.isfile(cfg): + m = re.search(r'-Dmaven\.repo\.local=(\S+)', open(cfg, encoding='utf-8').read()) + if m: + return m.group(1) if os.path.isabs(m.group(1)) else os.path.join(repo, m.group(1)) + return os.path.join(os.path.expanduser('~'), '.m2', 'repository') + + +GRADLE_DEP = re.compile(r"""["']([\w.-]+):([\w.-]+):([\w.+-]+)["']""") + + +def java_coordinates(repo): + """(group, artifact, version) of every dependency a pom.xml or build.gradle(.kts) in the repository declares.""" + out = [] + for pom in glob.glob(os.path.join(repo, '**', 'pom.xml'), recursive=True): + if any(x in SKIP for x in os.path.relpath(pom, repo).split(os.sep)): + continue + try: + root = ET.parse(pom).getroot() + except (ET.ParseError, OSError): + continue + ns = root.tag[:root.tag.index('}') + 1] if root.tag.startswith('{') else '' + props = {} + for pr in root.findall(f'{ns}properties'): + props.update({c.tag.replace(ns, ''): (c.text or '').strip() for c in pr}) + sub = lambda v: re.sub(r'\$\{([^}]+)\}', lambda m: props.get(m.group(1), m.group(0)), v or '') + for d in root.iter(f'{ns}dependency'): + g, a, v = (sub(d.findtext(f'{ns}{k}')) for k in ('groupId', 'artifactId', 'version')) + if g and a: + out.append((g, a, v)) + for gf in glob.glob(os.path.join(repo, '**', 'build.gradle*'), recursive=True): + if any(x in SKIP for x in os.path.relpath(gf, repo).split(os.sep)): + continue + out += GRADLE_DEP.findall(open(gf, encoding='utf-8', errors='replace').read()) + return out + + +def java_sources(repo): + """The dependencies' -sources.jar, unpacked once under the cache: Java libraries are compiled from source.""" + found, missing = [], [] + m2 = maven_repo(repo) + gradle = os.path.join(os.path.expanduser('~'), '.gradle', 'caches', 'modules-2', 'files-2.1') + for g, a, v in sorted(set(java_coordinates(repo))): + base = os.path.join(m2, *g.split('.'), a) + known = sorted(os.listdir(base)) if os.path.isdir(base) else [] + ver = v if v in known else (known[-1] if known and (not v or '$' in v) else v) + cands = glob.glob(os.path.join(base, ver or '-', f'{a}-{ver}-sources.jar')) + cands += glob.glob(os.path.join(gradle, g, a, v or '*', '*', f'{a}-*-sources.jar')) + if not cands: + missing.append(f'{g}:{a}:{v}'); continue + jar = sorted(cands)[-1] + dest = os.path.join(CACHE, 'src', os.path.basename(jar)[:-len('.jar')]) + if not os.path.isfile(os.path.join(dest, '.unpacked')): + os.makedirs(dest, exist_ok=True) + with zipfile.ZipFile(jar) as z: + z.extractall(dest, [n for n in z.namelist() if n.endswith('.java')]) + open(os.path.join(dest, '.unpacked'), 'w').close() + found.append(dest) + if found or missing: + note(f'auto: {len(found)} Java dependency source jar(s), from {m2}') + if missing: + note(f'auto: no sources jar for {len(missing)} Java dependency(ies) ({", ".join(missing[:5])}' + f'{" …" if len(missing) > 5 else ""}); `mvn dependency:sources` fetches them') return found @@ -137,7 +209,7 @@ def compiled(src, engine, cache): def main(): repo, engine, spec = sys.argv[1], sys.argv[2], sys.argv[3] - cache = os.path.join(os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'), 'axiomcode', 'libir') + cache = CACHE out, seen = [], set() for e in [x for x in spec.split(',') if x]: entries = discover(repo) if e == 'auto' else [e if os.path.isabs(e) else os.path.join(repo, e)] diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/.mvn/maven.config b/tests/cases/java/library-auto-discovers-imported-dependencies/.mvn/maven.config new file mode 100644 index 00000000..7bdde3e5 --- /dev/null +++ b/tests/cases/java/library-auto-discovers-imported-dependencies/.mvn/maven.config @@ -0,0 +1 @@ +-Dmaven.repo.local=m2repo diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/case.json b/tests/cases/java/library-auto-discovers-imported-dependencies/case.json new file mode 100644 index 00000000..e025acf6 --- /dev/null +++ b/tests/cases/java/library-auto-discovers-imported-dependencies/case.json @@ -0,0 +1,8 @@ +{"lang": "java", "src": ".", + "checks": [ + {"why": "`--library auto` reads the dependencies pom.xml declares (a ${property} version resolved), finds each one's sources jar in the Maven repository .mvn/maven.config names, and compiles it", + "run": ["index", "{repo}", "--library", "auto"], + "want": ["auto: 1 Java dependency source jar(s)", "no sources jar for 1 Java dependency(ies) (com.example:nosources:2.0)"]}, + {"why": "a chain is typed through the dependency's declared return type: hello() returns Greeting, so shout() resolves into the library", + "run": ["path", "welcome", "Greeting.shout"], + "want": ["[library · call @ src/main/java/app/Main.java:7] com.example.greeter.Greeting.shout"]}]} diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/greeter/1.0/greeter-1.0-sources.jar b/tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/greeter/1.0/greeter-1.0-sources.jar new file mode 100644 index 0000000000000000000000000000000000000000..f33d698753cdca850fecf2297ff9009691d252cb GIT binary patch literal 677 zcmbVKO-sW-5N+D(A?N;pv8NOv!E+G=42b)bTNT5H$tKe0Cj{X9F zLOpqOCfPz#u!6G;v)R1&*!PmaAB`siu8+6#>x-9@qbJ&P19h0E9xg>%Nc19ws8M-m z?R4+NLKGsr7ZC#CT!V8BZf@H}bt97yLMh4;3=JM_pfazT!z7C!Mk#aWQY(JQM2a&5 zWUxYAsSGlFXjKT~OdAGF;jo3z8`;{f?V#sqJ%0WEBq!VlyBv||zbx^EjVF~Xghsec zG9kfu(YVwU1>4y2ym8Bju3H@^rOStK3@w| + 4.0.0 + app + app + 0.0.0 + 1.0 + + com.examplegreeter${greeter.version} + com.examplenosources2.0 + + diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java b/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java new file mode 100644 index 00000000..b8d52662 --- /dev/null +++ b/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java @@ -0,0 +1,9 @@ +package app; + +import com.example.greeter.Greeter; + +public class Main { + public static String welcome(String name) { + return new Greeter().hello(name).shout(); + } +} From 0a7ec6a0f81eb59448cfbe99d52dd51891b848d2 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:31:04 -0700 Subject: [PATCH 11/14] feat(index): `--library auto` compiles a C# project's NuGet packages by decompiling them NuGet packages ship assemblies, not source. auto reads the PackageReferences each .csproj declares (a version Directory.Packages.props sets centrally too), finds the package in the folder NUGET_PACKAGES, nuget.config's globalPackagesFolder or ~/.nuget/packages names, decompiles its assembly once with ilspycmd and compiles the C# like any source library. A package never restored is named, and a missing ilspycmd is reported with the command that installs it; CI installs it for the C# leg. With this, auto covers all five languages, and the skill says so. Closes #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 5 ++ plugins/axiomcode/skills/axiomcode/SKILL.md | 13 +-- .../skills/axiomcode/scripts/ax_libs.py | 77 +++++++++++++++++- .../case.json | 8 ++ .../nuget.config | 6 ++ .../1.0.0/lib/net8.0/Example.Greeter.dll | Bin 0 -> 4096 bytes .../src/App/App.csproj | 9 ++ .../src/App/Welcome.cs | 11 +++ 8 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 tests/cases/csharp/library-auto-discovers-imported-dependencies/case.json create mode 100644 tests/cases/csharp/library-auto-discovers-imported-dependencies/nuget.config create mode 100644 tests/cases/csharp/library-auto-discovers-imported-dependencies/packages/example.greeter/1.0.0/lib/net8.0/Example.Greeter.dll create mode 100644 tests/cases/csharp/library-auto-discovers-imported-dependencies/src/App/App.csproj create mode 100644 tests/cases/csharp/library-auto-discovers-imported-dependencies/src/App/Welcome.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbb5196f..43a784dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -371,6 +371,11 @@ jobs: if: matrix.lang == 'csharp' run: dotnet build -c Release graph/test/csharp/ground-truth/AxiomCsOracle + # `--library auto` compiles a NuGet package by decompiling it; the query case for it needs the decompiler + - name: install the decompiler `--library auto` uses for NuGet packages + if: matrix.lang == 'csharp' + run: dotnet tool install -g ilspycmd --version 8.2.0.7535 + - name: cache the Soufflé package uses: actions/cache@v4 with: diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 42f67b0f..40e18b94 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -80,11 +80,14 @@ once on `index` and kept by every rebuild after it: axiomcode index --library auto # the dependencies the project imports axiomcode index --library .venv/lib/python3.12/site-packages/requests,/deps/ir/jdk # or name them -`auto` finds the packages the project's source imports in its virtual environment (`.venv`, `venv`, `$VIRTUAL_ENV`) -or under `node_modules`. A named entry, comma-separated with no spaces, is a dependency's source directory or a -library IR (a directory of the parser's CSV tables, as `axiomcode parser --library` writes it). Every -source directory is compiled once into `~/.cache/axiomcode/libir/` and reused until its files change. Java and C# -dependencies are not discovered: name their IR or source directories. Entries are relative to the repository or +`auto` finds what the project depends on, in every language: a Python package its source imports, in its virtual +environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a +Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's +cache; a NuGet package a `.csproj` references, decompiled from its assembly (needs `dotnet tool install -g ilspycmd`). +Whatever it cannot find (a package never restored, a jar without sources) it names, with the command that fetches it. +A named entry, comma-separated with no spaces, is a dependency's source directory or a library IR (a directory of the +parser's CSV tables, as `axiomcode parser --library` writes it). Every library is compiled once into +`~/.cache/axiomcode/libir/` and reused until its files change. Entries are relative to the repository or absolute; write `~` out, since one after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. With them, a call into a dependency resolves to its declaration and a chain is typed through its declared return diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index 1db6d83d..e57db46e 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -4,15 +4,16 @@ An entry is one of auto the project's dependencies, found where it installed them: a Python package in its virtual environment (.venv, venv, env, .env, $VIRTUAL_ENV) or a JavaScript / TypeScript package under node_modules, kept only - when the project's own source imports it + when the project's own source imports it; a Java dependency pom.xml / build.gradle declares, through its + -sources.jar; a NuGet package a .csproj references, decompiled with ilspycmd an IR root a directory already holding the parser's CSV tables (flat, or one level down), passed through - a source a dependency's source directory, compiled once with `bin/axiomcode parser --library` into + a source a dependency's source directory (a NuGet assembly is decompiled to C# first), compiled once with `bin/axiomcode parser --library` into ~/.cache/axiomcode/libir/ and reused until a file in it changes Without the cache a source entry was parsed again on every rebuild the graph's refresher started after an edit. Notes go to stderr; a failure to compile one entry drops that entry and says so, it never fails the build. """ -import glob, hashlib, json, os, re, subprocess, sys, zipfile +import glob, hashlib, json, os, re, shutil, subprocess, sys, zipfile import xml.etree.ElementTree as ET MARKERS = ('all-types.csv', 'all-typescript-modules.csv', 'all-python-modules.csv', 'all-javascript-modules.csv', @@ -105,6 +106,7 @@ def discover(repo): found.append(d) note(f'auto: {len(found) - before} JavaScript / TypeScript package(s) the project imports, from node_modules') found += java_sources(repo) + found += csharp_sources(repo) if not found: note('auto: no dependency found (no virtual environment, node_modules, Maven or Gradle sources jar, or NuGet ' 'package for this repository)') @@ -178,6 +180,75 @@ def java_sources(repo): return found +def nuget_folder(repo): + if os.environ.get('NUGET_PACKAGES'): + return os.environ['NUGET_PACKAGES'] + for cfg in glob.glob(os.path.join(repo, '[Nn]u[Gg]et.[Cc]onfig')): + m = re.search(r'key="globalPackagesFolder"\s+value="([^"]+)"', open(cfg, encoding='utf-8', errors='replace').read()) + if m: + return m.group(1) if os.path.isabs(m.group(1)) else os.path.join(repo, m.group(1)) + return os.path.join(os.path.expanduser('~'), '.nuget', 'packages') + + +def csharp_packages(repo): + """(id, version) of every PackageReference a .csproj declares; a version Directory.Packages.props sets centrally.""" + central, out = {}, [] + for props in glob.glob(os.path.join(repo, '**', 'Directory.Packages.props'), recursive=True): + for i, v in re.findall(r']*?)(?:/>|>(.*?))', text, re.S): + v = re.search(r'Version="([^"]+)"', attrs) or re.search(r'([^<]+)', body or '') + out.append((i, v.group(1) if v else central.get(i.lower(), ''))) + return out + + +def ilspy(): + for c in (os.environ.get('AXIOMCODE_ILSPY'), shutil.which('ilspycmd'), + os.path.join(os.path.expanduser('~'), '.dotnet', 'tools', 'ilspycmd')): + if c and os.path.isfile(c): + return c + return None + + +def csharp_sources(repo): + """NuGet packages ship assemblies, not source: each one is decompiled once into C# and compiled like a source.""" + pkgs = sorted(set(csharp_packages(repo))) + if not pkgs: + return [] + tool, folder, found, missing = ilspy(), nuget_folder(repo), [], [] + if not tool: + note(f'auto: {len(pkgs)} NuGet package(s) declared, but C# libraries are compiled by decompiling them and ' + 'ilspycmd is not installed: `dotnet tool install -g ilspycmd` (8.x runs on a .NET 8 SDK); not staged') + return [] + for pid, ver in pkgs: + base = os.path.join(folder, pid.lower()) + known = sorted(os.listdir(base)) if os.path.isdir(base) else [] + v = ver if ver in known else (known[-1] if known and not ver else None) + libdir = os.path.join(base, v, 'lib') if v else '' + tfms = sorted(os.listdir(libdir)) if libdir and os.path.isdir(libdir) else [] + dlls = glob.glob(os.path.join(libdir, tfms[-1], '*.dll')) if tfms else [] + if not dlls: + missing.append(f'{pid} {ver}'); continue + dest = os.path.join(CACHE, 'src', f'{pid.lower()}-{v}') + if not os.path.isfile(os.path.join(dest, '.decompiled')): + env = dict(os.environ, DOTNET_ROLL_FORWARD='Major') + for dll in dlls: + out = os.path.join(dest, os.path.splitext(os.path.basename(dll))[0]) + subprocess.run([tool, '-p', '-o', out, dll], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if not glob.glob(os.path.join(dest, '**', '*.cs'), recursive=True): + missing.append(f'{pid} {ver} (did not decompile)'); continue + open(os.path.join(dest, '.decompiled'), 'w').close() + found.append(dest) + note(f'auto: {len(found)} NuGet package(s), decompiled from {folder}') + if missing: + note(f'auto: {len(missing)} NuGet package(s) not in that folder ({", ".join(missing[:5])}); `dotnet restore` fetches them') + return found + + def stamp(d): """A dependency's identity: its path, file count and newest modification time.""" n, newest = 0, 0.0 diff --git a/tests/cases/csharp/library-auto-discovers-imported-dependencies/case.json b/tests/cases/csharp/library-auto-discovers-imported-dependencies/case.json new file mode 100644 index 00000000..d0d048ad --- /dev/null +++ b/tests/cases/csharp/library-auto-discovers-imported-dependencies/case.json @@ -0,0 +1,8 @@ +{"lang": "csharp", "src": ".", + "checks": [ + {"why": "`--library auto` reads the PackageReferences a .csproj declares, finds each package in the folder nuget.config names, and decompiles its assembly once; a package that was never restored is named", + "run": ["index", "{repo}", "--library", "auto"], + "want": ["auto: 1 NuGet package(s), decompiled from", "not in that folder (Example.NotRestored 2.0.0)"]}, + {"why": "a chain is typed through the package's declared return type: Hello() returns Greeting, so Shout() resolves into the library", + "run": ["path", "Welcome.Run", "Greeting.Shout"], + "want": ["[library · call @ src/App/Welcome.cs:9] Example.Greeter.Greeting.Shout"]}]} diff --git a/tests/cases/csharp/library-auto-discovers-imported-dependencies/nuget.config b/tests/cases/csharp/library-auto-discovers-imported-dependencies/nuget.config new file mode 100644 index 00000000..2c073445 --- /dev/null +++ b/tests/cases/csharp/library-auto-discovers-imported-dependencies/nuget.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/cases/csharp/library-auto-discovers-imported-dependencies/packages/example.greeter/1.0.0/lib/net8.0/Example.Greeter.dll b/tests/cases/csharp/library-auto-discovers-imported-dependencies/packages/example.greeter/1.0.0/lib/net8.0/Example.Greeter.dll new file mode 100644 index 0000000000000000000000000000000000000000..da9ef72e617bbae2d67cb36d4329f709574dbe06 GIT binary patch literal 4096 zcmd^BU2Ggz75--ZQ^#?(2_;nuBAt*xLW+0Se@PuES;y-nF0o_0o0Ra=cy{)BJ<06M zFf(h%RM6z1hzEq)H~P>Aq@aq5N>x>8w{HV)uofz7ygOWFcmClw^LTCBJNw=+&b+g) zw&Fx)5cd46oxl^M1@)A7v8*{a#{8>+Bl$KZj~^ui2q#^}M#r%t_>&GsjJ z+T3oGfB{PS6X~xGvNl=DEWl3P&Lw_}K>XKql}Ncz50;)0Rrz=7lob)FO}l5MQRZ}# zh|RPF3{kf5S)jJL?{2)AxSy#EkUyO>UgAw!De|T^1(EVf%A4}?7&v&TNdw4YWl+-@gX7Df_t3H0hNN1H8R>upa!dPi+ zWKx{ygDVwd+W%d^WxXKnY8*OVGm@^KGcU5t`_ENzBWs^JH(R0X7Wscj-wD^RvlilT z{=(^!w zg!llC6F-S3iTlwcK7yYR$M7og3wWD2jT^)-Yk8GuYdbbBf*!Q+He^?&{r`=Rz7E4=|E*j8mafvGBT~!a~D!)781NNNU+twj>CrmFxPL zxni{fR~6DBDA=(dX4)0UYI=SYJ9djFLEBc0v-L}y z4mJN=5U6n0TeU*R@?un1{C13s=_S5n1&TM63!l3i82#bgg>U}z8*@K6bNr=&_um{t zuW1+qJtmAkHg@e2HZRcEhg?3NXHS0MmE#wm-gR^ESgt=$t&a`m2aI$B@0F8Z%RTr+ zXa)1WH*>{Sfn=bz68dYAK|S6$q{j^GDa_B*ru|S&1;OXj1V6U=XrV*{`H$b(mK7%o zTx)%vyL3QCnA8|Da9@^02*b^ytVr=mAv40;m_Odt2KqF^OIhF7L=v6YF14n$hhF>u z#++Qyc!}jY-_Gk@hfzj6z#>o2JYim6!emmC`?i}FtCi}7rN4jb+KV?&JbV55=-3~A z`<~b+2BEWR#i|&$f}-tO?S>kjD2)_{&qX|8!_k@*c#a!buCr#j!xi6dx0Dx0#pR=u z~gdq~{_Rw&J>TmgAun*<59%Pe?lM_Gf5jbLzI*ysJds6krcj?s$VE@S z9llxyewOS*5wSf^JdY|a5@)c;)-1oY=E=_zPbOboc&qn=|0Z|0tf){@A5R$lxzlA< zODkIIJS>E?wH(Tu|HIPX z=!(Ipv)?%Ck^`0j%BN|sg+R}-&WJ2>JT0VCw5j-uSwixUYW>X93Z)U>E}1vj;MeAH z$voM(^V(X}6~_qQ!_u81^9<21$wV@ + + net8.0 + + + + + + diff --git a/tests/cases/csharp/library-auto-discovers-imported-dependencies/src/App/Welcome.cs b/tests/cases/csharp/library-auto-discovers-imported-dependencies/src/App/Welcome.cs new file mode 100644 index 00000000..0c8efd77 --- /dev/null +++ b/tests/cases/csharp/library-auto-discovers-imported-dependencies/src/App/Welcome.cs @@ -0,0 +1,11 @@ +using Example.Greeter; + +namespace App; + +public static class Welcome +{ + public static string Run(string name) + { + return new Greeter().Hello(name).Shout(); + } +} From ccc20b6094d9741d38d0ee3b311a2648241485bd Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:34:12 -0700 Subject: [PATCH 12/14] feat(index): `--library auto` decompiles a Java dependency that ships no sources jar A dependency with only a class jar was reported and left out. It is now decompiled with Vineflower (found in the Maven repository, or AXIOMCODE_JAVA_DECOMPILER) once into the cache and compiled like a source library; with no decompiler the dependency is named with the commands that fetch sources or the decompiler. CI fetches Vineflower for the Java leg. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 5 ++ plugins/axiomcode/skills/axiomcode/SKILL.md | 3 +- .../skills/axiomcode/scripts/ax_libs.py | 62 +++++++++++++----- .../case.json | 47 +++++++++++-- .../example/nosources/2.0/nosources-2.0.jar | Bin 0 -> 1420 bytes .../pom.xml | 1 + .../src/main/java/app/Main.java | 5 ++ 7 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/nosources/2.0/nosources-2.0.jar diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43a784dc..a9920765 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -371,6 +371,11 @@ jobs: if: matrix.lang == 'csharp' run: dotnet build -c Release graph/test/csharp/ground-truth/AxiomCsOracle + # `--library auto` decompiles a dependency that ships no sources jar; the Java query case for it needs Vineflower + - name: fetch the decompiler `--library auto` uses for Java class jars + if: matrix.lang == 'java' + run: mvn -q dependency:get -Dartifact=org.vineflower:vineflower:1.10.1 + # `--library auto` compiles a NuGet package by decompiling it; the query case for it needs the decompiler - name: install the decompiler `--library auto` uses for NuGet packages if: matrix.lang == 'csharp' diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 40e18b94..4d7690dd 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -83,7 +83,8 @@ once on `index` and kept by every rebuild after it: `auto` finds what the project depends on, in every language: a Python package its source imports, in its virtual environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's -cache; a NuGet package a `.csproj` references, decompiled from its assembly (needs `dotnet tool install -g ilspycmd`). +cache, or its class jar decompiled when it ships no sources (needs Vineflower: `mvn dependency:get +-Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly (needs `dotnet tool install -g ilspycmd`). Whatever it cannot find (a package never restored, a jar without sources) it names, with the command that fetches it. A named entry, comma-separated with no spaces, is a dependency's source directory or a library IR (a directory of the parser's CSV tables, as `axiomcode parser --library` writes it). Every library is compiled once into diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index e57db46e..426e8799 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -5,7 +5,7 @@ auto the project's dependencies, found where it installed them: a Python package in its virtual environment (.venv, venv, env, .env, $VIRTUAL_ENV) or a JavaScript / TypeScript package under node_modules, kept only when the project's own source imports it; a Java dependency pom.xml / build.gradle declares, through its - -sources.jar; a NuGet package a .csproj references, decompiled with ilspycmd + -sources.jar or its class jar decompiled; a NuGet package a .csproj references, decompiled with ilspycmd an IR root a directory already holding the parser's CSV tables (flat, or one level down), passed through a source a dependency's source directory (a NuGet assembly is decompiled to C# first), compiled once with `bin/axiomcode parser --library` into ~/.cache/axiomcode/libir/ and reused until a file in it changes @@ -151,10 +151,21 @@ def java_coordinates(repo): return out +def java_decompiler(m2): + if os.environ.get('AXIOMCODE_JAVA_DECOMPILER'): + return os.environ['AXIOMCODE_JAVA_DECOMPILER'] + for root in (m2, os.path.join(os.path.expanduser('~'), '.m2', 'repository')): + jars = sorted(glob.glob(os.path.join(root, 'org', 'vineflower', 'vineflower', '*', 'vineflower-*.jar'))) + if jars and shutil.which('java'): + return jars[-1] + return None + + def java_sources(repo): - """The dependencies' -sources.jar, unpacked once under the cache: Java libraries are compiled from source.""" - found, missing = [], [] + """The dependencies' -sources.jar, unpacked once under the cache, or their class jar decompiled with Vineflower.""" + found, decompiled, missing, undecompiled = [], [], [], [] m2 = maven_repo(repo) + decompiler = java_decompiler(m2) gradle = os.path.join(os.path.expanduser('~'), '.gradle', 'caches', 'modules-2', 'files-2.1') for g, a, v in sorted(set(java_coordinates(repo))): base = os.path.join(m2, *g.split('.'), a) @@ -162,22 +173,41 @@ def java_sources(repo): ver = v if v in known else (known[-1] if known and (not v or '$' in v) else v) cands = glob.glob(os.path.join(base, ver or '-', f'{a}-{ver}-sources.jar')) cands += glob.glob(os.path.join(gradle, g, a, v or '*', '*', f'{a}-*-sources.jar')) - if not cands: + if cands: + jar = sorted(cands)[-1] + dest = os.path.join(CACHE, 'src', os.path.basename(jar)[:-len('.jar')]) + if not os.path.isfile(os.path.join(dest, '.unpacked')): + os.makedirs(dest, exist_ok=True) + with zipfile.ZipFile(jar) as z: + z.extractall(dest, [n for n in z.namelist() if n.endswith('.java')]) + open(os.path.join(dest, '.unpacked'), 'w').close() + found.append(dest); continue + # no sources jar: the class jar is decompiled to Java instead + bins = glob.glob(os.path.join(base, ver or '-', f'{a}-{ver}.jar')) + bins += [j for j in glob.glob(os.path.join(gradle, g, a, v or '*', '*', f'{a}-*.jar')) if not j.endswith('-sources.jar')] + if not bins: missing.append(f'{g}:{a}:{v}'); continue - jar = sorted(cands)[-1] - dest = os.path.join(CACHE, 'src', os.path.basename(jar)[:-len('.jar')]) - if not os.path.isfile(os.path.join(dest, '.unpacked')): + if not decompiler: + undecompiled.append(f'{g}:{a}:{v}'); continue + jar = sorted(bins)[-1] + dest = os.path.join(CACHE, 'src', os.path.basename(jar)[:-len('.jar')] + '-decompiled') + if not os.path.isfile(os.path.join(dest, '.decompiled')): os.makedirs(dest, exist_ok=True) - with zipfile.ZipFile(jar) as z: - z.extractall(dest, [n for n in z.namelist() if n.endswith('.java')]) - open(os.path.join(dest, '.unpacked'), 'w').close() - found.append(dest) - if found or missing: - note(f'auto: {len(found)} Java dependency source jar(s), from {m2}') + subprocess.run(['java', '-jar', decompiler, jar, dest], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if not glob.glob(os.path.join(dest, '**', '*.java'), recursive=True): + missing.append(f'{g}:{a}:{v} (did not decompile)'); continue + open(os.path.join(dest, '.decompiled'), 'w').close() + decompiled.append(dest) + if found or decompiled: + note(f'auto: {len(found)} Java dependency source jar(s) and {len(decompiled)} decompiled class jar(s), from {m2}') + if undecompiled: + note(f'auto: {len(undecompiled)} Java dependency(ies) with no sources jar ({", ".join(undecompiled[:5])}) and no ' + 'decompiler: `mvn dependency:sources` fetches sources, or `mvn dependency:get ' + '-Dartifact=org.vineflower:vineflower:1.10.1` the decompiler') if missing: - note(f'auto: no sources jar for {len(missing)} Java dependency(ies) ({", ".join(missing[:5])}' - f'{" …" if len(missing) > 5 else ""}); `mvn dependency:sources` fetches them') - return found + note(f'auto: {len(missing)} Java dependency(ies) not in the repository ({", ".join(missing[:5])}' + f'{" …" if len(missing) > 5 else ""}); `mvn dependency:resolve` fetches them') + return found + decompiled def nuget_folder(repo): diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/case.json b/tests/cases/java/library-auto-discovers-imported-dependencies/case.json index e025acf6..d4dc319c 100644 --- a/tests/cases/java/library-auto-discovers-imported-dependencies/case.json +++ b/tests/cases/java/library-auto-discovers-imported-dependencies/case.json @@ -1,8 +1,41 @@ -{"lang": "java", "src": ".", +{ + "lang": "java", + "src": ".", "checks": [ - {"why": "`--library auto` reads the dependencies pom.xml declares (a ${property} version resolved), finds each one's sources jar in the Maven repository .mvn/maven.config names, and compiles it", - "run": ["index", "{repo}", "--library", "auto"], - "want": ["auto: 1 Java dependency source jar(s)", "no sources jar for 1 Java dependency(ies) (com.example:nosources:2.0)"]}, - {"why": "a chain is typed through the dependency's declared return type: hello() returns Greeting, so shout() resolves into the library", - "run": ["path", "welcome", "Greeting.shout"], - "want": ["[library · call @ src/main/java/app/Main.java:7] com.example.greeter.Greeting.shout"]}]} + { + "why": "`--library auto` reads the dependencies pom.xml declares (a ${property} version resolved) in the Maven repository .mvn/maven.config names: one through its sources jar, one with only a class jar decompiled, and one that is absent named", + "run": [ + "index", + "{repo}", + "--library", + "auto" + ], + "want": [ + "auto: 1 Java dependency source jar(s) and 1 decompiled class jar(s)", + "not in the repository (com.example:absent:3.0)" + ] + }, + { + "why": "a chain is typed through the dependency's declared return type: hello() returns Greeting, so shout() resolves into the library", + "run": [ + "path", + "welcome", + "Greeting.shout" + ], + "want": [ + "[library · call @ src/main/java/app/Main.java:8] com.example.greeter.Greeting.shout" + ] + }, + { + "why": "a dependency that ships no sources is decompiled from its class jar, and a chain is typed through it: next() returns Step, so size() resolves into the library", + "run": [ + "path", + "advance", + "Step.size" + ], + "want": [ + "[library · call @ src/main/java/app/Main.java:12] com.example.nosources.Step.size" + ] + } + ] +} \ No newline at end of file diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/nosources/2.0/nosources-2.0.jar b/tests/cases/java/library-auto-discovers-imported-dependencies/m2repo/com/example/nosources/2.0/nosources-2.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..eaafcfafd7404b87e8a093e8cf07b8672b5052fe GIT binary patch literal 1420 zcmWIWW@h1HVBlb2U|r)G%YXzp8CV#6T|*poJ^kGD|D9rB2mmS-Vc_84z)&gz)CO1T z>*(j{<{BKL=j-;__snS@Z(Y5MyxzK6=gyqp9At3C_`%a6JuhD!Pv48BtF{Cg7+o~J zU>tJcgT9x~InU67bNX5*^u2T#wXQvR`sgVW&{B4e_Jl86oq<+30WruC*xkzllugdh z)yJ)Z2dW~qA~ClhCl!xMF*KEV`NjFAMaikfU_ICaK^dnW=ls&VlGGx-_@#F1-j(4nvVO*sHoo5%~bjuS*4}%gZWS3qnsC> zCH(o%c;BDB{qtAdf0ly2#S*p^f`3>Toem|ei{GNPwKA;y=A6%M0&!x?GS+d#S=DuH zaJ4v>yR>5IYS)veZY<4xJ!g8>wGT>eEjQlYQv30;*Z7ShAxi=Ug4^KIVFXagrD^)EslR3drsw;-^LBz>k7Q3J(W2aFDgx{d%x*K+i#AY z-5vW+x^-}f{cHH6QCK$7Sw?8oQ~P|Od8W- z`qC$}Ue4W|l3I|T^>@SCqP>+tl`HRN-t&5Pu0DN@$Zh5DZ?Z8(D~`VxaxK7M1V^`CP)V&TjJIU%9sf72gpR$iJVPQeT com.examplegreeter${greeter.version} com.examplenosources2.0 + com.exampleabsent3.0 diff --git a/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java b/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java index b8d52662..395bc34a 100644 --- a/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java +++ b/tests/cases/java/library-auto-discovers-imported-dependencies/src/main/java/app/Main.java @@ -1,9 +1,14 @@ package app; import com.example.greeter.Greeter; +import com.example.nosources.Counter; public class Main { public static String welcome(String name) { return new Greeter().hello(name).shout(); } + + public static int advance() { + return new Counter().next().size(); + } } From c3138c81c6606fe9a76f79773c2bed53956bc99e Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:14:23 -0700 Subject: [PATCH 13/14] feat(refresh): a graph built with --library is stale when a dependency changes; old compiled libraries are pruned Only source files were compared, so a dependency added, removed or upgraded without a source edit (`pip install -U`, `npm install`, `dotnet restore`, a manifest or lockfile edit) left the graph on the libraries it was built with. The file table of a graph built with --library now also records the manifests and lockfiles and what is installed (a virtual environment's dist-info names, npm's install record, .NET's restore output); a difference marks the graph stale and the usual background rebuild runs, which re-discovers `auto` and compiles only what changed. A graph built without libraries is unaffected and never gains them by itself. A compiled library no build has used for 30 days is removed from the cache, with what was unpacked or decompiled for it; one in use is never removed. The skill states both, and how to turn libraries off again. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 18 +++++--- .../skills/axiomcode/scripts/ax_fresh.py | 42 ++++++++++++++++++- .../skills/axiomcode/scripts/ax_libs.py | 35 ++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index 4d7690dd..a04511c1 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -84,12 +84,18 @@ once on `index` and kept by every rebuild after it: environment (`.venv`, `venv`, `$VIRTUAL_ENV`); a JavaScript / TypeScript package it imports, under `node_modules`; a Java dependency `pom.xml` or `build.gradle` declares, through its `-sources.jar` in the Maven repository or Gradle's cache, or its class jar decompiled when it ships no sources (needs Vineflower: `mvn dependency:get --Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly (needs `dotnet tool install -g ilspycmd`). -Whatever it cannot find (a package never restored, a jar without sources) it names, with the command that fetches it. -A named entry, comma-separated with no spaces, is a dependency's source directory or a library IR (a directory of the -parser's CSV tables, as `axiomcode parser --library` writes it). Every library is compiled once into -`~/.cache/axiomcode/libir/` and reused until its files change. Entries are relative to the repository or -absolute; write `~` out, since one after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. +-Dartifact=org.vineflower:vineflower:1.10.1`); a NuGet package a `.csproj` references, decompiled from its assembly +(needs `dotnet tool install -g ilspycmd`). Whatever it cannot find (a package never restored, a jar without sources) +it names, with the command that fetches it. A named entry, comma-separated with no spaces, is a dependency's source +directory or a library IR (a directory of the parser's CSV tables, as `axiomcode parser --library` +writes it). Every library is compiled once into `~/.cache/axiomcode/libir/` and reused until its files change; one no +build has used for 30 days is removed. Entries are relative to the repository or absolute; write `~` out, since one +after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. + +The choice is kept with the graph. Every rebuild after it stages the libraries again (`auto` discovers them +again, so a new dependency is picked up), and with libraries on, a dependency change alone (a manifest or lockfile +edit, `pip install -U`, `npm install`, `dotnet restore`) also marks the graph stale and rebuilds it. A graph built +without `--library` never gains libraries by itself; `AXIOMCODE_REINDEX=1 axiomcode index` turns them off again. With them, a call into a dependency resolves to its declaration and a chain is typed through its declared return types (`client.post(...).json()`). Library bodies are not walked, so a dependency calling back into the project is diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index e895d32f..e404e907 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -42,7 +42,7 @@ takes, up to AXIOMCODE_FRESH_MAX (default 600); AXIOMCODE_BUILD_WAIT (seconds, default 900) is how long a query that finds no graph waits for a build that is running rather than starting its own; AXIOMCODE_NO_GITIGNORE=1 watches (and indexes) directories git ignores.""" -import re, errno, hashlib, json, os, subprocess, sys, time +import glob, re, errno, hashlib, json, os, subprocess, sys, time H = os.path.dirname(os.path.abspath(__file__)) @@ -230,6 +230,43 @@ def snapshot(repo, lang, src): except OSError: pass return files +# A GRAPH BUILT WITH --library DEPENDS ON THE DEPENDENCIES TOO. Only source files were compared, so a dependency added, +# removed or upgraded (a manifest edit, `pip install -U`, `npm install`, `dotnet restore`) left the graph answering from +# the libraries it was built with until the next source edit. The table of such a graph also records the manifests and +# lockfiles, and what is installed, cheaply: the dist-info names of a virtual environment, npm's own install record, and +# the restore output .NET writes. +DEP_FILES = {'requirements.txt', 'requirements-dev.txt', 'requirements-test.txt', 'pyproject.toml', 'setup.cfg', 'setup.py', + 'Pipfile', 'Pipfile.lock', 'poetry.lock', 'uv.lock', 'package.json', 'package-lock.json', 'yarn.lock', + 'pnpm-lock.yaml', 'pom.xml', 'build.gradle', 'build.gradle.kts', 'gradle.lockfile', 'Directory.Packages.props', + 'packages.lock.json', 'project.assets.json', '.package-lock.json'} +DEP_PRUNE = {'.git', '.axiomcode', '__pycache__', '.venv', 'venv', 'env', '.env', 'dist', 'build', 'target', 'bin', '.gradle'} + +def dep_snapshot(repo): + out = {} + for root, dirs, files in os.walk(repo): + rel_root = os.path.relpath(root, repo) + if os.path.basename(root) == 'node_modules': # npm's install record only, never the packages + dirs[:] = []; files = [f for f in files if f == '.package-lock.json'] + else: + dirs[:] = [d for d in dirs if d not in DEP_PRUNE and (d == 'node_modules' or d == 'obj' or not d.startswith('.'))] + if rel_root.split(os.sep)[-1:] == ['obj']: dirs[:] = [] + for f in files: + if f in DEP_FILES or f.endswith('.csproj') or f.startswith('requirements') and f.endswith('.txt'): + pth = os.path.join(root, f) + try: st = os.stat(pth); out[os.path.relpath(pth, repo)] = f'{st.st_size}:{st.st_mtime_ns}' + except OSError: pass + for env in ('.venv', 'venv', 'env', '.env'): + for sp in glob.glob(os.path.join(repo, env, 'lib*', '*', 'site-packages')) + glob.glob(os.path.join(repo, env, 'Lib', 'site-packages')): + try: out[os.path.relpath(sp, repo)] = hashlib.sha1('|'.join(sorted(d for d in os.listdir(sp) if d.endswith('.dist-info'))).encode()).hexdigest() + except OSError: pass + return out + +def dep_changes(t, repo): + """the dependency files and environments that differ from the table, for a graph built with --library""" + if not t.get('library') or 'deps' not in t: return [] + old, now = t['deps'], dep_snapshot(repo) + return sorted(f'dependency: {k}' for k in set(old) | set(now) if old.get(k) != now.get(k)) + def load_table(repo): try: return json.load(open(table_path(repo))) except (OSError, ValueError): return None @@ -263,6 +300,7 @@ def changes(repo, table=None): for p in watched(src, l): rel = os.path.relpath(p, repo) if p.endswith(SOURCE[l]) and rel not in old: added.append(rel); break + changed += dep_changes(t, repo) return sorted(changed), sorted(added), sorted(set(old) - seen) # ── WHAT BUILT THE GRAPH ─────────────────────────────────────────────────────────────────────────────────────────── @@ -1549,7 +1587,7 @@ def main(argv): # for a graph an earlier build made (AXIOMCODE_BUILT_BY_UNKNOWN), which must not be credited to this one by = {} if os.environ.get('AXIOMCODE_BUILT_BY_UNKNOWN') else dict(built_by=built_by(os.environ.get('AXIOMCODE_ENGINE') or current_engine(repo), lang)) json.dump(dict(lang=lang, lang_auto=bool(os.environ.get('AXIOMCODE_LANG_AUTO')), src=src_arg.strip('/'), src_arg=src_arg, library=lib, built=time.time(), - files=snapshot(repo, lang, os.path.join(repo, src_arg)), **by), sys.stdout); return 0 + files=snapshot(repo, lang, os.path.join(repo, src_arg)), **(dict(deps=dep_snapshot(repo)) if lib else {}), **by), sys.stdout); return 0 if cmd == 'count': # the SOURCE files of each language under repo, walked as the refresher walks (the parser's skip list and git's # ignore rules): java typescript python javascript csharp. axiomcode-build picks the main language from these, and diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index 426e8799..aea46785 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -293,6 +293,9 @@ def stamp(d): def compiled(src, engine, cache): + if os.path.realpath(src).startswith(os.path.realpath(cache) + os.sep): + try: os.utime(src) # unpacked or decompiled under the cache: in use while its compiled form is + except OSError: pass out = os.path.join(cache, f"{os.path.basename(os.path.normpath(src)) or 'lib'}-{stamp(src)}") if os.path.isfile(os.path.join(out, '.complete')): return out @@ -320,8 +323,40 @@ def main(): r = d if is_ir(d) else compiled(d, engine, cache) if r and os.path.realpath(r) not in seen: seen.add(os.path.realpath(r)); out.append(r) + prune(out) print(','.join(out)) +PRUNE_DAYS = 30 + + +def prune(used): + """Every compiled library this build staged is marked used; one no build has staged for PRUNE_DAYS is removed, + with its source unpacked or decompiled under src/. Only entries this cache wrote are touched.""" + import time + now, keep = time.time(), {os.path.realpath(u) for u in used} + for u in keep: + if u.startswith(os.path.realpath(CACHE) + os.sep): + try: os.utime(u) + except OSError: pass + try: + entries = [os.path.join(CACHE, e) for e in os.listdir(CACHE)] + \ + [os.path.join(CACHE, 'src', e) for e in (os.listdir(os.path.join(CACHE, 'src')) if os.path.isdir(os.path.join(CACHE, 'src')) else [])] + except OSError: + return + for e in entries: + if os.path.realpath(e) in keep or os.path.basename(e) == 'src' or not os.path.isdir(e): + continue + marked = any(os.path.isfile(os.path.join(e, m)) for m in ('.complete', '.unpacked', '.decompiled')) + try: + idle = now - os.stat(e).st_mtime + except OSError: + continue + if marked and idle > PRUNE_DAYS * 86400: + shutil.rmtree(e, ignore_errors=True) + try: os.remove(e + '.log') + except OSError: pass + + if __name__ == '__main__': main() From bab2b30a2b06ff56558250b508da272f670becf0 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:33:54 -0700 Subject: [PATCH 14/14] feat(refresh): a Java dependency fetched after the build makes a --library graph stale Fetching a missing Java dependency (`mvn dependency:resolve`, a Gradle build) writes only to ~/.m2 or Gradle's cache, which no file in the project records, so the graph stayed without that library until the next source or pom edit. The dependency fingerprint now includes which declared dependencies are present in the Maven repository or Gradle's cache; one appearing marks the graph stale, and the refresh stages it. Refs #1887. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/SKILL.md | 3 ++- .../axiomcode/skills/axiomcode/scripts/ax_fresh.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index a04511c1..b3a302d2 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -94,7 +94,8 @@ after a comma is not expanded. `AXIOMCODE_LIBRARY` takes the same list. The choice is kept with the graph. Every rebuild after it stages the libraries again (`auto` discovers them again, so a new dependency is picked up), and with libraries on, a dependency change alone (a manifest or lockfile -edit, `pip install -U`, `npm install`, `dotnet restore`) also marks the graph stale and rebuilds it. A graph built +edit, `pip install -U`, `npm install`, `dotnet restore`, a Maven or Gradle fetch of a dependency that was missing) +also marks the graph stale and rebuilds it. A graph built without `--library` never gains libraries by itself; `AXIOMCODE_REINDEX=1 axiomcode index` turns them off again. With them, a call into a dependency resolves to its declaration and a chain is typed through its declared return diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index e404e907..d763c2b4 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -255,6 +255,20 @@ def dep_snapshot(repo): pth = os.path.join(root, f) try: st = os.stat(pth); out[os.path.relpath(pth, repo)] = f'{st.st_size}:{st.st_mtime_ns}' except OSError: pass + # a Java dependency is fetched into a repository outside the project (~/.m2, Gradle's cache), which no file here + # records: which declared dependencies are present there is part of the fingerprint, so one fetched after the + # graph was built (`mvn dependency:resolve`) makes it stale + if any(k.endswith(('pom.xml', 'build.gradle', 'build.gradle.kts')) for k in out): + try: + import ax_libs + m2 = ax_libs.maven_repo(repo) + gradle = os.path.join(os.path.expanduser('~'), '.gradle', 'caches', 'modules-2', 'files-2.1') + have = sorted(f'{g}:{a}:{v}' for g, a, v in set(ax_libs.java_coordinates(repo)) + if os.path.isdir(os.path.join(m2, *g.split('.'), a, v) if v and '$' not in v else os.path.join(m2, *g.split('.'), a)) + or os.path.isdir(os.path.join(gradle, g, a))) + out['java dependencies present'] = hashlib.sha1('|'.join(have).encode()).hexdigest() + except Exception: + pass for env in ('.venv', 'venv', 'env', '.env'): for sp in glob.glob(os.path.join(repo, env, 'lib*', '*', 'site-packages')) + glob.glob(os.path.join(repo, env, 'Lib', 'site-packages')): try: out[os.path.relpath(sp, repo)] = hashlib.sha1('|'.join(sorted(d for d in os.listdir(sp) if d.endswith('.dist-info'))).encode()).hexdigest()