From 96297f9d97c71ff581f57a41049640e19c6ed3da Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:28:04 +0800 Subject: [PATCH 1/3] Retain structured value types across user function calls --- CHANGELOG.md | 5 + pineforge_codegen/analyzer/base.py | 179 ++++++++++++++++-- pineforge_codegen/analyzer/call_handlers.py | 33 +++- pineforge_codegen/analyzer/contracts.py | 6 + pineforge_codegen/analyzer/types.py | 120 +++++++++++- pineforge_codegen/codegen/base.py | 31 +++- pineforge_codegen/codegen/emit_top.py | 38 +++- pineforge_codegen/codegen/types.py | 9 + tests/test_callable_compound_type_flow.py | 195 ++++++++++++++++++++ 9 files changed, 594 insertions(+), 22 deletions(-) create mode 100644 tests/test_callable_compound_type_flow.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83af3833..cc444443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ supported as exact pairs; on the 0.x line they are independent. See the in the same lambda; on ENOMEM the environment stays unchanged and the conversion silently uses the previous zone. +- User functions retain array, object and string result types through branch + returns and history clones. Untyped arguments retain collection element + types through nested calls and tuple bindings, and reassigned matrix locals + keep their initializer's matrix type. (#391) + ## 1.12.0 — 2026-11-12 - Pine v6 reports located errors for a missing `color.new` transparency, diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index f3385cf2..b96347e4 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -309,6 +309,13 @@ def __init__(self, ast: Program, filename: str = "", # Probe: data/validation/udt-method-probe-20-udt-return-from-func. self._func_udt_return_types: dict[str, str] = {} self._func_return_type_specs: dict[str, "TypeSpec"] = {} + self._func_tuple_element_specs: dict[str, tuple] = {} + self._callable_bound_param_specs_by_node: dict[int, tuple] = {} + self._callable_bound_return_specs_by_node: dict[int, TypeSpec] = {} + self._callable_edge_specs: dict = {} + self._callable_return_specs_by_profile: dict = {} + self._func_callsite_param_specs: dict = {} + self._func_callsite_return_specs: dict = {} self._func_param_type_specs: dict[str, list] = {} # Direct terminal gets on an unshadowed built-in temporary are # registered while their lexical scope is live. A later-defined UDF @@ -769,6 +776,24 @@ def _analyze(self) -> AnalyzerContext: func_callsite_return_types=dict( self._func_callsite_return_types ), + func_callsite_param_specs={ + **{ + key: self._callable_bound_param_specs_by_node[node_id] + for node_id, key in self._func_call_cs_map.items() + if node_id in self._callable_bound_param_specs_by_node + }, + **self._func_callsite_param_specs, + }, + func_callsite_return_specs={ + **{ + key: self._callable_bound_return_specs_by_node[node_id] + for node_id, key in self._func_call_cs_map.items() + if node_id in self._callable_bound_return_specs_by_node + }, + **self._func_callsite_return_specs, + }, + func_value_call_edges=dict(self._callable_edge_specs), + func_value_return_specs=dict(self._callable_return_specs_by_profile), func_declared_param_type_specs={ name: tuple(specs) for name, specs in self._func_param_type_specs.items() @@ -3370,6 +3395,52 @@ def _edge_ta_variant_target_map( func_info_by_name, _bound_user_call_args, ) + self._resolve_callable_callsite_value_specs(call_edges, func_info_by_name) + + def _resolve_callable_callsite_value_specs(self, call_edges, infos): + """Bind exact value profiles to the variants the emitter actually uses.""" + profiles = self._func_callsite_param_specs + for owner, callee, call in call_edges: + target = self._func_call_cs_map.get(id(call)) + if owner is None and target is not None: + profile = self._callable_bound_param_specs_by_node.get(id(call)) + if profile is not None: + profiles[target] = profile + for _ in range(len(call_edges) + 1): + changed = False + for owner, callee, call in call_edges: + if owner is None or owner not in infos: + continue + info = infos[owner] + if info.node is None: + continue + count = self._func_call_site_count.get(owner, 0) + for owner_cs in range(count) if count else (None,): + profile = profiles.get( + (owner, owner_cs), tuple(info.param_type_specs), + ) + incoming = self._callable_edge_specs.get( + (owner, profile, id(call)) + ) + if incoming is None: + continue + target = self._func_call_cs_map.get(id(call)) + if target is None: + callee_count = self._func_call_site_count.get(callee, 0) + if owner_cs is None or owner_cs >= callee_count: + continue + target = (callee, owner_cs) + if target not in profiles: + profiles[target] = incoming + changed = True + if not changed: + break + if self._budget is not None: + self._budget.check(phase=Phase.ANALYZER) + for (name, cs), profile in profiles.items(): + exact = self._callable_return_specs_by_profile.get((name, profile)) + if exact is not None: + self._func_callsite_return_specs[(name, cs)] = exact # The primitive families a written call can pass an untyped parameter. _ARGUMENT_FAMILIES = frozenset({ @@ -3443,6 +3514,9 @@ def reads(value, names: set[str]) -> bool: owner_params = set(untyped_params(owner).values()) actuals = bound_args(callee, call) recorded = self._callable_bound_param_types_by_node.get(id(call), []) + recorded_specs = self._callable_bound_param_specs_by_node.get( + id(call), () + ) for index in params: if (index >= len(actuals) or index >= len(recorded) or actuals[index] is None @@ -3450,6 +3524,19 @@ def reads(value, names: set[str]) -> bool: continue family = recorded[index] shared = shared_family(callee, index) + info_specs = func_info_by_name[callee].param_type_specs + incoming_spec = ( + recorded_specs[index] if index < len(recorded_specs) else None + ) + shared_spec = info_specs[index] if index < len(info_specs) else None + if (incoming_spec is not None and shared_spec is not None + and incoming_spec != shared_spec + and (incoming_spec.kind != "primitive" + or shared_spec.kind != "primitive")): + # Structured types are argument families too. In particular + # vector& cannot receive a vector argument. + conflicted.add(callee) + break if (family in self._ARGUMENT_FAMILIES and shared in self._ARGUMENT_FAMILIES and family != shared @@ -5563,6 +5650,12 @@ def _visit_TupleAssign(self, node: TupleAssign) -> PineType: is_val_static = self._is_static_expression(node.value) declared_types: list[PineType] = [] + tuple_specs = ( + self._func_tuple_element_specs.get(node.value.callee.name, ()) + if isinstance(node.value, FuncCall) + and isinstance(node.value.callee, Identifier) + else () + ) for idx, name in enumerate(node.names): inferred_element_type = ( @@ -5612,6 +5705,13 @@ def _visit_TupleAssign(self, node: TupleAssign) -> PineType: setattr(sym, "_pf_decl_node_id", id(node)) setattr(sym, "_pf_decl_binding_name", name) setattr(sym, "_pf_tuple_binding", True) + if (idx < len(tuple_specs) and tuple_specs[idx] is not None + and (tuple_specs[idx].kind != "primitive" + or tuple_specs[idx].name == "string")): + sym.type_spec = tuple_specs[idx] + self._record_collection_type( + name, tuple_specs[idx], symbol_scope=sym.scope, + ) self._record_script_block_binding(sym, outer_symbol) self._note_member_writer(sym, name, node) @@ -5823,6 +5923,18 @@ def _visit_FuncDef(self, node: FuncDef) -> PineType: terminal_direct_return_spec = self._type_spec_from_expr( terminal_ret_expr ) + if isinstance(terminal_ret_expr, TupleLiteral): + self._func_tuple_element_specs[node.name] = tuple( + self._type_spec_from_expr(element) + for element in terminal_ret_expr.elements + ) + elif (isinstance(terminal_ret_expr, FuncCall) + and isinstance(terminal_ret_expr.callee, Identifier)): + self._func_tuple_element_specs[node.name] = ( + self._func_tuple_element_specs.get( + terminal_ret_expr.callee.name, () + ) + ) terminal_map_return = self._terminal_map_call_return( terminal_ret_expr, { @@ -8358,35 +8470,41 @@ def _validate_deferred_param_history_refs( ) validated.add(key) - def _propagate_deferred_map_callable_specs( + def _propagate_deferred_callable_specs( self, owner: str, parameter_specs: dict[str, TypeSpec | None], visiting: frozenset[str] = frozenset(), + _memo: set | None = None, ) -> bool: - """Monomorphize an all-untyped wrapper chain for one concrete map call. + """Carry exact value types through deferred callable argument edges. Function definitions are analyzed before their concrete top-level call sites. Consequently ``wrapper(a) => identity(a)`` initially creates both ``FuncInfo`` records with scalar fallbacks. The definition pass already captured identity-keyed parameter-flow edges for deferred map - history validation; reuse those exact edges to carry a later map - ``TypeSpec`` inward, then infer map returns on the way back out. - - This is deliberately bounded and map-triggered. Cycles stop at the - active owner, incompatible previously-established parameter specs stop - that edge, and scalar-only call graphs never mutate. PineForge emits - one C++ body per ordinary UDF, so silently replacing one concrete map - specialization with a different one would be a false polymorphic - inference rather than a valid widening. + history validation; reuse those exact edges to carry the concrete + ``TypeSpec`` inward, then infer returns on the way back out. + + Queries are memoized by function and input types, never by call path. + A diamond therefore visits each concrete signature once. Existing map + incompatibility diagnostics remain authoritative; other distinct value + families are retained at their written calls for clone specialization. """ if owner in visiting: return False if not any( - spec is not None and spec.kind == "map" + spec is not None + and (spec.kind != "primitive" or spec.name == "string") for spec in parameter_specs.values() ): return False + if _memo is None: + _memo = set() + profile = (owner, tuple(parameter_specs.items())) + if profile in _memo: + return False + _memo.add(profile) func_info = next( (info for info in self._func_infos if info.name == owner), @@ -8395,6 +8513,8 @@ def _propagate_deferred_map_callable_specs( func_def = self._func_defs.get(owner) if func_info is None or func_def is None: return False + if self._budget is not None: + self._budget.check(func_def.loc, Phase.ANALYZER) changed = False while len(func_info.param_type_specs) < len(func_def.params): @@ -8439,7 +8559,7 @@ def _propagate_deferred_map_callable_specs( for _ in range(max(1, len(edges) + 1)): pass_changed = False for ( - _call_id, + call_id, callee_owner, callee_param_names, bound_args, @@ -8493,10 +8613,19 @@ def _propagate_deferred_map_callable_specs( ) ): callee_specs[param_name] = contextual_map_spec - if self._propagate_deferred_map_callable_specs( + self._callable_bound_param_specs_by_node[call_id] = tuple( + callee_specs.get(name) for name in callee_param_names + ) + self._callable_edge_specs[( + owner, + tuple(parameter_specs.get(name) for name in func_def.params), + call_id, + )] = self._callable_bound_param_specs_by_node[call_id] + if self._propagate_deferred_callable_specs( callee_owner, callee_specs, next_visiting, + _memo, ): pass_changed = True changed = changed or pass_changed @@ -8506,6 +8635,16 @@ def _propagate_deferred_map_callable_specs( terminal = self._direct_terminal_return_expr(func_def) return_spec = None scalar_return_type = None + exact = self._callable_value_return_spec(func_def, parameter_specs) + if exact is not None and ( + exact.kind != "primitive" or exact.name == "string" + ): + return_spec = exact + self._callable_return_specs_by_profile[( + owner, tuple(parameter_specs.get(name) for name in func_def.params), + )] = exact + if exact.kind == "primitive": + scalar_return_type = self._primitive_pine_type_from_spec(exact) if isinstance(terminal, Identifier) and terminal.name in parameter_specs: candidate = parameter_specs.get(terminal.name) if candidate is not None and candidate.kind == "map": @@ -8532,7 +8671,9 @@ def _propagate_deferred_map_callable_specs( return_spec = candidate if return_spec is not None: established_return = self._func_return_type_specs.get(owner) - if established_return is None: + if (established_return is None + or (established_return.kind == "primitive" + and return_spec.kind != "primitive")): self._func_return_type_specs[owner] = return_spec func_info.return_type_spec = return_spec changed = True @@ -8540,6 +8681,14 @@ def _propagate_deferred_map_callable_specs( if func_info.return_type_spec is None: func_info.return_type_spec = return_spec changed = True + if return_spec.kind == "primitive": + result_type = self._primitive_pine_type_from_spec(return_spec) + if result_type != PineType.UNKNOWN: + func_info.return_type = result_type + self._func_return_types[owner] = result_type + elif return_spec.kind == "udt": + func_info.udt_return_type = return_spec.name + self._func_udt_return_types[owner] = return_spec.name # A different established return belongs to another concrete map # specialization. Keep it unchanged rather than falsely widening. diff --git a/pineforge_codegen/analyzer/call_handlers.py b/pineforge_codegen/analyzer/call_handlers.py index 8fef0994..7f25b6b2 100644 --- a/pineforge_codegen/analyzer/call_handlers.py +++ b/pineforge_codegen/analyzer/call_handlers.py @@ -1924,6 +1924,7 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType: for i in range(len(param_specs)): if param_specs[i] is None and i < len(arg_specs): param_specs[i] = arg_specs[i] + self._callable_bound_param_specs_by_node[id(node)] = tuple(param_specs) self._record_deferred_param_call_edge( node, func_name, @@ -2171,13 +2172,43 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType: # scalar fallbacks. Propagate the concrete specs through the exact # definition-time call edges and infer returns in post-order before # codegen snapshots the FuncInfo table. - self._propagate_deferred_map_callable_specs( + self._propagate_deferred_callable_specs( func_name, { name: spec for name, spec in zip(func_def.params, param_specs) }, ) + value_specs = [*param_specs, fi.return_type_spec] + exact_return = ( + self._callable_value_return_spec( + func_def, dict(zip(func_def.params, param_specs)), + ) + if any( + spec is not None + and (spec.kind != "primitive" or spec.name == "string") + for spec in value_specs + ) + else None + ) + if exact_return is not None and ( + exact_return.kind != "primitive" or exact_return.name == "string" + ): + self._callable_bound_return_specs_by_node[id(node)] = exact_return + self._callable_return_specs_by_profile[ + (func_name, tuple(param_specs)) + ] = exact_return + if (fi.return_type_spec is None + or fi.return_type_spec.kind == "primitive"): + fi.return_type_spec = exact_return + self._func_return_type_specs[func_name] = exact_return + if exact_return.kind == "primitive": + return_type = self._primitive_pine_type_from_spec(exact_return) + fi.return_type = return_type + self._func_return_types[func_name] = return_type + elif exact_return.kind == "udt": + fi.udt_return_type = exact_return.name + self._func_udt_return_types[func_name] = exact_return.name if return_type == PineType.NA: self._na_return_callables[id(node)] = func_def diff --git a/pineforge_codegen/analyzer/contracts.py b/pineforge_codegen/analyzer/contracts.py index 93478b1c..9b7a39a4 100644 --- a/pineforge_codegen/analyzer/contracts.py +++ b/pineforge_codegen/analyzer/contracts.py @@ -293,6 +293,12 @@ class AnalyzerContext: # helper called with both ``bar_index`` and ``close``). func_callsite_param_types: dict = field(default_factory=dict) func_callsite_return_types: dict = field(default_factory=dict) + # Exact argument/return value types retain collection element types, object + # identities and strings independently of the coarse primitive profiles. + func_callsite_param_specs: dict = field(default_factory=dict) + func_callsite_return_specs: dict = field(default_factory=dict) + func_value_call_edges: dict = field(default_factory=dict) + func_value_return_specs: dict = field(default_factory=dict) # Declaration-time parameter specs, before any call-site inference fills # untyped slots on the legacy FuncInfo record. Codegen uses this to apply # variant overrides only to genuinely untyped parameters. diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index 6c8fb0be..55302447 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -294,7 +294,7 @@ def _selection_type_spec( *, nullable: bool | None = None, ) -> TypeSpec | None: - """Keep exact collection joins; extend other families only around ``na``.""" + """A selection of identical value types retains that exact type.""" if nullable is None: nullable = any(self._selection_contains_na(node) for node, _ in branches) result: TypeSpec | None = None @@ -307,8 +307,6 @@ def _selection_type_spec( spec = TypeSpec.primitive("color") if spec is None: return None - if not nullable and spec.kind not in {"map", "matrix"}: - return None result = ( spec if result is None @@ -522,6 +520,11 @@ def direct_user_udt_ctor_name(node: ASTNode) -> str | None: )) return self._selection_type_spec(branches) if isinstance(value, FuncCall): + exact = getattr( + self, "_callable_bound_return_specs_by_node", {} + ).get(id(value)) + if exact is not None: + return exact cal = value.callee if (isinstance(cal, Identifier) and cal.name in ("line", "box") and len(value.args) == 1 and not value.kwargs @@ -869,6 +872,27 @@ def _callsite_terminal_return_spec( self._current_top_level_stmt = previous_owner self._symbols.current_scope = previous_scope + def _callable_value_return_spec(self, func_def, parameter_specs): + """Read a callable's value with its lexical locals and concrete inputs. + + This query never revisits the analyzer or allocates stateful call sites. + Each branch gets its own environment, so an expired local cannot borrow + the type of a same-named global or another branch's declaration. + """ + previous_owner = self._current_top_level_stmt + previous_scope = self._symbols.current_scope + self._symbols.current_scope = self._operator_definition_scopes.get( + id(func_def), self._symbols.global_scope, + ) + self._current_top_level_stmt = func_def + try: + return OperatorTypeQuery(self, dict(parameter_specs)).body_value_spec( + func_def.body + ) + finally: + self._current_top_level_stmt = previous_owner + self._symbols.current_scope = previous_scope + def _terminal_map_selection_return_spec( self, value: ASTNode | None, @@ -1442,6 +1466,79 @@ def check_body(self, body): for node in body: self._check_node(node) + def body_value_spec(self, body): + """Infer declaration and terminal value types without checking operators.""" + result = None + for node in body or (): + if self._analyzer._budget is not None: + self._analyzer._budget.check(node.loc, Phase.ANALYZER) + if isinstance(node, VarDecl): + result = ( + self._type_spec_from_hint(node.type_hint) + if node.type_hint else self._value_spec(node.value) + ) + self._parameter_types[node.name] = result + elif isinstance(node, Assignment): + result = ( + self._parameter_types.get(node.target.name) + if isinstance(node.target, Identifier) + else self._value_spec(node.target) + ) + if result is None: + result = self._value_spec(node.value) + elif isinstance(node, TupleAssign): + # Tuple elements are independent value bindings. + specs = () + if isinstance(node.value, FuncCall) and isinstance( + node.value.callee, Identifier + ): + specs = getattr( + self._analyzer, "_func_tuple_element_specs", {} + ).get(node.value.callee.name, ()) + for index, name in enumerate(node.names): + self._parameter_types[name] = ( + specs[index] if index < len(specs) else None + ) + result = None + elif isinstance(node, IfStmt): + branches = [ + OperatorTypeQuery( + self._analyzer, dict(self._parameter_types) + ).body_value_spec(branch) + for branch in (node.body, node.else_body) + ] + result = unify_type_specs( + *branches, + left_is_na=not node.body, + right_is_na=not node.else_body, + ) + elif isinstance(node, SwitchStmt): + result = None + first = True + for branch in [ + *(body for _, body in node.cases), node.default_body + ]: + spec = OperatorTypeQuery( + self._analyzer, dict(self._parameter_types) + ).body_value_spec(branch) + result = spec if first else unify_type_specs( + result, spec, right_is_na=not branch, + ) + first = False + elif isinstance(node, ExprStmt): + result = self._value_spec(node.expr) + else: + result = self._value_spec(node) + return result + + def _value_spec(self, value): + spec = self._type_spec_from_expr(value) + if spec is None: + spec = self._analyzer._operator_operand_type( + value, self._parameter_types + ) + return spec + def _check_node(self, node): if not isinstance(node, ASTNode): if isinstance(node, (tuple, list)): @@ -1538,6 +1635,23 @@ def _type_spec_from_expr(self, value, **kwargs): return self._analyzer._operator_operand_type( value, self._parameter_types, ) + if (isinstance(value, FuncCall) + and isinstance(value.callee, Identifier) + and value.callee.name in self._analyzer._func_defs): + name = value.callee.name + definition = self._analyzer._func_defs[name] + args = self._analyzer._bind_callable_args( + value, list(definition.params) + ) + profile = tuple( + self._value_spec(arg) if arg is not None else None + for arg in args + ) + exact = self._analyzer._callable_return_specs_by_profile.get( + (name, profile) + ) + if exact is not None: + return exact return super()._type_spec_from_expr(value, **kwargs) def _visit(self, node): diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index 6bd0a8ad..fdb388ba 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -2145,6 +2145,26 @@ def natural_name(fname: str, cs_idx: int) -> str: owns_non_ta_state = bool( g_var_originals or fixnan_originals(g_name) ) + owner_info = self._func_info_map.get(inst["fname"]) + owner_profile = inst.get("param_specs") + if owner_profile is None and owner_info is not None: + owner_profile = getattr( + ctx, "func_callsite_param_specs", {} + ).get( + (inst["fname"], inst.get("type_call_site_idx", + inst.get("call_site_idx"))), + tuple(owner_info.param_type_specs), + ) + incoming = getattr(ctx, "func_value_call_edges", {}).get( + (inst["fname"], owner_profile, id(callnode)) + ) + natural_profile = getattr( + ctx, "func_callsite_param_specs", {} + ).get((g_name, j)) + different_value_type = ( + incoming is not None and natural_profile is not None + and incoming != natural_profile + ) # Every non-cs0 enclosing variant is a distinct Pine written # call path. Even a pure wrapper can reach var/fixnan state # farther down the graph, so preserve the path identity here @@ -2155,6 +2175,7 @@ def natural_name(fname: str, cs_idx: int) -> str: ) if ( not path_requires_fresh_state + and not different_value_type and composed_ta == natural_ta and composed_var == natural_var and composed_fixnan == natural_fixnan @@ -2174,6 +2195,7 @@ def natural_name(fname: str, cs_idx: int) -> str: frozenset(composed_var.items()), frozenset(composed_fixnan.items()), path_identity, + incoming, ) ginst = interned.get(key) if ginst is None: @@ -2210,6 +2232,10 @@ def natural_name(fname: str, cs_idx: int) -> str: # takes the argument that call passes, as the # call's own cs{j} clone does. "type_call_site_idx": j, + "param_specs": incoming, + "return_spec": getattr( + ctx, "func_value_return_specs", {} + ).get((g_name, incoming)), "ta_remap": composed_ta, "var_remap": fvar_remap, "fixnan_remap": ffixnan_remap, @@ -2547,7 +2573,10 @@ def _register_global_aggregate_member_types(self) -> None: self._collection_types.setdefault(stmt.name, declared_spec) continue else: - continue + # Reassignment removes a binding from global_expr_map. Its + # initializer still establishes its matrix handle type, also + # for declarations inside top-level control-flow blocks. + declared_spec = self._type_spec_from_expr(stmt.value) if (from_selection and declared_spec is not None and declared_spec.kind == "array"): # ``a = switch mode ...`` whose every arm is an array of one diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index d9c67b33..0f4b2b24 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -2407,6 +2407,37 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], call_site_idx if call_site_idx is not None else (instance or {}).get("type_call_site_idx") ) + if type_call_site_idx is not None: + from dataclasses import replace + incoming = getattr( + self.ctx, "func_callsite_param_specs", {} + ).get((fi.name, type_call_site_idx), ()) + if (instance or {}).get("param_specs") is not None: + incoming = instance["param_specs"] + declared = getattr( + self.ctx, "func_declared_param_type_specs", {} + ).get(fi.name, ()) + specs = list(fi.param_type_specs) + while len(specs) < len(node.params): + specs.append(None) + for index, spec in enumerate(incoming): + if (index < len(specs) and spec is not None + and (spec.kind != "primitive" or spec.name == "string") + and (index >= len(declared) or declared[index] is None)): + specs[index] = spec + exact_return = getattr( + self.ctx, "func_callsite_return_specs", {} + ).get((fi.name, type_call_site_idx), fi.return_type_spec) + if (instance or {}).get("return_spec") is not None: + exact_return = instance["return_spec"] + fi = replace( + fi, param_type_specs=specs, return_type_spec=exact_return, + udt_return_type=( + exact_return.name + if exact_return is not None and exact_return.kind == "udt" + else fi.udt_return_type + ), + ) # Collection registries historically used raw variable names for the # whole translation unit. Emit each callable against copy-on-write @@ -2651,6 +2682,11 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], ret_type = "int64_t" elif ("return", fi.name) in self._array_widths.wide: ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) + elif (getattr(fi, "return_type_spec", None) is not None + and fi.return_type_spec.kind != "primitive"): + # Exact value types also apply to history/stateful clones. Their + # coarse primitive profile cannot represent a collection result. + ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) elif ( type_call_site_idx is not None and self._callsite_callable_return_pine_type( @@ -2664,8 +2700,6 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], "double", ) elif getattr(fi, "return_type_spec", None) is not None: - # Exact return TypeSpec (collections plus narrowly cached primitive - # terminal reads) takes precedence over the coarse PineType slot. ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) else: ret_type = PINE_TYPE_TO_CPP.get(fi.return_type, "double") diff --git a/pineforge_codegen/codegen/types.py b/pineforge_codegen/codegen/types.py index b6023534..945f693a 100644 --- a/pineforge_codegen/codegen/types.py +++ b/pineforge_codegen/codegen/types.py @@ -3068,6 +3068,15 @@ def _type_spec_from_expr_raw( return TypeSpec.udt(func_name) if namespace is None: func_info = self._func_info_map.get(func_name) + if func_info is not None: + exact = getattr( + self.ctx, "func_callsite_return_specs", {} + ).get(( + func_info.name, + self._callable_target_callsite_idx(func_info, node), + )) + if exact is not None: + return exact return_spec = (self._array_return_spec(func_info) if func_info is not None else None) if return_spec is not None: diff --git a/tests/test_callable_compound_type_flow.py b/tests/test_callable_compound_type_flow.py new file mode 100644 index 00000000..d759c3cc --- /dev/null +++ b/tests/test_callable_compound_type_flow.py @@ -0,0 +1,195 @@ +"""Structured values retain their types across callable boundaries.""" + +import re + +import pytest + +from pineforge_codegen import transpile +from tests._compile import compile_cpp + + +def signatures(cpp, name): + return re.findall( + rf"^ ([^\n]+?) {re.escape(name)}(?:_cs\d+|__ni\d+)?\(([^\n]*)\) \{{$", + cpp, + re.MULTILINE, + ) + + +CASES = [ + pytest.param( + """choose(array values, bool enabled) => + if enabled + values + else + values +items = array.from(close) +selected = choose(items, close > open) +plot(selected.get(0)) +""", + "choose", "std::vector", "std::vector& values, bool enabled", 1, + id="array-block-return", + ), + pytest.param( + """type Reading + float value +choose(bool enabled) => + Reading item = Reading.new(close) + if enabled + item + else + item +chosen = choose(close > open) +plot(chosen.value) +""", + "choose", "Reading", "bool enabled", 1, + id="object-block-return", + ), + pytest.param( + """choose(bool enabled, array left, array right) => + enabled ? left : right +left = array.from("first") +right = array.from("second") +chosen = choose(close > open, left, right) +log.info(chosen.get(0)) +""", + "choose", "std::vector", + "bool enabled, std::vector& left, std::vector& right", 1, + id="array-ternary-return", + ), + pytest.param( + """source = ta.ema(close, 7) +features(int offset) => + array items = array.from(source[offset]) + items +current = features(0) +previous = features(2) +plot(current.get(0) - previous.get(0)) +""", + "features", "std::vector", "int offset", 2, + id="array-history-clone-returns", + ), + pytest.param( + """suffix(value) => + pieces = str.split(value, "/") + pieces.size() > 1 ? pieces.get(1) : value +items = array.from("left/right", "whole") +for index = 0 to items.size() - 1 + value = items.get(index) + log.info(suffix(value)) +""", + "suffix", "std::string", "std::string value", 1, + id="variable-string-return", + ), + pytest.param( + """first(values) => values.get(0) +outer(values) => first(values) / values.size() +items = array.from(close) +plot(outer(items)) +""", + "first", "double", "std::vector& values", 1, + id="array-forwarding", + ), + pytest.param( + """pair() => + a = array.from(high) + b = array.from(low) + [a, b] +last(values) => + float result = na + for item in values + result := item + result +[upper, lower] = pair() +plot(last(upper) + last(lower)) +""", + "last", "double", "std::vector& values", 1, + id="tuple-array-argument", + ), + pytest.param( + """parts(value) => + [str.substring(value, 0, 1), str.substring(value, 1, 2)] +outer(value) => + [first, second] = parts(value) + first + second +value = input.string("ab") +log.info(outer(value)) +""", + "parts", "std::tuple", "std::string value", 1, + id="string-forwarding", + ), +] + + +@pytest.mark.parametrize("body,name,result,parameters,count", CASES) +def test_callable_boundaries_keep_the_value_type(body, name, result, parameters, count): + cpp = transpile('//@version=6\nstrategy("value type flow")\n' + body) + assert signatures(cpp, name) == [(result, parameters)] * count + compile_cpp(cpp, label=name) + + +def test_untyped_array_calls_keep_distinct_element_types(): + cpp = transpile("""//@version=6 +strategy("distinct array types") +trim(values, int maximum) => + if values.size() > maximum + values.shift() +integers = array.from(bar_index) +decimals = array.from(close) +trim(integers, 4) +trim(decimals, 4) +""") + found = signatures(cpp, "trim") + assert sorted(parameters for _, parameters in found) == [ + "std::vector& values, int maximum", + "std::vector& values, int maximum", + ] + targets = re.findall( + r"^ (trim(?:_cs\d+|__ni\d+)?)\((integers|decimals), 4\);$", + cpp, re.MULTILINE, + ) + assert len(targets) == 2 and len({name for name, _ in targets}) == 2 + compile_cpp(cpp, label="distinct array argument types") + + +def test_reassigned_matrix_local_keeps_its_initializer_type(): + cpp = transpile("""//@version=6 +strategy("matrix local type") +nextValue(matrix source) => + result = matrix.new(1, 1, 0.0) + result.set(0, 0, source.get(0, 0) / 2) + result +var state = matrix.new(1, 1, 0.0) +state.set(0, 0, close) +if barstate.islast + working = matrix.copy(state) + for index = 1 to 2 + working := nextValue(working) + log.info(str.tostring(working.get(0, 0))) +""") + declarations = re.findall( + r"^\s+(NumericMatrix|double) working(?: = [^\n;]*)?;$", + cpp, re.MULTILINE, + ) + assert declarations == ["NumericMatrix"] + compile_cpp(cpp, label="reassigned matrix local") + + +def test_nested_array_variants_keep_their_argument_and_return_types(): + cpp = transpile("""//@version=6 +strategy("nested array profiles") +identity(values) => values +outer(values) => identity(values) +integerInput = array.new() +decimalInput = array.new() +integerInput.push(7) +decimalInput.push(close) +integerOutput = outer(integerInput) +decimalOutput = outer(decimalInput) +plot(integerOutput.get(0) + decimalOutput.get(0)) +""") + for function in ("outer", "identity"): + found = signatures(cpp, function) + assert ("std::vector", "std::vector& values") in found + assert ("std::vector", "std::vector& values") in found + compile_cpp(cpp, label="nested structured type profiles") From 9237e1baeef8656aad592325ce36929b9ac81ae0 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:28:05 +0800 Subject: [PATCH 2/3] Check compound values through native execution Use the established integer storage and matrix alias in type-flow assertions. Check array specializations, mutations, matrix values and copy independence. --- tests/test_callable_compound_type_flow.py | 114 ++++++++++++++++++++-- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/tests/test_callable_compound_type_flow.py b/tests/test_callable_compound_type_flow.py index d759c3cc..791e82a6 100644 --- a/tests/test_callable_compound_type_flow.py +++ b/tests/test_callable_compound_type_flow.py @@ -5,7 +5,7 @@ import pytest from pineforge_codegen import transpile -from tests._compile import compile_cpp +from tests._compile import compile_cpp, run_emitted_tu def signatures(cpp, name): @@ -140,16 +140,77 @@ def test_untyped_array_calls_keep_distinct_element_types(): trim(decimals, 4) """) found = signatures(cpp, "trim") + # Small, non-epoch int arrays retain int storage; widening is a separate + # provenance rule pinned by test_wide_int_arrays_and_literals.py. assert sorted(parameters for _, parameters in found) == [ "std::vector& values, int maximum", - "std::vector& values, int maximum", + "std::vector& values, int maximum", ] targets = re.findall( r"^ (trim(?:_cs\d+|__ni\d+)?)\((integers|decimals), 4\);$", cpp, re.MULTILINE, ) assert len(targets) == 2 and len({name for name, _ in targets}) == 2 - compile_cpp(cpp, label="distinct array argument types") + assert {argument for _, argument in targets} == {"integers", "decimals"} + bound_targets = {argument: name for name, argument in targets} + driver = r''' +#include +#include +#include + +template +void dump_array_values(const std::vector& values) { + std::cout << values.size(); + for (auto value : values) std::cout << ' ' << value; + std::cout << '\n'; +} + +int main() { + const Bar bars[] = { + Bar{9.0, 11.0, 8.0, 10.25, 10.0, 1743465600000LL}, + Bar{19.0, 21.0, 18.0, 20.5, 11.0, 1743466500000LL}, + Bar{33.0, 35.0, 32.0, 33.75, 12.0, 1743467400000LL}, + }; + GeneratedStrategy strategy; + strategy.run(bars, 3); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 2; + } + if (strategy.integers.size() != 1 || strategy.decimals.size() != 1) return 3; + std::cout << std::setprecision(17) + << strategy.integers.size() << ' ' << strategy.integers.front() + << ' ' << strategy.decimals.size() << ' ' << strategy.decimals.front() + << '\n'; + + // The unchanged Pine source creates singleton arrays. Exercise both + // emitted bodies with larger native arguments as well, including the + // untaken branch, one removal, and equality at the size boundary. + std::vector integer_values{-3, 0, 7, 21, 41}; + std::vector decimal_values{-3.25, 0.5, 7.75, 21.125, 41.875}; + for (int maximum : {5, 4, 4}) { + strategy.INTEGER_TARGET(integer_values, maximum); + strategy.DECIMAL_TARGET(decimal_values, maximum); + dump_array_values(integer_values); + dump_array_values(decimal_values); + } +} +'''.replace("INTEGER_TARGET", bound_targets["integers"]).replace( + "DECIMAL_TARGET", bound_targets["decimals"], + ) + output = run_emitted_tu( + cpp, driver, opt="-O2", label="distinct array argument types", + ) + rows = [[float(value) for value in line.split()] for line in output.splitlines()] + assert rows == [ + [1, 2, 1, 33.75], + [5, -3, 0, 7, 21, 41], + [5, -3.25, 0.5, 7.75, 21.125, 41.875], + [4, 0, 7, 21, 41], + [4, 0.5, 7.75, 21.125, 41.875], + [4, 0, 7, 21, 41], + [4, 0.5, 7.75, 21.125, 41.875], + ] def test_reassigned_matrix_local_keeps_its_initializer_type(): @@ -168,11 +229,52 @@ def test_reassigned_matrix_local_keeps_its_initializer_type(): log.info(str.tostring(working.get(0, 0))) """) declarations = re.findall( - r"^\s+(NumericMatrix|double) working(?: = [^\n;]*)?;$", + r"^\s+(PineMatrix|double) working(?: = [^\n;]*)?;$", cpp, re.MULTILINE, ) - assert declarations == ["NumericMatrix"] - compile_cpp(cpp, label="reassigned matrix local") + assert declarations == ["PineMatrix"] + driver = r''' +#include +#include +#include +#include + +// The emitter's PineMatrix spelling denotes the actual numeric matrix handle. +static_assert(std::is_same_v< + decltype(std::declval().working), pineforge::NumericMatrix>); + +int main() { + const Bar bars[] = { + Bar{9.0, 11.0, 8.0, 10.25, 10.0, 1743465600000LL}, + Bar{19.0, 21.0, 18.0, 20.5, 11.0, 1743466500000LL}, + Bar{33.0, 35.0, 32.0, 33.75, 12.0, 1743467400000LL}, + }; + GeneratedStrategy strategy; + strategy.run(bars, 3); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 2; + } + std::cout << std::setprecision(17) + << strategy.state.rows() << ' ' << strategy.state.columns() + << ' ' << strategy.state.get(0, 0) + << ' ' << strategy.working.rows() << ' ' << strategy.working.columns() + << ' ' << strategy.working.get(0, 0) << '\n'; + strategy.working.set(0, 0, -17.5); + std::cout << strategy.state.get(0, 0) << ' ' << strategy.working.get(0, 0) << '\n'; + strategy.state.set(0, 0, 99.25); + std::cout << strategy.state.get(0, 0) << ' ' << strategy.working.get(0, 0) << '\n'; +} +''' + output = run_emitted_tu( + cpp, driver, opt="-O2", label="reassigned matrix local", + ) + rows = [[float(value) for value in line.split()] for line in output.splitlines()] + assert rows == [ + [1, 1, 33.75, 1, 1, 33.75 / 4], + [33.75, -17.5], + [99.25, -17.5], + ] def test_nested_array_variants_keep_their_argument_and_return_types(): From e004cbf5bea11d48d20df828b62b05ca6f9435dd Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:28:58 +0800 Subject: [PATCH 3/3] Bound type inference within expression queries Share operand and value inference while an expression query is active, then release cached results before bindings can change. Preserve ordinary matrix member initialization when the declaration expression is still available. --- CHANGELOG.md | 2 + pineforge_codegen/analyzer/base.py | 22 +++++++---- pineforge_codegen/analyzer/types.py | 58 ++++++++++++++++++++++++----- pineforge_codegen/codegen/base.py | 8 +++- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a19f76..45161886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ supported as exact pairs; on the 0.x line they are independent. See the returns and history clones. Untyped arguments retain collection element types through nested calls and tuple bindings, and reassigned matrix locals keep their initializer's matrix type. (#391) +- Deeply nested user-function calls share type-inference work within each + expression query, including when a return type remains unresolved. - `array.sort_indices` returns integer indices, accepts typed `array` results and honors ascending/default or descending order in namespace and method calls, including named arguments. Numeric `na` values sort last in diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index 80f2af34..37700cb6 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -7084,6 +7084,14 @@ def _operator_operand_type( parameter_types: dict[str, PineType | TypeSpec] | None = None, ) -> TypeSpec | None: """Use lexical type facts and written-call facts, never a float fallback.""" + return OperatorTypeQuery(self, parameter_types)._operand_type(node) + + def _operator_operand_type_uncached( + self, node: ASTNode, + parameter_types: dict[str, PineType | TypeSpec] | None, + query: OperatorTypeQuery, + ) -> TypeSpec | None: + """Resolve an operand using the current expression query's memo.""" if isinstance(node, Identifier) and parameter_types is not None: if node.name in parameter_types: ptype = parameter_types[node.name] @@ -7138,7 +7146,7 @@ def _operator_operand_type( arguments.append(node.kwargs.get(name)) result = source_builtin_result( ns, callee.member, tuple( - self._operator_operand_type(arg, parameter_types) + query._operand_type(arg) if arg is not None else None for arg in arguments ), ) @@ -7146,7 +7154,7 @@ def _operator_operand_type( return result # The structured return contract can prove a collection # even when no scalar result rule applies. - result = OperatorTypeQuery(self, parameter_types)._type_spec_from_expr(node) + result = query._type_spec_from_expr(node) return result if result is not None and result.kind != "primitive" else None if isinstance(callee, Identifier) and callee.name in {"nz", "fixnan", "input"}: return None @@ -7154,7 +7162,7 @@ def _operator_operand_type( if isinstance(node.object, Subscript): # Nested history is decided by the subscript checker. return None - receiver = self._operator_operand_type(node.object, parameter_types) + receiver = query._operand_type(node.object) if receiver is None: return None if receiver.kind in {"array", "matrix", "map"}: @@ -7176,7 +7184,7 @@ def _operator_operand_type( terminal = self._selection_terminal_expr(body) if terminal is None: continue - arm = self._operator_operand_type(terminal, parameter_types) + arm = query._operand_type(terminal) if arm is None: return None if result is None: @@ -7198,7 +7206,7 @@ def _operator_operand_type( else: op, operands = "?:", (node.condition, node.true_val, node.false_val) result, _ = resolve_operator(op, tuple( - self._operator_operand_type(expr, parameter_types) + query._operand_type(expr) for expr in operands )) return result @@ -7236,7 +7244,7 @@ def _operator_operand_type( or getattr(sym, "_pf_parameter_owner", None)): result = sym.type_spec if result is None and isinstance(node, (MemberAccess, FuncCall, Subscript)): - result = OperatorTypeQuery(self, parameter_types)._type_spec_from_expr(node) + result = query._type_spec_from_expr(node) if isinstance(node, FuncCall) and result is not None and result.kind == "primitive": # A definition-wide callable result is legacy inference. # Collection accessors instead have a declared element slot. @@ -7249,7 +7257,7 @@ def _operator_operand_type( else callee.object ) receiver_type = ( - self._operator_operand_type(receiver, parameter_types) + query._operand_type(receiver) if receiver is not None else None ) accessors = { diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index 7961ce2d..3eda26e9 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -1464,10 +1464,41 @@ class OperatorTypeQuery(TypeHelper): def __init__(self, analyzer, parameter_types): self._analyzer = analyzer self._parameter_types = parameter_types + self._type_query_memo = None def __getattr__(self, name): return getattr(self._analyzer, name) + def _memoized_type(self, node, kind, context, infer): + # Share successful and unresolved results across the value/operand + # fallback paths of one read-only query. Drop the entire memo when + # that query returns, before any declaration, scope or callable facts + # can change. A later query, even on this object, starts fresh. + memo = self._type_query_memo + root = memo is None + if root: + memo = self._type_query_memo = {} + try: + key = (kind, id(node), context) + if key not in memo: + if self._analyzer._budget is not None: + self._analyzer._budget.check( + getattr(node, "loc", None), Phase.ANALYZER, + ) + memo[key] = infer() + return memo[key] + finally: + if root: + self._type_query_memo = None + + def _operand_type(self, node): + return self._memoized_type( + node, "operand", None, + lambda: self._analyzer._operator_operand_type_uncached( + node, self._parameter_types, self, + ), + ) + def check_body(self, body): for node in body: self._check_node(node) @@ -1538,11 +1569,14 @@ def body_value_spec(self, body): return result def _value_spec(self, value): + return self._memoized_type( + value, "value", None, lambda: self._value_spec_uncached(value), + ) + + def _value_spec_uncached(self, value): spec = self._type_spec_from_expr(value) if spec is None: - spec = self._analyzer._operator_operand_type( - value, self._parameter_types - ) + spec = self._operand_type(value) return spec def _check_node(self, node): @@ -1636,11 +1670,17 @@ def _check_node(self, node): return self._analyzer._check_operator_operands(node, op, operands, self._parameter_types) - def _type_spec_from_expr(self, value, **kwargs): + def _type_spec_from_expr(self, value, *, _ternary_arms=None): + return self._memoized_type( + value, "spec", _ternary_arms, + lambda: self._type_spec_from_expr_uncached( + value, _ternary_arms=_ternary_arms, + ), + ) + + def _type_spec_from_expr_uncached(self, value, **kwargs): if isinstance(value, Identifier): - return self._analyzer._operator_operand_type( - value, self._parameter_types, - ) + return self._operand_type(value) if (isinstance(value, FuncCall) and isinstance(value.callee, Identifier) and value.callee.name in self._analyzer._func_defs): @@ -1663,7 +1703,7 @@ def _type_spec_from_expr(self, value, **kwargs): def _visit(self, node): # TypeHelper's array.from fallback normally visits its arguments. # Here it asks for facts only, so no TA or request is registered again. - spec = self._analyzer._operator_operand_type(node, self._parameter_types) + spec = self._operand_type(node) if spec is not None and spec.kind == "primitive": return next((ptype for ptype in PineType if ptype.value == spec.name), PineType.UNKNOWN) @@ -1672,7 +1712,7 @@ def _visit(self, node): def _array_from_element_spec(self, value): # Array construction must not recover a coarse symbol/call result # after the operator query deliberately left its source unresolved. - return self._analyzer._operator_operand_type(value, self._parameter_types) + return self._operand_type(value) @staticmethod def _pine_type_to_spec(pine_type): diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index fdb388ba..0f306988 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -2551,6 +2551,7 @@ def _register_global_aggregate_member_types(self) -> None: # final state and can already contain an invalid later reassignment's # element type. The original declaration must remain the compatibility # baseline used to reject matrix := matrix. + gem = getattr(self.ctx, "global_expr_map", {}) or {} for stmt, _in_loop in self._walk_global_scope_with_loopflag( getattr(self.ctx.ast, "body", []), False ): @@ -2572,11 +2573,15 @@ def _register_global_aggregate_member_types(self) -> None: self._array_vars.add(stmt.name) self._collection_types.setdefault(stmt.name, declared_spec) continue - else: + elif stmt.name not in gem: # Reassignment removes a binding from global_expr_map. Its # initializer still establishes its matrix handle type, also # for declarations inside top-level control-flow blocks. declared_spec = self._type_spec_from_expr(stmt.value) + else: + # An available initializer still uses the established member + # inference and default initialization below. + continue if (from_selection and declared_spec is not None and declared_spec.kind == "array"): # ``a = switch mode ...`` whose every arm is an array of one @@ -2591,7 +2596,6 @@ def _register_global_aggregate_member_types(self) -> None: self._matrix_specs.setdefault(stmt.name, declared_spec) self._collection_types[stmt.name] = self._matrix_specs[stmt.name] - gem = getattr(self.ctx, "global_expr_map", {}) or {} for name, _ptype in self.ctx.global_var_decls: expr = gem.get(name) if expr is None or not isinstance(expr, FuncCall):