diff --git a/CHANGELOG.md b/CHANGELOG.md index 88dac8f9..3cc677dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,12 @@ supported as exact pairs; on the 0.x line they are independent. See the bytes: the timezone macro block and newly reserved names remain. Regenerate C++ and relink affected strategies. +- User functions retain array, object and string result types through branch + returns and history clones. Untyped arguments retain collection element + types through nested calls and tuple bindings, and reassigned matrix locals + keep their initializer's matrix type. (#391) +- Deeply nested user-function calls share type-inference work within each + expression query, including when a return type remains unresolved. - `array.sort_indices` returns integer indices, accepts typed `array` results and honors ascending/default or descending order in namespace and method calls, including named arguments. Numeric `na` values sort last in diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index 772e2426..37700cb6 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -309,6 +309,13 @@ def __init__(self, ast: Program, filename: str = "", # Probe: data/validation/udt-method-probe-20-udt-return-from-func. self._func_udt_return_types: dict[str, str] = {} self._func_return_type_specs: dict[str, "TypeSpec"] = {} + self._func_tuple_element_specs: dict[str, tuple] = {} + self._callable_bound_param_specs_by_node: dict[int, tuple] = {} + self._callable_bound_return_specs_by_node: dict[int, TypeSpec] = {} + self._callable_edge_specs: dict = {} + self._callable_return_specs_by_profile: dict = {} + self._func_callsite_param_specs: dict = {} + self._func_callsite_return_specs: dict = {} self._func_param_type_specs: dict[str, list] = {} # Direct terminal gets on an unshadowed built-in temporary are # registered while their lexical scope is live. A later-defined UDF @@ -769,6 +776,24 @@ def _analyze(self) -> AnalyzerContext: func_callsite_return_types=dict( self._func_callsite_return_types ), + func_callsite_param_specs={ + **{ + key: self._callable_bound_param_specs_by_node[node_id] + for node_id, key in self._func_call_cs_map.items() + if node_id in self._callable_bound_param_specs_by_node + }, + **self._func_callsite_param_specs, + }, + func_callsite_return_specs={ + **{ + key: self._callable_bound_return_specs_by_node[node_id] + for node_id, key in self._func_call_cs_map.items() + if node_id in self._callable_bound_return_specs_by_node + }, + **self._func_callsite_return_specs, + }, + func_value_call_edges=dict(self._callable_edge_specs), + func_value_return_specs=dict(self._callable_return_specs_by_profile), func_declared_param_type_specs={ name: tuple(specs) for name, specs in self._func_param_type_specs.items() @@ -3370,6 +3395,52 @@ def _edge_ta_variant_target_map( func_info_by_name, _bound_user_call_args, ) + self._resolve_callable_callsite_value_specs(call_edges, func_info_by_name) + + def _resolve_callable_callsite_value_specs(self, call_edges, infos): + """Bind exact value profiles to the variants the emitter actually uses.""" + profiles = self._func_callsite_param_specs + for owner, callee, call in call_edges: + target = self._func_call_cs_map.get(id(call)) + if owner is None and target is not None: + profile = self._callable_bound_param_specs_by_node.get(id(call)) + if profile is not None: + profiles[target] = profile + for _ in range(len(call_edges) + 1): + changed = False + for owner, callee, call in call_edges: + if owner is None or owner not in infos: + continue + info = infos[owner] + if info.node is None: + continue + count = self._func_call_site_count.get(owner, 0) + for owner_cs in range(count) if count else (None,): + profile = profiles.get( + (owner, owner_cs), tuple(info.param_type_specs), + ) + incoming = self._callable_edge_specs.get( + (owner, profile, id(call)) + ) + if incoming is None: + continue + target = self._func_call_cs_map.get(id(call)) + if target is None: + callee_count = self._func_call_site_count.get(callee, 0) + if owner_cs is None or owner_cs >= callee_count: + continue + target = (callee, owner_cs) + if target not in profiles: + profiles[target] = incoming + changed = True + if not changed: + break + if self._budget is not None: + self._budget.check(phase=Phase.ANALYZER) + for (name, cs), profile in profiles.items(): + exact = self._callable_return_specs_by_profile.get((name, profile)) + if exact is not None: + self._func_callsite_return_specs[(name, cs)] = exact # The primitive families a written call can pass an untyped parameter. _ARGUMENT_FAMILIES = frozenset({ @@ -3443,6 +3514,9 @@ def reads(value, names: set[str]) -> bool: owner_params = set(untyped_params(owner).values()) actuals = bound_args(callee, call) recorded = self._callable_bound_param_types_by_node.get(id(call), []) + recorded_specs = self._callable_bound_param_specs_by_node.get( + id(call), () + ) for index in params: if (index >= len(actuals) or index >= len(recorded) or actuals[index] is None @@ -3450,6 +3524,19 @@ def reads(value, names: set[str]) -> bool: continue family = recorded[index] shared = shared_family(callee, index) + info_specs = func_info_by_name[callee].param_type_specs + incoming_spec = ( + recorded_specs[index] if index < len(recorded_specs) else None + ) + shared_spec = info_specs[index] if index < len(info_specs) else None + if (incoming_spec is not None and shared_spec is not None + and incoming_spec != shared_spec + and (incoming_spec.kind != "primitive" + or shared_spec.kind != "primitive")): + # Structured types are argument families too. In particular + # vector& cannot receive a vector argument. + conflicted.add(callee) + break if (family in self._ARGUMENT_FAMILIES and shared in self._ARGUMENT_FAMILIES and family != shared @@ -5563,6 +5650,12 @@ def _visit_TupleAssign(self, node: TupleAssign) -> PineType: is_val_static = self._is_static_expression(node.value) declared_types: list[PineType] = [] + tuple_specs = ( + self._func_tuple_element_specs.get(node.value.callee.name, ()) + if isinstance(node.value, FuncCall) + and isinstance(node.value.callee, Identifier) + else () + ) for idx, name in enumerate(node.names): inferred_element_type = ( @@ -5612,6 +5705,13 @@ def _visit_TupleAssign(self, node: TupleAssign) -> PineType: setattr(sym, "_pf_decl_node_id", id(node)) setattr(sym, "_pf_decl_binding_name", name) setattr(sym, "_pf_tuple_binding", True) + if (idx < len(tuple_specs) and tuple_specs[idx] is not None + and (tuple_specs[idx].kind != "primitive" + or tuple_specs[idx].name == "string")): + sym.type_spec = tuple_specs[idx] + self._record_collection_type( + name, tuple_specs[idx], symbol_scope=sym.scope, + ) self._record_script_block_binding(sym, outer_symbol) self._note_member_writer(sym, name, node) @@ -5823,6 +5923,18 @@ def _visit_FuncDef(self, node: FuncDef) -> PineType: terminal_direct_return_spec = self._type_spec_from_expr( terminal_ret_expr ) + if isinstance(terminal_ret_expr, TupleLiteral): + self._func_tuple_element_specs[node.name] = tuple( + self._type_spec_from_expr(element) + for element in terminal_ret_expr.elements + ) + elif (isinstance(terminal_ret_expr, FuncCall) + and isinstance(terminal_ret_expr.callee, Identifier)): + self._func_tuple_element_specs[node.name] = ( + self._func_tuple_element_specs.get( + terminal_ret_expr.callee.name, () + ) + ) terminal_map_return = self._terminal_map_call_return( terminal_ret_expr, { @@ -6972,6 +7084,14 @@ def _operator_operand_type( parameter_types: dict[str, PineType | TypeSpec] | None = None, ) -> TypeSpec | None: """Use lexical type facts and written-call facts, never a float fallback.""" + return OperatorTypeQuery(self, parameter_types)._operand_type(node) + + def _operator_operand_type_uncached( + self, node: ASTNode, + parameter_types: dict[str, PineType | TypeSpec] | None, + query: OperatorTypeQuery, + ) -> TypeSpec | None: + """Resolve an operand using the current expression query's memo.""" if isinstance(node, Identifier) and parameter_types is not None: if node.name in parameter_types: ptype = parameter_types[node.name] @@ -7026,7 +7146,7 @@ def _operator_operand_type( arguments.append(node.kwargs.get(name)) result = source_builtin_result( ns, callee.member, tuple( - self._operator_operand_type(arg, parameter_types) + query._operand_type(arg) if arg is not None else None for arg in arguments ), ) @@ -7034,7 +7154,7 @@ def _operator_operand_type( return result # The structured return contract can prove a collection # even when no scalar result rule applies. - result = OperatorTypeQuery(self, parameter_types)._type_spec_from_expr(node) + result = query._type_spec_from_expr(node) return result if result is not None and result.kind != "primitive" else None if isinstance(callee, Identifier) and callee.name in {"nz", "fixnan", "input"}: return None @@ -7042,7 +7162,7 @@ def _operator_operand_type( if isinstance(node.object, Subscript): # Nested history is decided by the subscript checker. return None - receiver = self._operator_operand_type(node.object, parameter_types) + receiver = query._operand_type(node.object) if receiver is None: return None if receiver.kind in {"array", "matrix", "map"}: @@ -7064,7 +7184,7 @@ def _operator_operand_type( terminal = self._selection_terminal_expr(body) if terminal is None: continue - arm = self._operator_operand_type(terminal, parameter_types) + arm = query._operand_type(terminal) if arm is None: return None if result is None: @@ -7086,7 +7206,7 @@ def _operator_operand_type( else: op, operands = "?:", (node.condition, node.true_val, node.false_val) result, _ = resolve_operator(op, tuple( - self._operator_operand_type(expr, parameter_types) + query._operand_type(expr) for expr in operands )) return result @@ -7124,7 +7244,7 @@ def _operator_operand_type( or getattr(sym, "_pf_parameter_owner", None)): result = sym.type_spec if result is None and isinstance(node, (MemberAccess, FuncCall, Subscript)): - result = OperatorTypeQuery(self, parameter_types)._type_spec_from_expr(node) + result = query._type_spec_from_expr(node) if isinstance(node, FuncCall) and result is not None and result.kind == "primitive": # A definition-wide callable result is legacy inference. # Collection accessors instead have a declared element slot. @@ -7137,7 +7257,7 @@ def _operator_operand_type( else callee.object ) receiver_type = ( - self._operator_operand_type(receiver, parameter_types) + query._operand_type(receiver) if receiver is not None else None ) accessors = { @@ -8358,35 +8478,41 @@ def _validate_deferred_param_history_refs( ) validated.add(key) - def _propagate_deferred_map_callable_specs( + def _propagate_deferred_callable_specs( self, owner: str, parameter_specs: dict[str, TypeSpec | None], visiting: frozenset[str] = frozenset(), + _memo: set | None = None, ) -> bool: - """Monomorphize an all-untyped wrapper chain for one concrete map call. + """Carry exact value types through deferred callable argument edges. Function definitions are analyzed before their concrete top-level call sites. Consequently ``wrapper(a) => identity(a)`` initially creates both ``FuncInfo`` records with scalar fallbacks. The definition pass already captured identity-keyed parameter-flow edges for deferred map - history validation; reuse those exact edges to carry a later map - ``TypeSpec`` inward, then infer map returns on the way back out. - - This is deliberately bounded and map-triggered. Cycles stop at the - active owner, incompatible previously-established parameter specs stop - that edge, and scalar-only call graphs never mutate. PineForge emits - one C++ body per ordinary UDF, so silently replacing one concrete map - specialization with a different one would be a false polymorphic - inference rather than a valid widening. + history validation; reuse those exact edges to carry the concrete + ``TypeSpec`` inward, then infer returns on the way back out. + + Queries are memoized by function and input types, never by call path. + A diamond therefore visits each concrete signature once. Existing map + incompatibility diagnostics remain authoritative; other distinct value + families are retained at their written calls for clone specialization. """ if owner in visiting: return False if not any( - spec is not None and spec.kind == "map" + spec is not None + and (spec.kind != "primitive" or spec.name == "string") for spec in parameter_specs.values() ): return False + if _memo is None: + _memo = set() + profile = (owner, tuple(parameter_specs.items())) + if profile in _memo: + return False + _memo.add(profile) func_info = next( (info for info in self._func_infos if info.name == owner), @@ -8395,6 +8521,8 @@ def _propagate_deferred_map_callable_specs( func_def = self._func_defs.get(owner) if func_info is None or func_def is None: return False + if self._budget is not None: + self._budget.check(func_def.loc, Phase.ANALYZER) changed = False while len(func_info.param_type_specs) < len(func_def.params): @@ -8439,7 +8567,7 @@ def _propagate_deferred_map_callable_specs( for _ in range(max(1, len(edges) + 1)): pass_changed = False for ( - _call_id, + call_id, callee_owner, callee_param_names, bound_args, @@ -8493,10 +8621,19 @@ def _propagate_deferred_map_callable_specs( ) ): callee_specs[param_name] = contextual_map_spec - if self._propagate_deferred_map_callable_specs( + self._callable_bound_param_specs_by_node[call_id] = tuple( + callee_specs.get(name) for name in callee_param_names + ) + self._callable_edge_specs[( + owner, + tuple(parameter_specs.get(name) for name in func_def.params), + call_id, + )] = self._callable_bound_param_specs_by_node[call_id] + if self._propagate_deferred_callable_specs( callee_owner, callee_specs, next_visiting, + _memo, ): pass_changed = True changed = changed or pass_changed @@ -8506,6 +8643,16 @@ def _propagate_deferred_map_callable_specs( terminal = self._direct_terminal_return_expr(func_def) return_spec = None scalar_return_type = None + exact = self._callable_value_return_spec(func_def, parameter_specs) + if exact is not None and ( + exact.kind != "primitive" or exact.name == "string" + ): + return_spec = exact + self._callable_return_specs_by_profile[( + owner, tuple(parameter_specs.get(name) for name in func_def.params), + )] = exact + if exact.kind == "primitive": + scalar_return_type = self._primitive_pine_type_from_spec(exact) if isinstance(terminal, Identifier) and terminal.name in parameter_specs: candidate = parameter_specs.get(terminal.name) if candidate is not None and candidate.kind == "map": @@ -8532,7 +8679,9 @@ def _propagate_deferred_map_callable_specs( return_spec = candidate if return_spec is not None: established_return = self._func_return_type_specs.get(owner) - if established_return is None: + if (established_return is None + or (established_return.kind == "primitive" + and return_spec.kind != "primitive")): self._func_return_type_specs[owner] = return_spec func_info.return_type_spec = return_spec changed = True @@ -8540,6 +8689,14 @@ def _propagate_deferred_map_callable_specs( if func_info.return_type_spec is None: func_info.return_type_spec = return_spec changed = True + if return_spec.kind == "primitive": + result_type = self._primitive_pine_type_from_spec(return_spec) + if result_type != PineType.UNKNOWN: + func_info.return_type = result_type + self._func_return_types[owner] = result_type + elif return_spec.kind == "udt": + func_info.udt_return_type = return_spec.name + self._func_udt_return_types[owner] = return_spec.name # A different established return belongs to another concrete map # specialization. Keep it unchanged rather than falsely widening. diff --git a/pineforge_codegen/analyzer/call_handlers.py b/pineforge_codegen/analyzer/call_handlers.py index 8fef0994..7f25b6b2 100644 --- a/pineforge_codegen/analyzer/call_handlers.py +++ b/pineforge_codegen/analyzer/call_handlers.py @@ -1924,6 +1924,7 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType: for i in range(len(param_specs)): if param_specs[i] is None and i < len(arg_specs): param_specs[i] = arg_specs[i] + self._callable_bound_param_specs_by_node[id(node)] = tuple(param_specs) self._record_deferred_param_call_edge( node, func_name, @@ -2171,13 +2172,43 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType: # scalar fallbacks. Propagate the concrete specs through the exact # definition-time call edges and infer returns in post-order before # codegen snapshots the FuncInfo table. - self._propagate_deferred_map_callable_specs( + self._propagate_deferred_callable_specs( func_name, { name: spec for name, spec in zip(func_def.params, param_specs) }, ) + value_specs = [*param_specs, fi.return_type_spec] + exact_return = ( + self._callable_value_return_spec( + func_def, dict(zip(func_def.params, param_specs)), + ) + if any( + spec is not None + and (spec.kind != "primitive" or spec.name == "string") + for spec in value_specs + ) + else None + ) + if exact_return is not None and ( + exact_return.kind != "primitive" or exact_return.name == "string" + ): + self._callable_bound_return_specs_by_node[id(node)] = exact_return + self._callable_return_specs_by_profile[ + (func_name, tuple(param_specs)) + ] = exact_return + if (fi.return_type_spec is None + or fi.return_type_spec.kind == "primitive"): + fi.return_type_spec = exact_return + self._func_return_type_specs[func_name] = exact_return + if exact_return.kind == "primitive": + return_type = self._primitive_pine_type_from_spec(exact_return) + fi.return_type = return_type + self._func_return_types[func_name] = return_type + elif exact_return.kind == "udt": + fi.udt_return_type = exact_return.name + self._func_udt_return_types[func_name] = exact_return.name if return_type == PineType.NA: self._na_return_callables[id(node)] = func_def diff --git a/pineforge_codegen/analyzer/contracts.py b/pineforge_codegen/analyzer/contracts.py index 93478b1c..9b7a39a4 100644 --- a/pineforge_codegen/analyzer/contracts.py +++ b/pineforge_codegen/analyzer/contracts.py @@ -293,6 +293,12 @@ class AnalyzerContext: # helper called with both ``bar_index`` and ``close``). func_callsite_param_types: dict = field(default_factory=dict) func_callsite_return_types: dict = field(default_factory=dict) + # Exact argument/return value types retain collection element types, object + # identities and strings independently of the coarse primitive profiles. + func_callsite_param_specs: dict = field(default_factory=dict) + func_callsite_return_specs: dict = field(default_factory=dict) + func_value_call_edges: dict = field(default_factory=dict) + func_value_return_specs: dict = field(default_factory=dict) # Declaration-time parameter specs, before any call-site inference fills # untyped slots on the legacy FuncInfo record. Codegen uses this to apply # variant overrides only to genuinely untyped parameters. diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index f65daf91..3eda26e9 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -294,7 +294,7 @@ def _selection_type_spec( *, nullable: bool | None = None, ) -> TypeSpec | None: - """Keep exact collection joins; extend other families only around ``na``.""" + """A selection of identical value types retains that exact type.""" if nullable is None: nullable = any(self._selection_contains_na(node) for node, _ in branches) result: TypeSpec | None = None @@ -307,8 +307,6 @@ def _selection_type_spec( spec = TypeSpec.primitive("color") if spec is None: return None - if not nullable and spec.kind not in {"map", "matrix"}: - return None result = ( spec if result is None @@ -522,6 +520,11 @@ def direct_user_udt_ctor_name(node: ASTNode) -> str | None: )) return self._selection_type_spec(branches) if isinstance(value, FuncCall): + exact = getattr( + self, "_callable_bound_return_specs_by_node", {} + ).get(id(value)) + if exact is not None: + return exact cal = value.callee if (isinstance(cal, Identifier) and cal.name in ("line", "box") and len(value.args) == 1 and not value.kwargs @@ -875,6 +878,27 @@ def _callsite_terminal_return_spec( self._current_top_level_stmt = previous_owner self._symbols.current_scope = previous_scope + def _callable_value_return_spec(self, func_def, parameter_specs): + """Read a callable's value with its lexical locals and concrete inputs. + + This query never revisits the analyzer or allocates stateful call sites. + Each branch gets its own environment, so an expired local cannot borrow + the type of a same-named global or another branch's declaration. + """ + previous_owner = self._current_top_level_stmt + previous_scope = self._symbols.current_scope + self._symbols.current_scope = self._operator_definition_scopes.get( + id(func_def), self._symbols.global_scope, + ) + self._current_top_level_stmt = func_def + try: + return OperatorTypeQuery(self, dict(parameter_specs)).body_value_spec( + func_def.body + ) + finally: + self._current_top_level_stmt = previous_owner + self._symbols.current_scope = previous_scope + def _terminal_map_selection_return_spec( self, value: ASTNode | None, @@ -1440,14 +1464,121 @@ class OperatorTypeQuery(TypeHelper): def __init__(self, analyzer, parameter_types): self._analyzer = analyzer self._parameter_types = parameter_types + self._type_query_memo = None def __getattr__(self, name): return getattr(self._analyzer, name) + def _memoized_type(self, node, kind, context, infer): + # Share successful and unresolved results across the value/operand + # fallback paths of one read-only query. Drop the entire memo when + # that query returns, before any declaration, scope or callable facts + # can change. A later query, even on this object, starts fresh. + memo = self._type_query_memo + root = memo is None + if root: + memo = self._type_query_memo = {} + try: + key = (kind, id(node), context) + if key not in memo: + if self._analyzer._budget is not None: + self._analyzer._budget.check( + getattr(node, "loc", None), Phase.ANALYZER, + ) + memo[key] = infer() + return memo[key] + finally: + if root: + self._type_query_memo = None + + def _operand_type(self, node): + return self._memoized_type( + node, "operand", None, + lambda: self._analyzer._operator_operand_type_uncached( + node, self._parameter_types, self, + ), + ) + def check_body(self, body): for node in body: self._check_node(node) + def body_value_spec(self, body): + """Infer declaration and terminal value types without checking operators.""" + result = None + for node in body or (): + if self._analyzer._budget is not None: + self._analyzer._budget.check(node.loc, Phase.ANALYZER) + if isinstance(node, VarDecl): + result = ( + self._type_spec_from_hint(node.type_hint) + if node.type_hint else self._value_spec(node.value) + ) + self._parameter_types[node.name] = result + elif isinstance(node, Assignment): + result = ( + self._parameter_types.get(node.target.name) + if isinstance(node.target, Identifier) + else self._value_spec(node.target) + ) + if result is None: + result = self._value_spec(node.value) + elif isinstance(node, TupleAssign): + # Tuple elements are independent value bindings. + specs = () + if isinstance(node.value, FuncCall) and isinstance( + node.value.callee, Identifier + ): + specs = getattr( + self._analyzer, "_func_tuple_element_specs", {} + ).get(node.value.callee.name, ()) + for index, name in enumerate(node.names): + self._parameter_types[name] = ( + specs[index] if index < len(specs) else None + ) + result = None + elif isinstance(node, IfStmt): + branches = [ + OperatorTypeQuery( + self._analyzer, dict(self._parameter_types) + ).body_value_spec(branch) + for branch in (node.body, node.else_body) + ] + result = unify_type_specs( + *branches, + left_is_na=not node.body, + right_is_na=not node.else_body, + ) + elif isinstance(node, SwitchStmt): + result = None + first = True + for branch in [ + *(body for _, body in node.cases), node.default_body + ]: + spec = OperatorTypeQuery( + self._analyzer, dict(self._parameter_types) + ).body_value_spec(branch) + result = spec if first else unify_type_specs( + result, spec, right_is_na=not branch, + ) + first = False + elif isinstance(node, ExprStmt): + result = self._value_spec(node.expr) + else: + result = self._value_spec(node) + return result + + def _value_spec(self, value): + return self._memoized_type( + value, "value", None, lambda: self._value_spec_uncached(value), + ) + + def _value_spec_uncached(self, value): + spec = self._type_spec_from_expr(value) + if spec is None: + spec = self._operand_type(value) + return spec + def _check_node(self, node): if not isinstance(node, ASTNode): if isinstance(node, (tuple, list)): @@ -1539,17 +1670,40 @@ def _check_node(self, node): return self._analyzer._check_operator_operands(node, op, operands, self._parameter_types) - def _type_spec_from_expr(self, value, **kwargs): + def _type_spec_from_expr(self, value, *, _ternary_arms=None): + return self._memoized_type( + value, "spec", _ternary_arms, + lambda: self._type_spec_from_expr_uncached( + value, _ternary_arms=_ternary_arms, + ), + ) + + def _type_spec_from_expr_uncached(self, value, **kwargs): if isinstance(value, Identifier): - return self._analyzer._operator_operand_type( - value, self._parameter_types, + return self._operand_type(value) + if (isinstance(value, FuncCall) + and isinstance(value.callee, Identifier) + and value.callee.name in self._analyzer._func_defs): + name = value.callee.name + definition = self._analyzer._func_defs[name] + args = self._analyzer._bind_callable_args( + value, list(definition.params) + ) + profile = tuple( + self._value_spec(arg) if arg is not None else None + for arg in args + ) + exact = self._analyzer._callable_return_specs_by_profile.get( + (name, profile) ) + if exact is not None: + return exact return super()._type_spec_from_expr(value, **kwargs) def _visit(self, node): # TypeHelper's array.from fallback normally visits its arguments. # Here it asks for facts only, so no TA or request is registered again. - spec = self._analyzer._operator_operand_type(node, self._parameter_types) + spec = self._operand_type(node) if spec is not None and spec.kind == "primitive": return next((ptype for ptype in PineType if ptype.value == spec.name), PineType.UNKNOWN) @@ -1558,7 +1712,7 @@ def _visit(self, node): def _array_from_element_spec(self, value): # Array construction must not recover a coarse symbol/call result # after the operator query deliberately left its source unresolved. - return self._analyzer._operator_operand_type(value, self._parameter_types) + return self._operand_type(value) @staticmethod def _pine_type_to_spec(pine_type): diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index 6bd0a8ad..0f306988 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -2145,6 +2145,26 @@ def natural_name(fname: str, cs_idx: int) -> str: owns_non_ta_state = bool( g_var_originals or fixnan_originals(g_name) ) + owner_info = self._func_info_map.get(inst["fname"]) + owner_profile = inst.get("param_specs") + if owner_profile is None and owner_info is not None: + owner_profile = getattr( + ctx, "func_callsite_param_specs", {} + ).get( + (inst["fname"], inst.get("type_call_site_idx", + inst.get("call_site_idx"))), + tuple(owner_info.param_type_specs), + ) + incoming = getattr(ctx, "func_value_call_edges", {}).get( + (inst["fname"], owner_profile, id(callnode)) + ) + natural_profile = getattr( + ctx, "func_callsite_param_specs", {} + ).get((g_name, j)) + different_value_type = ( + incoming is not None and natural_profile is not None + and incoming != natural_profile + ) # Every non-cs0 enclosing variant is a distinct Pine written # call path. Even a pure wrapper can reach var/fixnan state # farther down the graph, so preserve the path identity here @@ -2155,6 +2175,7 @@ def natural_name(fname: str, cs_idx: int) -> str: ) if ( not path_requires_fresh_state + and not different_value_type and composed_ta == natural_ta and composed_var == natural_var and composed_fixnan == natural_fixnan @@ -2174,6 +2195,7 @@ def natural_name(fname: str, cs_idx: int) -> str: frozenset(composed_var.items()), frozenset(composed_fixnan.items()), path_identity, + incoming, ) ginst = interned.get(key) if ginst is None: @@ -2210,6 +2232,10 @@ def natural_name(fname: str, cs_idx: int) -> str: # takes the argument that call passes, as the # call's own cs{j} clone does. "type_call_site_idx": j, + "param_specs": incoming, + "return_spec": getattr( + ctx, "func_value_return_specs", {} + ).get((g_name, incoming)), "ta_remap": composed_ta, "var_remap": fvar_remap, "fixnan_remap": ffixnan_remap, @@ -2525,6 +2551,7 @@ def _register_global_aggregate_member_types(self) -> None: # final state and can already contain an invalid later reassignment's # element type. The original declaration must remain the compatibility # baseline used to reject matrix := matrix. + gem = getattr(self.ctx, "global_expr_map", {}) or {} for stmt, _in_loop in self._walk_global_scope_with_loopflag( getattr(self.ctx.ast, "body", []), False ): @@ -2546,7 +2573,14 @@ def _register_global_aggregate_member_types(self) -> None: self._array_vars.add(stmt.name) self._collection_types.setdefault(stmt.name, declared_spec) continue + elif stmt.name not in gem: + # Reassignment removes a binding from global_expr_map. Its + # initializer still establishes its matrix handle type, also + # for declarations inside top-level control-flow blocks. + declared_spec = self._type_spec_from_expr(stmt.value) else: + # An available initializer still uses the established member + # inference and default initialization below. continue if (from_selection and declared_spec is not None and declared_spec.kind == "array"): @@ -2562,7 +2596,6 @@ def _register_global_aggregate_member_types(self) -> None: self._matrix_specs.setdefault(stmt.name, declared_spec) self._collection_types[stmt.name] = self._matrix_specs[stmt.name] - gem = getattr(self.ctx, "global_expr_map", {}) or {} for name, _ptype in self.ctx.global_var_decls: expr = gem.get(name) if expr is None or not isinstance(expr, FuncCall): diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index d9c67b33..0f4b2b24 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -2407,6 +2407,37 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], call_site_idx if call_site_idx is not None else (instance or {}).get("type_call_site_idx") ) + if type_call_site_idx is not None: + from dataclasses import replace + incoming = getattr( + self.ctx, "func_callsite_param_specs", {} + ).get((fi.name, type_call_site_idx), ()) + if (instance or {}).get("param_specs") is not None: + incoming = instance["param_specs"] + declared = getattr( + self.ctx, "func_declared_param_type_specs", {} + ).get(fi.name, ()) + specs = list(fi.param_type_specs) + while len(specs) < len(node.params): + specs.append(None) + for index, spec in enumerate(incoming): + if (index < len(specs) and spec is not None + and (spec.kind != "primitive" or spec.name == "string") + and (index >= len(declared) or declared[index] is None)): + specs[index] = spec + exact_return = getattr( + self.ctx, "func_callsite_return_specs", {} + ).get((fi.name, type_call_site_idx), fi.return_type_spec) + if (instance or {}).get("return_spec") is not None: + exact_return = instance["return_spec"] + fi = replace( + fi, param_type_specs=specs, return_type_spec=exact_return, + udt_return_type=( + exact_return.name + if exact_return is not None and exact_return.kind == "udt" + else fi.udt_return_type + ), + ) # Collection registries historically used raw variable names for the # whole translation unit. Emit each callable against copy-on-write @@ -2651,6 +2682,11 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], ret_type = "int64_t" elif ("return", fi.name) in self._array_widths.wide: ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) + elif (getattr(fi, "return_type_spec", None) is not None + and fi.return_type_spec.kind != "primitive"): + # Exact value types also apply to history/stateful clones. Their + # coarse primitive profile cannot represent a collection result. + ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) elif ( type_call_site_idx is not None and self._callsite_callable_return_pine_type( @@ -2664,8 +2700,6 @@ def _emit_func_def_body(self, fi: FuncInfo, lines: list[str], "double", ) elif getattr(fi, "return_type_spec", None) is not None: - # Exact return TypeSpec (collections plus narrowly cached primitive - # terminal reads) takes precedence over the coarse PineType slot. ret_type = self._type_spec_to_cpp(self._array_return_spec(fi)) else: ret_type = PINE_TYPE_TO_CPP.get(fi.return_type, "double") diff --git a/pineforge_codegen/codegen/types.py b/pineforge_codegen/codegen/types.py index abc3fb2b..9e58a0ee 100644 --- a/pineforge_codegen/codegen/types.py +++ b/pineforge_codegen/codegen/types.py @@ -3070,6 +3070,15 @@ def _type_spec_from_expr_raw( return TypeSpec.udt(func_name) if namespace is None: func_info = self._func_info_map.get(func_name) + if func_info is not None: + exact = getattr( + self.ctx, "func_callsite_return_specs", {} + ).get(( + func_info.name, + self._callable_target_callsite_idx(func_info, node), + )) + if exact is not None: + return exact return_spec = (self._array_return_spec(func_info) if func_info is not None else None) if return_spec is not None: diff --git a/tests/test_callable_compound_type_flow.py b/tests/test_callable_compound_type_flow.py new file mode 100644 index 00000000..791e82a6 --- /dev/null +++ b/tests/test_callable_compound_type_flow.py @@ -0,0 +1,297 @@ +"""Structured values retain their types across callable boundaries.""" + +import re + +import pytest + +from pineforge_codegen import transpile +from tests._compile import compile_cpp, run_emitted_tu + + +def signatures(cpp, name): + return re.findall( + rf"^ ([^\n]+?) {re.escape(name)}(?:_cs\d+|__ni\d+)?\(([^\n]*)\) \{{$", + cpp, + re.MULTILINE, + ) + + +CASES = [ + pytest.param( + """choose(array values, bool enabled) => + if enabled + values + else + values +items = array.from(close) +selected = choose(items, close > open) +plot(selected.get(0)) +""", + "choose", "std::vector", "std::vector& values, bool enabled", 1, + id="array-block-return", + ), + pytest.param( + """type Reading + float value +choose(bool enabled) => + Reading item = Reading.new(close) + if enabled + item + else + item +chosen = choose(close > open) +plot(chosen.value) +""", + "choose", "Reading", "bool enabled", 1, + id="object-block-return", + ), + pytest.param( + """choose(bool enabled, array left, array right) => + enabled ? left : right +left = array.from("first") +right = array.from("second") +chosen = choose(close > open, left, right) +log.info(chosen.get(0)) +""", + "choose", "std::vector", + "bool enabled, std::vector& left, std::vector& right", 1, + id="array-ternary-return", + ), + pytest.param( + """source = ta.ema(close, 7) +features(int offset) => + array items = array.from(source[offset]) + items +current = features(0) +previous = features(2) +plot(current.get(0) - previous.get(0)) +""", + "features", "std::vector", "int offset", 2, + id="array-history-clone-returns", + ), + pytest.param( + """suffix(value) => + pieces = str.split(value, "/") + pieces.size() > 1 ? pieces.get(1) : value +items = array.from("left/right", "whole") +for index = 0 to items.size() - 1 + value = items.get(index) + log.info(suffix(value)) +""", + "suffix", "std::string", "std::string value", 1, + id="variable-string-return", + ), + pytest.param( + """first(values) => values.get(0) +outer(values) => first(values) / values.size() +items = array.from(close) +plot(outer(items)) +""", + "first", "double", "std::vector& values", 1, + id="array-forwarding", + ), + pytest.param( + """pair() => + a = array.from(high) + b = array.from(low) + [a, b] +last(values) => + float result = na + for item in values + result := item + result +[upper, lower] = pair() +plot(last(upper) + last(lower)) +""", + "last", "double", "std::vector& values", 1, + id="tuple-array-argument", + ), + pytest.param( + """parts(value) => + [str.substring(value, 0, 1), str.substring(value, 1, 2)] +outer(value) => + [first, second] = parts(value) + first + second +value = input.string("ab") +log.info(outer(value)) +""", + "parts", "std::tuple", "std::string value", 1, + id="string-forwarding", + ), +] + + +@pytest.mark.parametrize("body,name,result,parameters,count", CASES) +def test_callable_boundaries_keep_the_value_type(body, name, result, parameters, count): + cpp = transpile('//@version=6\nstrategy("value type flow")\n' + body) + assert signatures(cpp, name) == [(result, parameters)] * count + compile_cpp(cpp, label=name) + + +def test_untyped_array_calls_keep_distinct_element_types(): + cpp = transpile("""//@version=6 +strategy("distinct array types") +trim(values, int maximum) => + if values.size() > maximum + values.shift() +integers = array.from(bar_index) +decimals = array.from(close) +trim(integers, 4) +trim(decimals, 4) +""") + found = signatures(cpp, "trim") + # Small, non-epoch int arrays retain int storage; widening is a separate + # provenance rule pinned by test_wide_int_arrays_and_literals.py. + assert sorted(parameters for _, parameters in found) == [ + "std::vector& values, int maximum", + "std::vector& values, int maximum", + ] + targets = re.findall( + r"^ (trim(?:_cs\d+|__ni\d+)?)\((integers|decimals), 4\);$", + cpp, re.MULTILINE, + ) + assert len(targets) == 2 and len({name for name, _ in targets}) == 2 + assert {argument for _, argument in targets} == {"integers", "decimals"} + bound_targets = {argument: name for name, argument in targets} + driver = r''' +#include +#include +#include + +template +void dump_array_values(const std::vector& values) { + std::cout << values.size(); + for (auto value : values) std::cout << ' ' << value; + std::cout << '\n'; +} + +int main() { + const Bar bars[] = { + Bar{9.0, 11.0, 8.0, 10.25, 10.0, 1743465600000LL}, + Bar{19.0, 21.0, 18.0, 20.5, 11.0, 1743466500000LL}, + Bar{33.0, 35.0, 32.0, 33.75, 12.0, 1743467400000LL}, + }; + GeneratedStrategy strategy; + strategy.run(bars, 3); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 2; + } + if (strategy.integers.size() != 1 || strategy.decimals.size() != 1) return 3; + std::cout << std::setprecision(17) + << strategy.integers.size() << ' ' << strategy.integers.front() + << ' ' << strategy.decimals.size() << ' ' << strategy.decimals.front() + << '\n'; + + // The unchanged Pine source creates singleton arrays. Exercise both + // emitted bodies with larger native arguments as well, including the + // untaken branch, one removal, and equality at the size boundary. + std::vector integer_values{-3, 0, 7, 21, 41}; + std::vector decimal_values{-3.25, 0.5, 7.75, 21.125, 41.875}; + for (int maximum : {5, 4, 4}) { + strategy.INTEGER_TARGET(integer_values, maximum); + strategy.DECIMAL_TARGET(decimal_values, maximum); + dump_array_values(integer_values); + dump_array_values(decimal_values); + } +} +'''.replace("INTEGER_TARGET", bound_targets["integers"]).replace( + "DECIMAL_TARGET", bound_targets["decimals"], + ) + output = run_emitted_tu( + cpp, driver, opt="-O2", label="distinct array argument types", + ) + rows = [[float(value) for value in line.split()] for line in output.splitlines()] + assert rows == [ + [1, 2, 1, 33.75], + [5, -3, 0, 7, 21, 41], + [5, -3.25, 0.5, 7.75, 21.125, 41.875], + [4, 0, 7, 21, 41], + [4, 0.5, 7.75, 21.125, 41.875], + [4, 0, 7, 21, 41], + [4, 0.5, 7.75, 21.125, 41.875], + ] + + +def test_reassigned_matrix_local_keeps_its_initializer_type(): + cpp = transpile("""//@version=6 +strategy("matrix local type") +nextValue(matrix source) => + result = matrix.new(1, 1, 0.0) + result.set(0, 0, source.get(0, 0) / 2) + result +var state = matrix.new(1, 1, 0.0) +state.set(0, 0, close) +if barstate.islast + working = matrix.copy(state) + for index = 1 to 2 + working := nextValue(working) + log.info(str.tostring(working.get(0, 0))) +""") + declarations = re.findall( + r"^\s+(PineMatrix|double) working(?: = [^\n;]*)?;$", + cpp, re.MULTILINE, + ) + assert declarations == ["PineMatrix"] + driver = r''' +#include +#include +#include +#include + +// The emitter's PineMatrix spelling denotes the actual numeric matrix handle. +static_assert(std::is_same_v< + decltype(std::declval().working), pineforge::NumericMatrix>); + +int main() { + const Bar bars[] = { + Bar{9.0, 11.0, 8.0, 10.25, 10.0, 1743465600000LL}, + Bar{19.0, 21.0, 18.0, 20.5, 11.0, 1743466500000LL}, + Bar{33.0, 35.0, 32.0, 33.75, 12.0, 1743467400000LL}, + }; + GeneratedStrategy strategy; + strategy.run(bars, 3); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 2; + } + std::cout << std::setprecision(17) + << strategy.state.rows() << ' ' << strategy.state.columns() + << ' ' << strategy.state.get(0, 0) + << ' ' << strategy.working.rows() << ' ' << strategy.working.columns() + << ' ' << strategy.working.get(0, 0) << '\n'; + strategy.working.set(0, 0, -17.5); + std::cout << strategy.state.get(0, 0) << ' ' << strategy.working.get(0, 0) << '\n'; + strategy.state.set(0, 0, 99.25); + std::cout << strategy.state.get(0, 0) << ' ' << strategy.working.get(0, 0) << '\n'; +} +''' + output = run_emitted_tu( + cpp, driver, opt="-O2", label="reassigned matrix local", + ) + rows = [[float(value) for value in line.split()] for line in output.splitlines()] + assert rows == [ + [1, 1, 33.75, 1, 1, 33.75 / 4], + [33.75, -17.5], + [99.25, -17.5], + ] + + +def test_nested_array_variants_keep_their_argument_and_return_types(): + cpp = transpile("""//@version=6 +strategy("nested array profiles") +identity(values) => values +outer(values) => identity(values) +integerInput = array.new() +decimalInput = array.new() +integerInput.push(7) +decimalInput.push(close) +integerOutput = outer(integerInput) +decimalOutput = outer(decimalInput) +plot(integerOutput.get(0) + decimalOutput.get(0)) +""") + for function in ("outer", "identity"): + found = signatures(cpp, function) + assert ("std::vector", "std::vector& values") in found + assert ("std::vector", "std::vector& values") in found + compile_cpp(cpp, label="nested structured type profiles")