Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ supported as exact pairs; on the 0.x line they are independent. See the
bytes: the timezone macro block and newly reserved names remain.
Regenerate C++ and relink affected strategies.

- User functions retain array, object and string result types through branch
returns and history clones. Untyped arguments retain collection element
types through nested calls and tuple bindings, and reassigned matrix locals
keep their initializer's matrix type. (#391)
- Deeply nested user-function calls share type-inference work within each
expression query, including when a return type remains unresolved.
- `array.sort_indices` returns integer indices, accepts typed `array<int>`
results and honors ascending/default or descending order in namespace and
method calls, including named arguments. Numeric `na` values sort last in
Expand Down
201 changes: 179 additions & 22 deletions pineforge_codegen/analyzer/base.py

Large diffs are not rendered by default.

33 changes: 32 additions & 1 deletion pineforge_codegen/analyzer/call_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1924,6 +1924,7 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType:
for i in range(len(param_specs)):
if param_specs[i] is None and i < len(arg_specs):
param_specs[i] = arg_specs[i]
self._callable_bound_param_specs_by_node[id(node)] = tuple(param_specs)
self._record_deferred_param_call_edge(
node,
func_name,
Expand Down Expand Up @@ -2171,13 +2172,43 @@ def _handle_user_func_call(self, func_name: str, node: FuncCall) -> PineType:
# scalar fallbacks. Propagate the concrete specs through the exact
# definition-time call edges and infer returns in post-order before
# codegen snapshots the FuncInfo table.
self._propagate_deferred_map_callable_specs(
self._propagate_deferred_callable_specs(
func_name,
{
name: spec
for name, spec in zip(func_def.params, param_specs)
},
)
value_specs = [*param_specs, fi.return_type_spec]
exact_return = (
self._callable_value_return_spec(
func_def, dict(zip(func_def.params, param_specs)),
)
if any(
spec is not None
and (spec.kind != "primitive" or spec.name == "string")
for spec in value_specs
)
else None
)
if exact_return is not None and (
exact_return.kind != "primitive" or exact_return.name == "string"
):
self._callable_bound_return_specs_by_node[id(node)] = exact_return
self._callable_return_specs_by_profile[
(func_name, tuple(param_specs))
] = exact_return
if (fi.return_type_spec is None
or fi.return_type_spec.kind == "primitive"):
fi.return_type_spec = exact_return
self._func_return_type_specs[func_name] = exact_return
if exact_return.kind == "primitive":
return_type = self._primitive_pine_type_from_spec(exact_return)
fi.return_type = return_type
self._func_return_types[func_name] = return_type
elif exact_return.kind == "udt":
fi.udt_return_type = exact_return.name
self._func_udt_return_types[func_name] = exact_return.name

if return_type == PineType.NA:
self._na_return_callables[id(node)] = func_def
Expand Down
6 changes: 6 additions & 0 deletions pineforge_codegen/analyzer/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,12 @@ class AnalyzerContext:
# helper called with both ``bar_index`` and ``close``).
func_callsite_param_types: dict = field(default_factory=dict)
func_callsite_return_types: dict = field(default_factory=dict)
# Exact argument/return value types retain collection element types, object
# identities and strings independently of the coarse primitive profiles.
func_callsite_param_specs: dict = field(default_factory=dict)
func_callsite_return_specs: dict = field(default_factory=dict)
func_value_call_edges: dict = field(default_factory=dict)
func_value_return_specs: dict = field(default_factory=dict)
# Declaration-time parameter specs, before any call-site inference fills
# untyped slots on the legacy FuncInfo record. Codegen uses this to apply
# variant overrides only to genuinely untyped parameters.
Expand Down
170 changes: 162 additions & 8 deletions pineforge_codegen/analyzer/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ def _selection_type_spec(
*,
nullable: bool | None = None,
) -> TypeSpec | None:
"""Keep exact collection joins; extend other families only around ``na``."""
"""A selection of identical value types retains that exact type."""
if nullable is None:
nullable = any(self._selection_contains_na(node) for node, _ in branches)
result: TypeSpec | None = None
Expand All @@ -307,8 +307,6 @@ def _selection_type_spec(
spec = TypeSpec.primitive("color")
if spec is None:
return None
if not nullable and spec.kind not in {"map", "matrix"}:
return None
result = (
spec
if result is None
Expand Down Expand Up @@ -522,6 +520,11 @@ def direct_user_udt_ctor_name(node: ASTNode) -> str | None:
))
return self._selection_type_spec(branches)
if isinstance(value, FuncCall):
exact = getattr(
self, "_callable_bound_return_specs_by_node", {}
).get(id(value))
if exact is not None:
return exact
cal = value.callee
if (isinstance(cal, Identifier) and cal.name in ("line", "box")
and len(value.args) == 1 and not value.kwargs
Expand Down Expand Up @@ -875,6 +878,27 @@ def _callsite_terminal_return_spec(
self._current_top_level_stmt = previous_owner
self._symbols.current_scope = previous_scope

def _callable_value_return_spec(self, func_def, parameter_specs):
"""Read a callable's value with its lexical locals and concrete inputs.

This query never revisits the analyzer or allocates stateful call sites.
Each branch gets its own environment, so an expired local cannot borrow
the type of a same-named global or another branch's declaration.
"""
previous_owner = self._current_top_level_stmt
previous_scope = self._symbols.current_scope
self._symbols.current_scope = self._operator_definition_scopes.get(
id(func_def), self._symbols.global_scope,
)
self._current_top_level_stmt = func_def
try:
return OperatorTypeQuery(self, dict(parameter_specs)).body_value_spec(
func_def.body
)
finally:
self._current_top_level_stmt = previous_owner
self._symbols.current_scope = previous_scope

def _terminal_map_selection_return_spec(
self,
value: ASTNode | None,
Expand Down Expand Up @@ -1440,14 +1464,121 @@ class OperatorTypeQuery(TypeHelper):
def __init__(self, analyzer, parameter_types):
self._analyzer = analyzer
self._parameter_types = parameter_types
self._type_query_memo = None

def __getattr__(self, name):
return getattr(self._analyzer, name)

def _memoized_type(self, node, kind, context, infer):
# Share successful and unresolved results across the value/operand
# fallback paths of one read-only query. Drop the entire memo when
# that query returns, before any declaration, scope or callable facts
# can change. A later query, even on this object, starts fresh.
memo = self._type_query_memo
root = memo is None
if root:
memo = self._type_query_memo = {}
try:
key = (kind, id(node), context)
if key not in memo:
if self._analyzer._budget is not None:
self._analyzer._budget.check(
getattr(node, "loc", None), Phase.ANALYZER,
)
memo[key] = infer()
return memo[key]
finally:
if root:
self._type_query_memo = None

def _operand_type(self, node):
return self._memoized_type(
node, "operand", None,
lambda: self._analyzer._operator_operand_type_uncached(
node, self._parameter_types, self,
),
)

def check_body(self, body):
for node in body:
self._check_node(node)

def body_value_spec(self, body):
"""Infer declaration and terminal value types without checking operators."""
result = None
for node in body or ():
if self._analyzer._budget is not None:
self._analyzer._budget.check(node.loc, Phase.ANALYZER)
if isinstance(node, VarDecl):
result = (
self._type_spec_from_hint(node.type_hint)
if node.type_hint else self._value_spec(node.value)
)
self._parameter_types[node.name] = result
elif isinstance(node, Assignment):
result = (
self._parameter_types.get(node.target.name)
if isinstance(node.target, Identifier)
else self._value_spec(node.target)
)
if result is None:
result = self._value_spec(node.value)
elif isinstance(node, TupleAssign):
# Tuple elements are independent value bindings.
specs = ()
if isinstance(node.value, FuncCall) and isinstance(
node.value.callee, Identifier
):
specs = getattr(
self._analyzer, "_func_tuple_element_specs", {}
).get(node.value.callee.name, ())
for index, name in enumerate(node.names):
self._parameter_types[name] = (
specs[index] if index < len(specs) else None
)
result = None
elif isinstance(node, IfStmt):
branches = [
OperatorTypeQuery(
self._analyzer, dict(self._parameter_types)
).body_value_spec(branch)
for branch in (node.body, node.else_body)
]
result = unify_type_specs(
*branches,
left_is_na=not node.body,
right_is_na=not node.else_body,
)
elif isinstance(node, SwitchStmt):
result = None
first = True
for branch in [
*(body for _, body in node.cases), node.default_body
]:
spec = OperatorTypeQuery(
self._analyzer, dict(self._parameter_types)
).body_value_spec(branch)
result = spec if first else unify_type_specs(
result, spec, right_is_na=not branch,
)
first = False
elif isinstance(node, ExprStmt):
result = self._value_spec(node.expr)
else:
result = self._value_spec(node)
return result

def _value_spec(self, value):
return self._memoized_type(
value, "value", None, lambda: self._value_spec_uncached(value),
)

def _value_spec_uncached(self, value):
spec = self._type_spec_from_expr(value)
if spec is None:
spec = self._operand_type(value)
return spec

def _check_node(self, node):
if not isinstance(node, ASTNode):
if isinstance(node, (tuple, list)):
Expand Down Expand Up @@ -1539,17 +1670,40 @@ def _check_node(self, node):
return
self._analyzer._check_operator_operands(node, op, operands, self._parameter_types)

def _type_spec_from_expr(self, value, **kwargs):
def _type_spec_from_expr(self, value, *, _ternary_arms=None):
return self._memoized_type(
value, "spec", _ternary_arms,
lambda: self._type_spec_from_expr_uncached(
value, _ternary_arms=_ternary_arms,
),
)

def _type_spec_from_expr_uncached(self, value, **kwargs):
if isinstance(value, Identifier):
return self._analyzer._operator_operand_type(
value, self._parameter_types,
return self._operand_type(value)
if (isinstance(value, FuncCall)
and isinstance(value.callee, Identifier)
and value.callee.name in self._analyzer._func_defs):
name = value.callee.name
definition = self._analyzer._func_defs[name]
args = self._analyzer._bind_callable_args(
value, list(definition.params)
)
profile = tuple(
self._value_spec(arg) if arg is not None else None
for arg in args
)
exact = self._analyzer._callable_return_specs_by_profile.get(
(name, profile)
)
if exact is not None:
return exact
return super()._type_spec_from_expr(value, **kwargs)

def _visit(self, node):
# TypeHelper's array.from fallback normally visits its arguments.
# Here it asks for facts only, so no TA or request is registered again.
spec = self._analyzer._operator_operand_type(node, self._parameter_types)
spec = self._operand_type(node)
if spec is not None and spec.kind == "primitive":
return next((ptype for ptype in PineType if ptype.value == spec.name),
PineType.UNKNOWN)
Expand All @@ -1558,7 +1712,7 @@ def _visit(self, node):
def _array_from_element_spec(self, value):
# Array construction must not recover a coarse symbol/call result
# after the operator query deliberately left its source unresolved.
return self._analyzer._operator_operand_type(value, self._parameter_types)
return self._operand_type(value)

@staticmethod
def _pine_type_to_spec(pine_type):
Expand Down
35 changes: 34 additions & 1 deletion pineforge_codegen/codegen/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2145,6 +2145,26 @@ def natural_name(fname: str, cs_idx: int) -> str:
owns_non_ta_state = bool(
g_var_originals or fixnan_originals(g_name)
)
owner_info = self._func_info_map.get(inst["fname"])
owner_profile = inst.get("param_specs")
if owner_profile is None and owner_info is not None:
owner_profile = getattr(
ctx, "func_callsite_param_specs", {}
).get(
(inst["fname"], inst.get("type_call_site_idx",
inst.get("call_site_idx"))),
tuple(owner_info.param_type_specs),
)
incoming = getattr(ctx, "func_value_call_edges", {}).get(
(inst["fname"], owner_profile, id(callnode))
)
natural_profile = getattr(
ctx, "func_callsite_param_specs", {}
).get((g_name, j))
different_value_type = (
incoming is not None and natural_profile is not None
and incoming != natural_profile
)
# Every non-cs0 enclosing variant is a distinct Pine written
# call path. Even a pure wrapper can reach var/fixnan state
# farther down the graph, so preserve the path identity here
Expand All @@ -2155,6 +2175,7 @@ def natural_name(fname: str, cs_idx: int) -> str:
)
if (
not path_requires_fresh_state
and not different_value_type
and composed_ta == natural_ta
and composed_var == natural_var
and composed_fixnan == natural_fixnan
Expand All @@ -2174,6 +2195,7 @@ def natural_name(fname: str, cs_idx: int) -> str:
frozenset(composed_var.items()),
frozenset(composed_fixnan.items()),
path_identity,
incoming,
)
ginst = interned.get(key)
if ginst is None:
Expand Down Expand Up @@ -2210,6 +2232,10 @@ def natural_name(fname: str, cs_idx: int) -> str:
# takes the argument that call passes, as the
# call's own cs{j} clone does.
"type_call_site_idx": j,
"param_specs": incoming,
"return_spec": getattr(
ctx, "func_value_return_specs", {}
).get((g_name, incoming)),
"ta_remap": composed_ta,
"var_remap": fvar_remap,
"fixnan_remap": ffixnan_remap,
Expand Down Expand Up @@ -2525,6 +2551,7 @@ def _register_global_aggregate_member_types(self) -> None:
# final state and can already contain an invalid later reassignment's
# element type. The original declaration must remain the compatibility
# baseline used to reject matrix<int> := matrix<float>.
gem = getattr(self.ctx, "global_expr_map", {}) or {}
for stmt, _in_loop in self._walk_global_scope_with_loopflag(
getattr(self.ctx.ast, "body", []), False
):
Expand All @@ -2546,7 +2573,14 @@ def _register_global_aggregate_member_types(self) -> None:
self._array_vars.add(stmt.name)
self._collection_types.setdefault(stmt.name, declared_spec)
continue
elif stmt.name not in gem:
# Reassignment removes a binding from global_expr_map. Its
# initializer still establishes its matrix handle type, also
# for declarations inside top-level control-flow blocks.
declared_spec = self._type_spec_from_expr(stmt.value)
else:
# An available initializer still uses the established member
# inference and default initialization below.
continue
if (from_selection and declared_spec is not None
and declared_spec.kind == "array"):
Expand All @@ -2562,7 +2596,6 @@ def _register_global_aggregate_member_types(self) -> None:
self._matrix_specs.setdefault(stmt.name, declared_spec)
self._collection_types[stmt.name] = self._matrix_specs[stmt.name]

gem = getattr(self.ctx, "global_expr_map", {}) or {}
for name, _ptype in self.ctx.global_var_decls:
expr = gem.get(name)
if expr is None or not isinstance(expr, FuncCall):
Expand Down
Loading
Loading