From da4f908d22de7580db066bd3bff10cd15d8b3061 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Mon, 16 Nov 2026 04:51:06 +0800 Subject: [PATCH 1/2] Preserve requested-context scalar copy history Track mutable dependencies at each request and replay scalar copies at their original declaration boundary. Resolve helper reads by lexical binding and recover global dependencies from declaration initializers when a same-named parameter has history. Integer-tail cases have moved to a separate follow-up. Fixes #393 --- CHANGELOG.md | 5 + pineforge_codegen/analyzer/base.py | 167 +++++++++- pineforge_codegen/analyzer/call_handlers.py | 9 +- pineforge_codegen/codegen/security.py | 67 ++-- tests/test_security_value_copy_boundaries.py | 302 +++++++++++++++++++ tests/test_security_var_alias_history.py | 261 ++++++++++++++++ 6 files changed, 776 insertions(+), 35 deletions(-) create mode 100644 tests/test_security_value_copy_boundaries.py create mode 100644 tests/test_security_var_alias_history.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcc1e7d..617f10e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,11 @@ supported as exact pairs; on the 0.x line they are independent. See the method calls, including named arguments. Numeric `na` values sort last in ascending order and first in descending order; string arrays use lexical ordering. The source array is unchanged. (#392) +- `request.security` keeps the requested-bar history and declaration-time + value of top-level scalar copies of persistent or reassigned globals, + including copy chains. A declared float copy preserves an integer source's + missing value, and replayed updates use the requested `bar_index`. + See [#393](https://github.com/pineforge-4pass/pineforge-codegen-oss/issues/393). ## 1.12.0 — 2026-11-12 diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index ae29864b..f08d631b 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -7,6 +7,7 @@ from __future__ import annotations +from dataclasses import replace from typing import Any from ..ast_nodes import ( @@ -36,7 +37,7 @@ from ..input_keys import assign_input_keys from ..loop_types import FLOAT_REASSIGNMENT, LoopTypeSources, for_counter_type from ..security_contexts import ( - PASS_WARNINGS_ANNOTATION, context_key, reads_bar_series, + GLOBAL_ANNOTATION, PASS_WARNINGS_ANNOTATION, context_key, reads_bar_series, ) from ..session_reads import emitted_session_reads from ..collection_history import ( @@ -182,6 +183,8 @@ def __init__(self, ast: Program, filename: str = "", self._session_unresolved_method_calls: dict[int, set[str]] = {} self._symbols = SymbolTable() self._ta_call_sites: list[TACallSite] = [] + # Final analysis consumes this inventory even when no request was seen. + self._security_calls: list[SecurityCallInfo] = [] self._series_vars: set[str] = set() self._series_var_members: set[str] = set() self._series_decl_nodes: set[int] = set() @@ -731,6 +734,7 @@ def _analyze(self) -> AnalyzerContext: for name, info in self._global_binding_infos.items() if info.is_var or name in self._global_reassigned_names } + self._retain_security_value_copies(mutable_global_infos) pure_global_expr_map = { k: v for k, v in self._global_expr_map.items() if k not in mutable_global_infos @@ -2467,9 +2471,15 @@ def _collect_security_mutable_globals( out: set[str] = set() if isinstance(node, Identifier): + scope = self._identifier_binding_scopes.get(id(node)) + if scope != "global" and not ( + scope is None and (node.annotations or {}).get(GLOBAL_ANNOTATION) + ): + # Parameters and locals do not read a same-spelled global. + return out name = node.name - if name in self._global_binding_infos: - info = self._global_binding_infos[name] + info = self._global_binding_infos.get(name) + if info is not None: if info.is_var or name in self._global_reassigned_names: out.add(name) if name in resolving: @@ -2479,9 +2489,15 @@ def _collect_security_mutable_globals( out |= self._collect_security_mutable_globals(stmt, resolving) resolving.remove(name) return out - if name in self._global_expr_map and name not in resolving: + value = self._global_expr_map.get(name) + if value is None and info is not None and isinstance(info.decl_node, VarDecl): + # The legacy series-name set can omit this global from the + # expression map after a helper parameter reads its history. + # Its declaration still records the call-time dependency. + value = info.decl_node.value + if value is not None and name not in resolving: resolving.add(name) - out |= self._collect_security_mutable_globals(self._global_expr_map[name], resolving) + out |= self._collect_security_mutable_globals(value, resolving) resolving.remove(name) return out @@ -2530,6 +2546,147 @@ def _collect_security_mutable_globals( return out + def _retain_security_value_copies( + self, mutable_infos: dict[str, MutableGlobalInfo], + ) -> None: + """Replay scalar copies whose value cannot be substituted in a request. + + A history read needs the copy's own requested-bar samples. A copy made + before a later store to its source also needs its declaration replayed + at its original position, even without history. Chart storage and + helper-local bindings keep their independently analyzed identities. + """ + if not self._security_calls or not mutable_infos: + return + budget = getattr(self, "_budget", None) + positions = {id(stmt): index for index, stmt in enumerate(self._ast.body)} + + def is_global(node): + scope = self._identifier_binding_scopes.get(id(node)) + annotations = getattr(node, "annotations", None) or {} + return scope == "global" or ( + scope is None and bool(annotations.get(GLOBAL_ANNOTATION)) + ) + + aliases = {} + last_write = {} + primitive = {PineType.FLOAT, PineType.INT, PineType.BOOL, + PineType.STRING, PineType.COLOR} + for name, info in self._global_binding_infos.items(): + if budget is not None: + budget.check(phase=Phase.ANALYZER) + last_write[name] = max( + (positions.get(id(stmt), -1) for stmt in info.source_stmts), + default=-1, + ) + decl = info.decl_node + if (name in mutable_infos or info.pine_type not in primitive + or not isinstance(decl, VarDecl) or id(decl) not in positions + or not isinstance(decl.value, Identifier) or not is_global(decl.value)): + continue + source = self._global_binding_infos.get(decl.value.name) + if source is None or source.pine_type not in primitive: + continue + binding = self._identifier_decl_bindings.get(id(decl.value)) + if (binding is not None and source.decl_node is not None + and binding[0] != id(source.decl_node)): + continue + aliases[name] = source.name + + # Memoize chains rather than walking once per path through a diamond. + rooted = dict.fromkeys(mutable_infos, True) + for name in aliases: + path, seen = [], set() + current = name + while current in aliases and current not in rooted and current not in seen: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + seen.add(current) + path.append(current) + current = aliases[current] + reaches_state = rooted.get(current, False) + for alias in path: + rooted[alias] = reaches_state + aliases = {name: source for name, source in aliases.items() if rooted[name]} + if not aliases: + return + snapshots = { + name for name, source in aliases.items() + if last_write[source] > positions[id(self._global_binding_infos[name].decl_node)] + } + + def dependencies(root): + pending, seen, names, history = [root], set(), set(), set() + while pending: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + node = pending.pop() + if not isinstance(node, ASTNode) or id(node) in seen: + continue + seen.add(id(node)) + if (isinstance(node, Subscript) and isinstance(node.object, Identifier) + and is_global(node.object) and node.object.name in aliases): + history.add(node.object.name) + if isinstance(node, Identifier) and is_global(node): + name = node.name + if name in aliases or name in mutable_infos: + names.add(name) + pending.extend(self._global_binding_infos[name].source_stmts) + elif name in self._global_expr_map: + pending.append(self._global_expr_map[name]) + if isinstance(node, FuncCall): + if self._is_input_func_call(node): + # Cosmetic input arguments are not data dependencies. + pending.append(node.args[0] if node.args else node.kwargs.get("defval")) + continue + if isinstance(node.callee, Identifier): + function = self._func_defs.get(node.callee.name) + if function is not None: + pending.extend(function.body) + pending.extend(syntax_children(node)) + return names, history + + required, histories = set(), set() + requests = [] + for call in self._security_calls: + names, history = dependencies(call.expression) + histories.update(history) + required.update(names & snapshots) + requests.append((call, names)) + required.update(histories) + if not required: + return + for name in sorted(required): + info = self._global_binding_infos[name] + mutable_infos[name] = replace(info, is_series=info.is_series or name in histories) + sources = {} + for name in sorted(required): + chain = set() + source = name + while source in aliases and source not in chain: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + chain.add(source) + source = aliases[source] + chain.add(source) + # The original call-time inventory already bounds which inputs + # of a source's updates are replayed. Retaining a copy must not + # widen that boundary to writes discovered later in the script. + sources[name] = chain & mutable_infos.keys() + for call, names in requests: + # Preserve each request's original replay boundary. A helper read + # elsewhere in its payload must not pull later chart-only writes + # into the request merely because another operand copies state. + original = set(getattr(call, "mutable_globals", ()) or ()) + additions = set() + for name in names & required: + # Newly retained aliases need their own replay. A pre-existing + # mutable source participates only if this call already read + # it as mutable; its first later write is not retroactive. + additions.update(sources[name] & (original | required)) + call.mutable_globals = tuple(sorted(original | additions)) + call.depends_on_mutable_globals = bool(call.mutable_globals) + def _session_history_unsafe_functions( self, func_defs: dict, find_calls, known_func_names: set[str], calls_by_parent: dict, call_edges: list) -> dict[str, str]: diff --git a/pineforge_codegen/analyzer/call_handlers.py b/pineforge_codegen/analyzer/call_handlers.py index 4635541b..f8a8b3df 100644 --- a/pineforge_codegen/analyzer/call_handlers.py +++ b/pineforge_codegen/analyzer/call_handlers.py @@ -18,10 +18,9 @@ index used to mint unique ``_ta__`` member names. - ``self._series_bar_fields`` (``set[str]``): bar-field identifiers used as TA inputs anywhere in the program. -- ``self._security_calls`` (``list[SecurityCallInfo]``): created on - first ``request.security(...)`` via ``getattr(..., "_security_calls", - [])`` and stored back on ``self`` -- the analyzer also reads this - attribute via ``getattr`` in ``analyze()``. +- ``self._security_calls`` (``list[SecurityCallInfo]``): a per-analyzer + inventory initialized empty by the host constructor. Both security + handlers append to it; final analysis also consumes the empty inventory. - ``self._fixnan_counter`` / ``self._fixnan_sites``: ``fixnan(...)`` member-name counter + site list. - ``self._symbols`` (``SymbolTable``): consulted by @@ -859,7 +858,6 @@ def _handle_request_call(self, func_name: str, node: FuncCall) -> PineType: security_ta_range = (ta_start, ta_end) if ta_end > ta_start else None # Assign ID and record the call - self._security_calls = getattr(self, "_security_calls", []) sec_id = len(self._security_calls) returns_tuple = isinstance(expr_node, TupleLiteral) @@ -1095,7 +1093,6 @@ def _handle_request_security_lower_tf(self, node: FuncCall) -> PineType: node.loc, ) - self._security_calls = getattr(self, "_security_calls", []) sec_id = len(self._security_calls) mutable_globals = tuple(sorted(self._collect_security_mutable_globals(expr_node))) diff --git a/pineforge_codegen/codegen/security.py b/pineforge_codegen/codegen/security.py index 297e3bdc..e20c8291 100644 --- a/pineforge_codegen/codegen/security.py +++ b/pineforge_codegen/codegen/security.py @@ -4830,7 +4830,8 @@ def _security_expr_hist_series_names(self, sec_id: int) -> list[str]: def _emit_security_expr_hist_members( self, sec_id: int, expr_node, lines: list[str], mbb_suffix: str ) -> None: - if self._security_reads_bar_index(expr_node): + mutable_names = self._security_eval_info[sec_id].get("mutable_globals") or () + if self._security_reads_bar_index(expr_node, mutable_names): # The requested bar's ``bar_index``: one count per requested bar, # advanced where the evaluator opens its slot. self._security_bar_index_secs.add(sec_id) @@ -7878,6 +7879,11 @@ def _emit_security_rebind_var_decl( emitted_lines=emitted_lines, ) expr_cpp = self._security_copy_store_cpp(node.name, info, node.value, expr_cpp) + cpp_type = self._security_cpp_type_for_mutable(node.name, info) + if node.value is None or isinstance(node.value, NaLiteral) or cpp_type == "double": + # A replayed declaration stores into its declared destination: + # bare na has that type, and a float copy preserves integer na. + expr_cpp = self._security_local_value(expr_cpp, node.value, cpp_type, ()) if getattr(info, "is_var", False): if getattr(info, "is_series", False): @@ -11697,35 +11703,48 @@ def _security_is_bar_index(self, node) -> bool: return (isinstance(node, Identifier) and node.name == "bar_index" and "bar_index" not in getattr(self, "_safe_name_bound", ())) - def _security_reads_bar_index(self, expr_node) -> bool: + def _security_reads_bar_index(self, expr_node, mutable_names=()) -> bool: """Whether a payload reads ``bar_index``: in itself, the globals it - reads or the user functions it calls. TradingView evaluates it on - the requested bar -- the count of requested bars before it -- where - the evaluator used to read the chart's (``pine_bar_index()``).""" - from ..limits import iter_ast_nodes - + reads, replayed state or the user functions it calls. Follow only + this request's mutable inventory, so retaining a copy does not + discover updates outside its original source boundary.""" global_expr_map = getattr(self.ctx, "global_expr_map", {}) or {} + mutable_infos = getattr(self, "_global_mutable_infos", None) or {} + mutable_names = set(mutable_names or ()) + func_names = getattr(self, "_func_names", None) or () + func_infos = getattr(self, "_func_info_map", None) or {} + budget = getattr(self, "_budget", None) seen: set[tuple[str, str]] = set() + visited: set[int] = set() pending = [expr_node] while pending: - root = pending.pop() - if not isinstance(root, ASTNode): + node = pending.pop() + if not isinstance(node, ASTNode) or id(node) in visited: continue - for n, _depth in iter_ast_nodes(root): - if self._security_is_bar_index(n): - return True - if (isinstance(n, Identifier) and n.name in global_expr_map - and ("global", n.name) not in seen - and self._security_identifier_is_global_binding(n)): - seen.add(("global", n.name)) - pending.append(global_expr_map[n.name]) - if (isinstance(n, FuncCall) and isinstance(n.callee, Identifier) - and n.callee.name in self._func_names - and ("func", n.callee.name) not in seen): - seen.add(("func", n.callee.name)) - info = self._func_info_map.get(n.callee.name) - if info is not None and getattr(info, "node", None) is not None: - pending.extend(info.node.body) + visited.add(id(node)) + if budget is not None: + budget.check(getattr(node, "loc", None), Phase.CODEGEN) + if self._security_is_bar_index(node): + return True + if (isinstance(node, Identifier) + and self._security_identifier_is_global_binding(node)): + key = ("global", node.name) + if key not in seen: + seen.add(key) + if node.name in mutable_names: + info = mutable_infos.get(node.name) + pending.extend(getattr(info, "source_stmts", None) or ()) + elif node.name in global_expr_map: + pending.append(global_expr_map[node.name]) + if (isinstance(node, FuncCall) and isinstance(node.callee, Identifier) + and node.callee.name in func_names + and ("func", node.callee.name) not in seen): + seen.add(("func", node.callee.name)) + info = func_infos.get(node.callee.name) + function = getattr(info, "node", None) + if function is not None: + pending.extend(function.body or ()) + pending.extend(syntax_children(node)) return False def _security_variant_order_key(self, signature: tuple, binding_stack) -> tuple: diff --git a/tests/test_security_value_copy_boundaries.py b/tests/test_security_value_copy_boundaries.py new file mode 100644 index 00000000..f70dbc9c --- /dev/null +++ b/tests/test_security_value_copy_boundaries.py @@ -0,0 +1,302 @@ +"""Requested copies retain missing values and their original replay boundary.""" + +from __future__ import annotations + +import math + +import pytest + +from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser +from tests._compile import run_emitted_tu, skip_if_no_engine_lib +from tests._e2e import reference_codegen, transpile_json +from tests._numeric_expected import integer_to_double +from tests.test_security_helper_local_var import _eval_body +from tests.test_security_var_alias_history import HOUR_MS, REPORTER, chart_bars + + +MISSING_INTEGER_COPY = """//@version=6 +strategy("requested scalar value copies") +var int cnt = na +float x = cnt +if close > open + cnt := bar_index +htf = request.security(syminfo.tickerid, "240", x) +""" + +LATE_LOOP_UPDATE = """//@version=6 +strategy("requested scalar value copies") +var float acc = 0.0 +accAlias = acc +acc += close - open +float level = close +getLevel() => level +htf = request.security(syminfo.tickerid, "240", accAlias + getLevel()) +for i = 0 to 1 + level += 1 +plot(htf) +""" + +LATE_SOURCE_INPUT_UPDATE = """//@version=6 +strategy("t") +var float acc = 0.0 +float level = close +accAlias = acc +acc += level +htf = request.security(syminfo.tickerid, "240", accAlias) +for i = 0 to 1 + level += 1 +plot(htf) +""" + + +LATE_COPIED_SOURCE_UPDATE = """//@version=6 +strategy("t") +float acc = close +accAlias = acc +htf = request.security(syminfo.tickerid, "240", accAlias) +for i = 0 to 1 + acc += 1 +plot(htf) +""" + + +def test_copy_capture_adds_only_its_sources_to_requests_that_read_it(): + source = LATE_LOOP_UPDATE.replace( + "for i = 0 to 1", + 'other = request.security(syminfo.tickerid, "60", getLevel())\n' + "for i = 0 to 1", + ) + program = Parser(Lexer(source).tokenize(), source=source).parse() + context = Analyzer(program).analyze() + assert len(context.security_calls) == 2 + assert context.security_calls[0].mutable_globals == ("acc", "accAlias") + assert context.security_calls[1].mutable_globals == () + + +def test_copy_capture_preserves_the_call_time_boundary_of_source_updates(): + program = Parser( + Lexer(LATE_SOURCE_INPUT_UPDATE).tokenize(), source=LATE_SOURCE_INPUT_UPDATE, + ).parse() + context = Analyzer(program).analyze() + assert len(context.security_calls) == 1 + assert context.security_calls[0].mutable_globals == ("acc", "accAlias") + + +@pytest.mark.parametrize("chain", [False, True], ids=["copy", "chain"]) +def test_copy_capture_does_not_add_a_source_first_reassigned_after_the_call(chain): + source = LATE_COPIED_SOURCE_UPDATE + if chain: + source = source.replace("accAlias = acc\n", "accAlias = acc\nnextAlias = accAlias\n") + source = source.replace('"240", accAlias)', '"240", nextAlias)') + program = Parser(Lexer(source).tokenize(), source=source).parse() + context = Analyzer(program, budget=None).analyze() + # The last, unchanged link can expand to the retained earlier copy. + assert context.security_calls[0].mutable_globals == ("accAlias",) + body = _eval_body(transpile(source), 0, None) + assert "_sec0_acc " not in body + assert "_sec0_accAlias = acc;" in body + + +@pytest.mark.parametrize("assignment, integer_type", [ + ("bar_index", "int"), + ("int(close)", "int64_t"), +]) +def test_requested_float_copy_emits_defined_integer_na_and_float_conversion( + assignment, integer_type, +): + source = MISSING_INTEGER_COPY.replace("cnt := bar_index", f"cnt := {assignment}") + cpp = transpile(source) + body = _eval_body(cpp, 0, None) + assert f"_sec0_cnt = na<{integer_type}>();" in body + assert f"_sec0_x = {integer_to_double('_sec0_cnt')};" in body + + +def test_requested_state_updates_discover_the_requested_bar_index(): + cpp = transpile(MISSING_INTEGER_COPY) + body = _eval_body(cpp, 0, None) + assert "int _sec0_bar_index_ = -1;" in cpp + assert "if (security_series_slot_is_new(0)) ++_sec0_bar_index_;" in body + assert "_sec0_cnt = _sec0_bar_index_;" in body + assert "pine_bar_index()" not in body + + +def test_requested_counter_discovery_keeps_later_chart_only_updates_outside(): + source = LATE_COPIED_SOURCE_UPDATE.replace("acc += 1", "acc += bar_index") + cpp = transpile(source) + assert "_sec0_bar_index_" not in cpp + assert "_sec0_accAlias = acc;" in _eval_body(cpp, 0, None) + + +def test_requested_alias_history_emits_its_series_replay_and_read(): + cpp = transpile(REPORTER) + assert "Series _sec0_accAlias;" in cpp + body = _eval_body(cpp, 0, None) + replay = ( + " if (security_series_slot_is_new(0)) {\n" + " _sec0_accAlias.push(_sec0_acc);\n" + " } else {\n" + " _sec0_accAlias.update(_sec0_acc);\n" + " }\n" + ) + assert body.count(replay) == 1 + assert body.count("_req_sec_0 = (_sec0_accAlias[0] - _sec0_accAlias[1]);") == 1 + + +def observe_requested(source, *, cpp=None, bars=None, opt="-O0"): + bars = list(chart_bars()) if bars is None else bars + literals = ",\n".join( + " Bar{" + ", ".join(repr(value) for value in bar) + "}" + for bar in bars + ) + driver = r""" +#include +#include +struct ValueCopyObserver : GeneratedStrategy { + void evaluate_security(int sec_id, const Bar& bar, bool complete) override { + GeneratedStrategy::evaluate_security(sec_id, bar, complete); + if (sec_id == 0) + std::cout << bar.timestamp << ' ' << complete << ' ' + << _req_sec_0 << '\n'; + } +}; +int main() { + std::cout << std::setprecision(17); + ValueCopyObserver strategy; + Bar bars[] = { +BARS + }; + strategy.run(bars, COUNT, "60", "60"); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 1; + } + return 0; +} +""".replace("BARS", literals).replace("COUNT", str(len(bars))) + output = run_emitted_tu( + transpile(source) if cpp is None else cpp, + driver, opt=opt, label="requested scalar value copy", + ) + print(output, end="") + rows = [] + for line in output.splitlines(): + parts = line.split() + assert len(parts) == 3 and parts[1] in {"0", "1"}, line + rows.append((int(parts[0]), parts[1] == "1", float(parts[2]))) + completed = [(timestamp, value) for timestamp, complete, value in rows if complete] + assert [timestamp for timestamp, _ in completed] == [ + bar[-1] for bar in bars[::4] + ] + return bars, completed + + +def missing_integer_bars(): + # Begin with an unassigned requested candle, then alternate assignments + # and held values so missingness and finite copy chronology are distinct. + bars = [] + open_ = 100.0 + closes = (99.0, 98.0, 97.0, 96.0, 97.0, 98.0, 99.0, 100.0, + 99.0, 98.0, 97.0, 96.0, 97.0, 98.0, 99.0, 100.0) + for index, close in enumerate(closes): + bars.append((open_, max(open_, close), min(open_, close), close, + float(index + 1), index * HOUR_MS)) + open_ = close + return bars + + +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_requested_float_copy_preserves_integer_na_before_later_assignment(opt): + skip_if_no_engine_lib() + bars = missing_integer_bars() + _, completed = observe_requested(MISSING_INTEGER_COPY, bars=bars, opt=opt) + previous = math.nan + for index, (timestamp, observed) in enumerate(completed): + # The copy precedes this candle's possible store to the integer. + if math.isnan(previous): + assert math.isnan(observed), (timestamp, observed) + else: + assert observed == previous, (timestamp, observed, previous) + group = bars[index * 4:(index + 1) * 4] + if group[-1][3] > group[0][0]: + previous = float(index) + + +@pytest.mark.parametrize("source, source_reads_level", [ + pytest.param(LATE_LOOP_UPDATE, False, id="helper-reads-chart"), + pytest.param(LATE_SOURCE_INPUT_UPDATE, True, id="source-update-reads-chart"), +]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_later_loop_update_keeps_its_existing_chart_value_boundary( + tmp_path, source, source_reads_level, opt, +): + skip_if_no_engine_lib() + base = reference_codegen("ecbbfe392f993e8ce6d4cf12067f44b1b95dcf51") + assert base is not None, "reference commit is not in this checkout's history" + pine = tmp_path / "value-copy.pine" + pine.write_text(source) + result = transpile_json(pine, root=base) + assert result["ok"], result.get("diagnostics") + _, base_completed = observe_requested( + source, cpp=result["cpp"], opt=opt, + ) + bars, completed = observe_requested(source, opt=opt) + accumulator = 0.0 + assert [timestamp for timestamp, _ in base_completed] == [ + timestamp for timestamp, _ in completed + ] + for index, ((timestamp, observed), (_, base_value)) in enumerate( + zip(completed, base_completed) + ): + # The mutable source is replayed in the request. The separate helper + # retains the chart value from the preceding chart evaluation, before + # this requested close's on_source_bar runs. + group = bars[index * 4:(index + 1) * 4] + preceding_chart = bars[index * 4 + 2] + chart_value = preceding_chart[3] + 2.0 + delta = chart_value if source_reads_level else group[-1][3] - group[0][0] + chart_operand = 0.0 if source_reads_level else chart_value + expected = accumulator + chart_operand + assert timestamp == index * 4 * HOUR_MS + assert observed == expected, (timestamp, observed, expected) + # Base expanded the copy after its source's update. Preserve its + # separate chart contribution while correcting the copy chronology. + assert base_value == accumulator + delta + chart_operand + accumulator += delta + + +def assert_requested_values(rows, expected): + assert len(rows) == len(expected) + for (timestamp, actual), wanted in zip(rows, expected): + if math.isnan(wanted): + assert math.isnan(actual), (timestamp, actual, wanted) + else: + assert actual == pytest.approx(wanted), (timestamp, actual, wanted) + + +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +@pytest.mark.parametrize("chain", [False, True], ids=["copy", "chain"]) +def test_late_copied_source_loop_keeps_the_original_boundary(tmp_path, chain, opt): + skip_if_no_engine_lib() + source = LATE_COPIED_SOURCE_UPDATE + if chain: + source = source.replace("accAlias = acc\n", "accAlias = acc\nnextAlias = accAlias\n") + source = source.replace('"240", accAlias)', '"240", nextAlias)') + base = reference_codegen("ecbbfe392f993e8ce6d4cf12067f44b1b95dcf51") + assert base is not None, "reference commit is not in this checkout's history" + pine = tmp_path / "late-source-copy.pine" + pine.write_text(source) + result = transpile_json(pine, root=base) + assert result["ok"], result.get("diagnostics") + bars, observed = observe_requested(source, opt=opt) + _, previous = observe_requested(source, cpp=result["cpp"], bars=bars, opt=opt) + control = source.replace('"240", nextAlias)', '"240", acc)').replace( + '"240", accAlias)', '"240", acc)', + ) + _, direct = observe_requested(control, bars=bars, opt=opt) + expected = [bars[start + 2][3] + 2.0 for start in range(0, len(bars), 4)] + assert_requested_values(observed, expected) + assert_requested_values(previous, expected) + assert_requested_values(direct, expected) diff --git a/tests/test_security_var_alias_history.py b/tests/test_security_var_alias_history.py new file mode 100644 index 00000000..7d35bd57 --- /dev/null +++ b/tests/test_security_var_alias_history.py @@ -0,0 +1,261 @@ +"""A copied accumulator's history belongs to the requested clock. + +The reporter strategy is complete and unchanged. Variants change only the +binding or call form under test. Native observations use the repository's +compile-and-run harness and real engine aggregation with changing chart-candle +directions. The arithmetic oracle uses the input candles, not emitted C++. +""" + +from __future__ import annotations + +import math + +import pytest + +from pineforge_codegen import transpile +from tests._compile import run_emitted_tu, skip_if_no_engine_lib + + +REPORTER = """//@version=6 +strategy("security history of a var alias", overlay = true, initial_capital = 10000) +fast = ta.ema(close, 10) +slow = ta.ema(close, 30) +var float acc = 0.0 +acc += close > open ? volume : -volume +accAlias = acc +impulse = accAlias - accAlias[1] +htf = request.security(syminfo.tickerid, "240", impulse) +plot(htf) +if ta.crossover(fast, slow) + strategy.entry("Long", strategy.long, qty = 0.01) +if ta.crossunder(fast, slow) + strategy.close("Long") +""" + +DIRECT = REPORTER.replace( + "accAlias = acc\nimpulse = accAlias - accAlias[1]\n", + "impulse = acc - acc[1]\n", +) +OUTSIDE = REPORTER.replace( + 'htf = request.security(syminfo.tickerid, "240", impulse)\nplot(htf)', + "plot(accAlias - accAlias[1])", +) +TWO_ALIASES = REPORTER.replace( + "impulse = accAlias - accAlias[1]", + "secondAlias = accAlias\nimpulse = secondAlias - secondAlias[1]", +) +THREE_ALIASES = TWO_ALIASES.replace( + "impulse = secondAlias - secondAlias[1]", + "thirdAlias = secondAlias\nimpulse = thirdAlias - thirdAlias[1]", +) +LOCAL_SHADOW = REPORTER.replace( + "fast = ta.ema", + "shadow() =>\n" + " accAlias = close * 2.0\n" + " accAlias - accAlias[1]\n" + "fast = ta.ema", +).replace('"240", impulse)', '"240", impulse + shadow())') +PARAMETER_SHADOW = REPORTER.replace( + "fast = ta.ema", + "shadow(float accAlias) => accAlias - accAlias[1]\nfast = ta.ema", +).replace('"240", impulse)', '"240", impulse + shadow(close))') +NO_HISTORY = REPORTER.replace( + "impulse = accAlias - accAlias[1]", "impulse = accAlias" +) +INPUT_UPDATE = REPORTER.replace( + "var float acc", 'stepScale = input.float(1.75, "Step scale")\nvar float acc' +).replace( + "acc += close > open ? volume : -volume", + "acc += (close > open ? volume : -volume) * stepScale", +) +NAMED = REPORTER.replace( + 'request.security(syminfo.tickerid, "240", impulse)', + 'request.security(expression = impulse, timeframe = "240", symbol = syminfo.tickerid)', +) +DIRECT_PAYLOAD = REPORTER.replace( + 'request.security(syminfo.tickerid, "240", impulse)', + 'request.security(syminfo.tickerid, "240", accAlias - accAlias[1])', +) +COPY_BEFORE_UPDATE = REPORTER.replace( + "accAlias = acc\n", "accAlias = acc\nacc += close - open\n", +) +COPY_VALUE_BEFORE_UPDATE = COPY_BEFORE_UPDATE.replace( + "impulse = accAlias - accAlias[1]", "impulse = accAlias", +) +COPY_CHAIN_BEFORE_UPDATE = TWO_ALIASES.replace( + "accAlias = acc\n", "accAlias = acc\nacc += close - open\n", +) + +# Each group is one requested 4h candle, made from four 1h chart candles. +# The completed signs alternate, and developing candles can change sign. +CHILD_CLOSES = ( + (105.0, 110.0, 109.0, 108.0), + (110.0, 100.0, 95.0, 90.0), + (95.0, 100.0, 109.0, 110.0), + (115.0, 104.0, 103.0, 103.0), + (107.0, 120.0, 122.0, 123.0), + (125.0, 110.0, 100.0, 95.0), + (96.0, 98.0, 102.0, 102.0), + (104.0, 100.0, 98.0, 100.0), +) +HOUR_MS = 60 * 60 * 1000 + + +def chart_bars(): + open_ = 100.0 + for i, close in enumerate(v for group in CHILD_CLOSES for v in group): + yield (open_, max(open_, close) + 1.0, min(open_, close) - 1.0, + close, float(i + 1), i * HOUR_MS) + open_ = close + + +def native_driver(*, outside: bool = False, override: float | None = None) -> str: + bars = list(chart_bars()) + literals = ",\n".join( + " Bar{" + ", ".join(repr(v) for v in row) + "}" for row in bars + ) + if outside: + observer = """ + void on_source_bar(const Bar& bar) override { + GeneratedStrategy::on_source_bar(bar); + std::cout << "chart " << bar.timestamp << " 1 " + << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << static_cast(impulse) << '\\n'; + } +""" + else: + observer = """ + void evaluate_security(int sec_id, const Bar& bar, bool complete) override { + GeneratedStrategy::evaluate_security(sec_id, bar, complete); + if (sec_id == 0) { + std::cout << "requested " << bar.timestamp << ' ' << complete + << ' ' << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << _req_sec_0 << '\\n'; + } + } + + void on_source_bar(const Bar& bar) override { + GeneratedStrategy::on_source_bar(bar); + std::cout << "chart " << bar.timestamp << " 1 " + << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << static_cast(impulse) + << ' ' << static_cast(htf) << '\\n'; + } +""" + setting = "" if override is None else f' strategy.set_input("Step scale", "{override}");\n' + return ( + "#include \n#include \n" + "struct AliasHistoryObserver : GeneratedStrategy {\n" + observer + "};\n" + "int main() {\n std::cout << std::setprecision(17);\n" + " AliasHistoryObserver strategy;\n" + setting + + " Bar bars[] = {\n" + literals + "\n };\n" + + f' strategy.run(bars, {len(bars)}, "60", "60");\n' + + " return 0;\n}\n" + ) + + +CASES = ( + pytest.param(REPORTER, "difference", 1.0, None, id="R01-reporter"), + pytest.param(DIRECT, "difference", 1.0, None, id="C01-direct"), + pytest.param(OUTSIDE, "outside", 1.0, None, id="C02-outside"), + pytest.param(TWO_ALIASES, "difference", 1.0, None, id="A01-two-aliases"), + pytest.param(THREE_ALIASES, "difference", 1.0, None, id="A02-three-aliases"), + pytest.param(LOCAL_SHADOW, "local-shadow", 1.0, None, id="S01-local-shadow"), + pytest.param(PARAMETER_SHADOW, "parameter-shadow", 1.0, None, id="S02-parameter-shadow"), + pytest.param(NO_HISTORY, "cumulative", 1.0, None, id="N01-no-history"), + pytest.param(INPUT_UPDATE, "difference", 1.75, None, id="I01-input-default"), + pytest.param(INPUT_UPDATE, "difference", 2.5, 2.5, id="I02-input-override"), + pytest.param(NAMED, "difference", 1.0, None, id="K01-named"), + pytest.param(DIRECT_PAYLOAD, "difference", 1.0, None, id="H02-direct-payload"), + pytest.param(COPY_BEFORE_UPDATE, "snapshot-history", 1.0, None, id="M01-copy-before-update"), + pytest.param(COPY_VALUE_BEFORE_UPDATE, "snapshot-value", 1.0, None, id="M02-copy-value"), + pytest.param(COPY_CHAIN_BEFORE_UPDATE, "snapshot-history", 1.0, None, id="M03-copy-chain"), +) + + +def expected_values(groups, mode, scale): + """Pine value copies from the input candles, including subsequent stores.""" + result = [] + accumulator = 0.0 + previous_copy = previous_close = None + for group in groups: + open_, close = group[0][0], group[-1][3] + volume = sum(bar[4] for bar in group) + accumulator += (volume if close > open_ else -volume) * scale + copied = accumulator + if mode.startswith("snapshot-"): + accumulator += close - open_ + if mode in {"cumulative", "snapshot-value"}: + value = copied + else: + value = math.nan if previous_copy is None else copied - previous_copy + if mode == "local-shadow" and previous_close is not None: + value += 2.0 * (close - previous_close) + if mode == "parameter-shadow" and previous_close is not None: + value += close - previous_close + result.append(value) + previous_copy, previous_close = copied, close + return result + + +@pytest.mark.parametrize("source, mode, scale, override", CASES) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_alias_history_native_values(source, mode, scale, override, opt): + skip_if_no_engine_lib() + cpp = transpile(source) + output = run_emitted_tu( + cpp, native_driver(outside=mode == "outside", override=override), + opt=opt, label="requested accumulator alias history", + ) + print(output, end="") + rows = [] + chart_rows = [] + for line in output.splitlines(): + parts = line.split() + if mode != "outside" and parts and parts[0] == "chart": + assert len(parts) == 8 and parts[2] == "1", line + chart_rows.append((int(parts[1]), *(float(v) for v in parts[3:]))) + continue + assert len(parts) == 7, line + context, timestamp, complete, open_, close, volume, value = parts + assert context == ("chart" if mode == "outside" else "requested"), line + assert complete in {"0", "1"}, line + rows.append((int(timestamp), complete == "1", float(open_), float(close), + float(volume), float(value))) + + bars = list(chart_bars()) + groups = [bars[i:i + 4] for i in range(0, len(bars), 4)] + if mode == "outside": + groups = [[bar] for bar in bars] + completed = [row for row in rows if row[1]] + assert [row[0] for row in completed] == [group[0][-1] for group in groups] + assert len(rows) >= len(groups) > 1 + + expected_requested = expected_values(groups, mode, scale) + for i, (row, group, wanted) in enumerate(zip(completed, groups, expected_requested)): + open_, close = group[0][0], group[-1][3] + volume = sum(bar[4] for bar in group) + assert row[2:5] == pytest.approx((open_, close, volume)) + if math.isnan(wanted): + assert math.isnan(row[5]), row + else: + assert row[5] == pytest.approx(wanted), (i, row, wanted) + + if mode != "outside": + assert [row[0] for row in chart_rows] == [bar[-1] for bar in bars] + chart_mode = "difference" if mode.endswith("-shadow") else mode + impulses = expected_values([[bar] for bar in bars], chart_mode, scale) + for i, (row, bar, chart_impulse) in enumerate(zip(chart_rows, bars, impulses)): + open_, _, _, close, volume, _ = bar + assert row[1:4] == pytest.approx((open_, close, volume)) + completed_slot = (i + 1) // 4 - 1 + held = (expected_requested[completed_slot] + if completed_slot >= 0 else math.nan) + for actual, wanted in ( + (row[4], chart_impulse), + (row[5], held), + ): + if math.isnan(wanted): + assert math.isnan(actual), (i, row, wanted) + else: + assert actual == pytest.approx(wanted), (i, row, wanted) From c726d58ed629f9044dbe19df729abd2e2496f2d6 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:51:24 +0800 Subject: [PATCH 2/2] Retain lexical bindings for assignment targets Resolve identifier assignment targets through the existing identifier visitor while their lexical scope is live. Requested selections then retain persistent var and varip dependencies for compound writes. Keep the existing outer-write validation and test expectations intact. --- CHANGELOG.md | 2 ++ pineforge_codegen/analyzer/base.py | 4 ++++ pineforge_codegen/codegen/security.py | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 617f10e8..b36dae95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,8 @@ supported as exact pairs; on the 0.x line they are independent. See the value of top-level scalar copies of persistent or reassigned globals, including copy chains. A declared float copy preserves an integer source's missing value, and replayed updates use the requested `bar_index`. + Compound assignments in requested selections retain the lexical binding + of their target and its request-owned `var` or `varip` storage. See [#393](https://github.com/pineforge-4pass/pineforge-codegen-oss/issues/393). ## 1.12.0 — 2026-11-12 diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index f08d631b..517d12d7 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -5745,6 +5745,10 @@ def _visit_Assignment(self, node: Assignment) -> PineType: node.loc or node.target.loc, ) else: + # Retain the target's lexical binding just like a read. + # A compound store may have no other read of its target + # to expose the dependency to a requested expression. + self._visit(node.target) if (node.op in (":=", "+=", "-=", "*=", "/=", "%=") and self._pine_version == 6 and not self._in_v5_library_definition(node) diff --git a/pineforge_codegen/codegen/security.py b/pineforge_codegen/codegen/security.py index e20c8291..50553ea6 100644 --- a/pineforge_codegen/codegen/security.py +++ b/pineforge_codegen/codegen/security.py @@ -10789,8 +10789,7 @@ def walk(value, env): types[current] = ("std::string" if series in self._security_string_series else "double") elif value.target.name in args[4] and bound is None: - # Assignment targets are resolved directly by the analyzer, - # so unlike reads they need not carry identifier provenance. + # The checked outer binding writes its request-owned state. info = self._global_mutable_infos[value.target.name] state = self._security_state_name(args[0], value.target.name) current = f"{state}[0]" if info.is_series else state