diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcc1e7d..b36dae95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,13 @@ supported as exact pairs; on the 0.x line they are independent. See the method calls, including named arguments. Numeric `na` values sort last in ascending order and first in descending order; string arrays use lexical ordering. The source array is unchanged. (#392) +- `request.security` keeps the requested-bar history and declaration-time + value of top-level scalar copies of persistent or reassigned globals, + including copy chains. A declared float copy preserves an integer source's + missing value, and replayed updates use the requested `bar_index`. + Compound assignments in requested selections retain the lexical binding + of their target and its request-owned `var` or `varip` storage. + See [#393](https://github.com/pineforge-4pass/pineforge-codegen-oss/issues/393). ## 1.12.0 — 2026-11-12 diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index ae29864b..517d12d7 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -7,6 +7,7 @@ from __future__ import annotations +from dataclasses import replace from typing import Any from ..ast_nodes import ( @@ -36,7 +37,7 @@ from ..input_keys import assign_input_keys from ..loop_types import FLOAT_REASSIGNMENT, LoopTypeSources, for_counter_type from ..security_contexts import ( - PASS_WARNINGS_ANNOTATION, context_key, reads_bar_series, + GLOBAL_ANNOTATION, PASS_WARNINGS_ANNOTATION, context_key, reads_bar_series, ) from ..session_reads import emitted_session_reads from ..collection_history import ( @@ -182,6 +183,8 @@ def __init__(self, ast: Program, filename: str = "", self._session_unresolved_method_calls: dict[int, set[str]] = {} self._symbols = SymbolTable() self._ta_call_sites: list[TACallSite] = [] + # Final analysis consumes this inventory even when no request was seen. + self._security_calls: list[SecurityCallInfo] = [] self._series_vars: set[str] = set() self._series_var_members: set[str] = set() self._series_decl_nodes: set[int] = set() @@ -731,6 +734,7 @@ def _analyze(self) -> AnalyzerContext: for name, info in self._global_binding_infos.items() if info.is_var or name in self._global_reassigned_names } + self._retain_security_value_copies(mutable_global_infos) pure_global_expr_map = { k: v for k, v in self._global_expr_map.items() if k not in mutable_global_infos @@ -2467,9 +2471,15 @@ def _collect_security_mutable_globals( out: set[str] = set() if isinstance(node, Identifier): + scope = self._identifier_binding_scopes.get(id(node)) + if scope != "global" and not ( + scope is None and (node.annotations or {}).get(GLOBAL_ANNOTATION) + ): + # Parameters and locals do not read a same-spelled global. + return out name = node.name - if name in self._global_binding_infos: - info = self._global_binding_infos[name] + info = self._global_binding_infos.get(name) + if info is not None: if info.is_var or name in self._global_reassigned_names: out.add(name) if name in resolving: @@ -2479,9 +2489,15 @@ def _collect_security_mutable_globals( out |= self._collect_security_mutable_globals(stmt, resolving) resolving.remove(name) return out - if name in self._global_expr_map and name not in resolving: + value = self._global_expr_map.get(name) + if value is None and info is not None and isinstance(info.decl_node, VarDecl): + # The legacy series-name set can omit this global from the + # expression map after a helper parameter reads its history. + # Its declaration still records the call-time dependency. + value = info.decl_node.value + if value is not None and name not in resolving: resolving.add(name) - out |= self._collect_security_mutable_globals(self._global_expr_map[name], resolving) + out |= self._collect_security_mutable_globals(value, resolving) resolving.remove(name) return out @@ -2530,6 +2546,147 @@ def _collect_security_mutable_globals( return out + def _retain_security_value_copies( + self, mutable_infos: dict[str, MutableGlobalInfo], + ) -> None: + """Replay scalar copies whose value cannot be substituted in a request. + + A history read needs the copy's own requested-bar samples. A copy made + before a later store to its source also needs its declaration replayed + at its original position, even without history. Chart storage and + helper-local bindings keep their independently analyzed identities. + """ + if not self._security_calls or not mutable_infos: + return + budget = getattr(self, "_budget", None) + positions = {id(stmt): index for index, stmt in enumerate(self._ast.body)} + + def is_global(node): + scope = self._identifier_binding_scopes.get(id(node)) + annotations = getattr(node, "annotations", None) or {} + return scope == "global" or ( + scope is None and bool(annotations.get(GLOBAL_ANNOTATION)) + ) + + aliases = {} + last_write = {} + primitive = {PineType.FLOAT, PineType.INT, PineType.BOOL, + PineType.STRING, PineType.COLOR} + for name, info in self._global_binding_infos.items(): + if budget is not None: + budget.check(phase=Phase.ANALYZER) + last_write[name] = max( + (positions.get(id(stmt), -1) for stmt in info.source_stmts), + default=-1, + ) + decl = info.decl_node + if (name in mutable_infos or info.pine_type not in primitive + or not isinstance(decl, VarDecl) or id(decl) not in positions + or not isinstance(decl.value, Identifier) or not is_global(decl.value)): + continue + source = self._global_binding_infos.get(decl.value.name) + if source is None or source.pine_type not in primitive: + continue + binding = self._identifier_decl_bindings.get(id(decl.value)) + if (binding is not None and source.decl_node is not None + and binding[0] != id(source.decl_node)): + continue + aliases[name] = source.name + + # Memoize chains rather than walking once per path through a diamond. + rooted = dict.fromkeys(mutable_infos, True) + for name in aliases: + path, seen = [], set() + current = name + while current in aliases and current not in rooted and current not in seen: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + seen.add(current) + path.append(current) + current = aliases[current] + reaches_state = rooted.get(current, False) + for alias in path: + rooted[alias] = reaches_state + aliases = {name: source for name, source in aliases.items() if rooted[name]} + if not aliases: + return + snapshots = { + name for name, source in aliases.items() + if last_write[source] > positions[id(self._global_binding_infos[name].decl_node)] + } + + def dependencies(root): + pending, seen, names, history = [root], set(), set(), set() + while pending: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + node = pending.pop() + if not isinstance(node, ASTNode) or id(node) in seen: + continue + seen.add(id(node)) + if (isinstance(node, Subscript) and isinstance(node.object, Identifier) + and is_global(node.object) and node.object.name in aliases): + history.add(node.object.name) + if isinstance(node, Identifier) and is_global(node): + name = node.name + if name in aliases or name in mutable_infos: + names.add(name) + pending.extend(self._global_binding_infos[name].source_stmts) + elif name in self._global_expr_map: + pending.append(self._global_expr_map[name]) + if isinstance(node, FuncCall): + if self._is_input_func_call(node): + # Cosmetic input arguments are not data dependencies. + pending.append(node.args[0] if node.args else node.kwargs.get("defval")) + continue + if isinstance(node.callee, Identifier): + function = self._func_defs.get(node.callee.name) + if function is not None: + pending.extend(function.body) + pending.extend(syntax_children(node)) + return names, history + + required, histories = set(), set() + requests = [] + for call in self._security_calls: + names, history = dependencies(call.expression) + histories.update(history) + required.update(names & snapshots) + requests.append((call, names)) + required.update(histories) + if not required: + return + for name in sorted(required): + info = self._global_binding_infos[name] + mutable_infos[name] = replace(info, is_series=info.is_series or name in histories) + sources = {} + for name in sorted(required): + chain = set() + source = name + while source in aliases and source not in chain: + if budget is not None: + budget.check(phase=Phase.ANALYZER) + chain.add(source) + source = aliases[source] + chain.add(source) + # The original call-time inventory already bounds which inputs + # of a source's updates are replayed. Retaining a copy must not + # widen that boundary to writes discovered later in the script. + sources[name] = chain & mutable_infos.keys() + for call, names in requests: + # Preserve each request's original replay boundary. A helper read + # elsewhere in its payload must not pull later chart-only writes + # into the request merely because another operand copies state. + original = set(getattr(call, "mutable_globals", ()) or ()) + additions = set() + for name in names & required: + # Newly retained aliases need their own replay. A pre-existing + # mutable source participates only if this call already read + # it as mutable; its first later write is not retroactive. + additions.update(sources[name] & (original | required)) + call.mutable_globals = tuple(sorted(original | additions)) + call.depends_on_mutable_globals = bool(call.mutable_globals) + def _session_history_unsafe_functions( self, func_defs: dict, find_calls, known_func_names: set[str], calls_by_parent: dict, call_edges: list) -> dict[str, str]: @@ -5588,6 +5745,10 @@ def _visit_Assignment(self, node: Assignment) -> PineType: node.loc or node.target.loc, ) else: + # Retain the target's lexical binding just like a read. + # A compound store may have no other read of its target + # to expose the dependency to a requested expression. + self._visit(node.target) if (node.op in (":=", "+=", "-=", "*=", "/=", "%=") and self._pine_version == 6 and not self._in_v5_library_definition(node) diff --git a/pineforge_codegen/analyzer/call_handlers.py b/pineforge_codegen/analyzer/call_handlers.py index 4635541b..f8a8b3df 100644 --- a/pineforge_codegen/analyzer/call_handlers.py +++ b/pineforge_codegen/analyzer/call_handlers.py @@ -18,10 +18,9 @@ index used to mint unique ``_ta__`` member names. - ``self._series_bar_fields`` (``set[str]``): bar-field identifiers used as TA inputs anywhere in the program. -- ``self._security_calls`` (``list[SecurityCallInfo]``): created on - first ``request.security(...)`` via ``getattr(..., "_security_calls", - [])`` and stored back on ``self`` -- the analyzer also reads this - attribute via ``getattr`` in ``analyze()``. +- ``self._security_calls`` (``list[SecurityCallInfo]``): a per-analyzer + inventory initialized empty by the host constructor. Both security + handlers append to it; final analysis also consumes the empty inventory. - ``self._fixnan_counter`` / ``self._fixnan_sites``: ``fixnan(...)`` member-name counter + site list. - ``self._symbols`` (``SymbolTable``): consulted by @@ -859,7 +858,6 @@ def _handle_request_call(self, func_name: str, node: FuncCall) -> PineType: security_ta_range = (ta_start, ta_end) if ta_end > ta_start else None # Assign ID and record the call - self._security_calls = getattr(self, "_security_calls", []) sec_id = len(self._security_calls) returns_tuple = isinstance(expr_node, TupleLiteral) @@ -1095,7 +1093,6 @@ def _handle_request_security_lower_tf(self, node: FuncCall) -> PineType: node.loc, ) - self._security_calls = getattr(self, "_security_calls", []) sec_id = len(self._security_calls) mutable_globals = tuple(sorted(self._collect_security_mutable_globals(expr_node))) diff --git a/pineforge_codegen/codegen/security.py b/pineforge_codegen/codegen/security.py index 297e3bdc..50553ea6 100644 --- a/pineforge_codegen/codegen/security.py +++ b/pineforge_codegen/codegen/security.py @@ -4830,7 +4830,8 @@ def _security_expr_hist_series_names(self, sec_id: int) -> list[str]: def _emit_security_expr_hist_members( self, sec_id: int, expr_node, lines: list[str], mbb_suffix: str ) -> None: - if self._security_reads_bar_index(expr_node): + mutable_names = self._security_eval_info[sec_id].get("mutable_globals") or () + if self._security_reads_bar_index(expr_node, mutable_names): # The requested bar's ``bar_index``: one count per requested bar, # advanced where the evaluator opens its slot. self._security_bar_index_secs.add(sec_id) @@ -7878,6 +7879,11 @@ def _emit_security_rebind_var_decl( emitted_lines=emitted_lines, ) expr_cpp = self._security_copy_store_cpp(node.name, info, node.value, expr_cpp) + cpp_type = self._security_cpp_type_for_mutable(node.name, info) + if node.value is None or isinstance(node.value, NaLiteral) or cpp_type == "double": + # A replayed declaration stores into its declared destination: + # bare na has that type, and a float copy preserves integer na. + expr_cpp = self._security_local_value(expr_cpp, node.value, cpp_type, ()) if getattr(info, "is_var", False): if getattr(info, "is_series", False): @@ -10783,8 +10789,7 @@ def walk(value, env): types[current] = ("std::string" if series in self._security_string_series else "double") elif value.target.name in args[4] and bound is None: - # Assignment targets are resolved directly by the analyzer, - # so unlike reads they need not carry identifier provenance. + # The checked outer binding writes its request-owned state. info = self._global_mutable_infos[value.target.name] state = self._security_state_name(args[0], value.target.name) current = f"{state}[0]" if info.is_series else state @@ -11697,35 +11702,48 @@ def _security_is_bar_index(self, node) -> bool: return (isinstance(node, Identifier) and node.name == "bar_index" and "bar_index" not in getattr(self, "_safe_name_bound", ())) - def _security_reads_bar_index(self, expr_node) -> bool: + def _security_reads_bar_index(self, expr_node, mutable_names=()) -> bool: """Whether a payload reads ``bar_index``: in itself, the globals it - reads or the user functions it calls. TradingView evaluates it on - the requested bar -- the count of requested bars before it -- where - the evaluator used to read the chart's (``pine_bar_index()``).""" - from ..limits import iter_ast_nodes - + reads, replayed state or the user functions it calls. Follow only + this request's mutable inventory, so retaining a copy does not + discover updates outside its original source boundary.""" global_expr_map = getattr(self.ctx, "global_expr_map", {}) or {} + mutable_infos = getattr(self, "_global_mutable_infos", None) or {} + mutable_names = set(mutable_names or ()) + func_names = getattr(self, "_func_names", None) or () + func_infos = getattr(self, "_func_info_map", None) or {} + budget = getattr(self, "_budget", None) seen: set[tuple[str, str]] = set() + visited: set[int] = set() pending = [expr_node] while pending: - root = pending.pop() - if not isinstance(root, ASTNode): + node = pending.pop() + if not isinstance(node, ASTNode) or id(node) in visited: continue - for n, _depth in iter_ast_nodes(root): - if self._security_is_bar_index(n): - return True - if (isinstance(n, Identifier) and n.name in global_expr_map - and ("global", n.name) not in seen - and self._security_identifier_is_global_binding(n)): - seen.add(("global", n.name)) - pending.append(global_expr_map[n.name]) - if (isinstance(n, FuncCall) and isinstance(n.callee, Identifier) - and n.callee.name in self._func_names - and ("func", n.callee.name) not in seen): - seen.add(("func", n.callee.name)) - info = self._func_info_map.get(n.callee.name) - if info is not None and getattr(info, "node", None) is not None: - pending.extend(info.node.body) + visited.add(id(node)) + if budget is not None: + budget.check(getattr(node, "loc", None), Phase.CODEGEN) + if self._security_is_bar_index(node): + return True + if (isinstance(node, Identifier) + and self._security_identifier_is_global_binding(node)): + key = ("global", node.name) + if key not in seen: + seen.add(key) + if node.name in mutable_names: + info = mutable_infos.get(node.name) + pending.extend(getattr(info, "source_stmts", None) or ()) + elif node.name in global_expr_map: + pending.append(global_expr_map[node.name]) + if (isinstance(node, FuncCall) and isinstance(node.callee, Identifier) + and node.callee.name in func_names + and ("func", node.callee.name) not in seen): + seen.add(("func", node.callee.name)) + info = func_infos.get(node.callee.name) + function = getattr(info, "node", None) + if function is not None: + pending.extend(function.body or ()) + pending.extend(syntax_children(node)) return False def _security_variant_order_key(self, signature: tuple, binding_stack) -> tuple: diff --git a/tests/test_security_value_copy_boundaries.py b/tests/test_security_value_copy_boundaries.py new file mode 100644 index 00000000..f70dbc9c --- /dev/null +++ b/tests/test_security_value_copy_boundaries.py @@ -0,0 +1,302 @@ +"""Requested copies retain missing values and their original replay boundary.""" + +from __future__ import annotations + +import math + +import pytest + +from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser +from tests._compile import run_emitted_tu, skip_if_no_engine_lib +from tests._e2e import reference_codegen, transpile_json +from tests._numeric_expected import integer_to_double +from tests.test_security_helper_local_var import _eval_body +from tests.test_security_var_alias_history import HOUR_MS, REPORTER, chart_bars + + +MISSING_INTEGER_COPY = """//@version=6 +strategy("requested scalar value copies") +var int cnt = na +float x = cnt +if close > open + cnt := bar_index +htf = request.security(syminfo.tickerid, "240", x) +""" + +LATE_LOOP_UPDATE = """//@version=6 +strategy("requested scalar value copies") +var float acc = 0.0 +accAlias = acc +acc += close - open +float level = close +getLevel() => level +htf = request.security(syminfo.tickerid, "240", accAlias + getLevel()) +for i = 0 to 1 + level += 1 +plot(htf) +""" + +LATE_SOURCE_INPUT_UPDATE = """//@version=6 +strategy("t") +var float acc = 0.0 +float level = close +accAlias = acc +acc += level +htf = request.security(syminfo.tickerid, "240", accAlias) +for i = 0 to 1 + level += 1 +plot(htf) +""" + + +LATE_COPIED_SOURCE_UPDATE = """//@version=6 +strategy("t") +float acc = close +accAlias = acc +htf = request.security(syminfo.tickerid, "240", accAlias) +for i = 0 to 1 + acc += 1 +plot(htf) +""" + + +def test_copy_capture_adds_only_its_sources_to_requests_that_read_it(): + source = LATE_LOOP_UPDATE.replace( + "for i = 0 to 1", + 'other = request.security(syminfo.tickerid, "60", getLevel())\n' + "for i = 0 to 1", + ) + program = Parser(Lexer(source).tokenize(), source=source).parse() + context = Analyzer(program).analyze() + assert len(context.security_calls) == 2 + assert context.security_calls[0].mutable_globals == ("acc", "accAlias") + assert context.security_calls[1].mutable_globals == () + + +def test_copy_capture_preserves_the_call_time_boundary_of_source_updates(): + program = Parser( + Lexer(LATE_SOURCE_INPUT_UPDATE).tokenize(), source=LATE_SOURCE_INPUT_UPDATE, + ).parse() + context = Analyzer(program).analyze() + assert len(context.security_calls) == 1 + assert context.security_calls[0].mutable_globals == ("acc", "accAlias") + + +@pytest.mark.parametrize("chain", [False, True], ids=["copy", "chain"]) +def test_copy_capture_does_not_add_a_source_first_reassigned_after_the_call(chain): + source = LATE_COPIED_SOURCE_UPDATE + if chain: + source = source.replace("accAlias = acc\n", "accAlias = acc\nnextAlias = accAlias\n") + source = source.replace('"240", accAlias)', '"240", nextAlias)') + program = Parser(Lexer(source).tokenize(), source=source).parse() + context = Analyzer(program, budget=None).analyze() + # The last, unchanged link can expand to the retained earlier copy. + assert context.security_calls[0].mutable_globals == ("accAlias",) + body = _eval_body(transpile(source), 0, None) + assert "_sec0_acc " not in body + assert "_sec0_accAlias = acc;" in body + + +@pytest.mark.parametrize("assignment, integer_type", [ + ("bar_index", "int"), + ("int(close)", "int64_t"), +]) +def test_requested_float_copy_emits_defined_integer_na_and_float_conversion( + assignment, integer_type, +): + source = MISSING_INTEGER_COPY.replace("cnt := bar_index", f"cnt := {assignment}") + cpp = transpile(source) + body = _eval_body(cpp, 0, None) + assert f"_sec0_cnt = na<{integer_type}>();" in body + assert f"_sec0_x = {integer_to_double('_sec0_cnt')};" in body + + +def test_requested_state_updates_discover_the_requested_bar_index(): + cpp = transpile(MISSING_INTEGER_COPY) + body = _eval_body(cpp, 0, None) + assert "int _sec0_bar_index_ = -1;" in cpp + assert "if (security_series_slot_is_new(0)) ++_sec0_bar_index_;" in body + assert "_sec0_cnt = _sec0_bar_index_;" in body + assert "pine_bar_index()" not in body + + +def test_requested_counter_discovery_keeps_later_chart_only_updates_outside(): + source = LATE_COPIED_SOURCE_UPDATE.replace("acc += 1", "acc += bar_index") + cpp = transpile(source) + assert "_sec0_bar_index_" not in cpp + assert "_sec0_accAlias = acc;" in _eval_body(cpp, 0, None) + + +def test_requested_alias_history_emits_its_series_replay_and_read(): + cpp = transpile(REPORTER) + assert "Series _sec0_accAlias;" in cpp + body = _eval_body(cpp, 0, None) + replay = ( + " if (security_series_slot_is_new(0)) {\n" + " _sec0_accAlias.push(_sec0_acc);\n" + " } else {\n" + " _sec0_accAlias.update(_sec0_acc);\n" + " }\n" + ) + assert body.count(replay) == 1 + assert body.count("_req_sec_0 = (_sec0_accAlias[0] - _sec0_accAlias[1]);") == 1 + + +def observe_requested(source, *, cpp=None, bars=None, opt="-O0"): + bars = list(chart_bars()) if bars is None else bars + literals = ",\n".join( + " Bar{" + ", ".join(repr(value) for value in bar) + "}" + for bar in bars + ) + driver = r""" +#include +#include +struct ValueCopyObserver : GeneratedStrategy { + void evaluate_security(int sec_id, const Bar& bar, bool complete) override { + GeneratedStrategy::evaluate_security(sec_id, bar, complete); + if (sec_id == 0) + std::cout << bar.timestamp << ' ' << complete << ' ' + << _req_sec_0 << '\n'; + } +}; +int main() { + std::cout << std::setprecision(17); + ValueCopyObserver strategy; + Bar bars[] = { +BARS + }; + strategy.run(bars, COUNT, "60", "60"); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << '\n'; + return 1; + } + return 0; +} +""".replace("BARS", literals).replace("COUNT", str(len(bars))) + output = run_emitted_tu( + transpile(source) if cpp is None else cpp, + driver, opt=opt, label="requested scalar value copy", + ) + print(output, end="") + rows = [] + for line in output.splitlines(): + parts = line.split() + assert len(parts) == 3 and parts[1] in {"0", "1"}, line + rows.append((int(parts[0]), parts[1] == "1", float(parts[2]))) + completed = [(timestamp, value) for timestamp, complete, value in rows if complete] + assert [timestamp for timestamp, _ in completed] == [ + bar[-1] for bar in bars[::4] + ] + return bars, completed + + +def missing_integer_bars(): + # Begin with an unassigned requested candle, then alternate assignments + # and held values so missingness and finite copy chronology are distinct. + bars = [] + open_ = 100.0 + closes = (99.0, 98.0, 97.0, 96.0, 97.0, 98.0, 99.0, 100.0, + 99.0, 98.0, 97.0, 96.0, 97.0, 98.0, 99.0, 100.0) + for index, close in enumerate(closes): + bars.append((open_, max(open_, close), min(open_, close), close, + float(index + 1), index * HOUR_MS)) + open_ = close + return bars + + +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_requested_float_copy_preserves_integer_na_before_later_assignment(opt): + skip_if_no_engine_lib() + bars = missing_integer_bars() + _, completed = observe_requested(MISSING_INTEGER_COPY, bars=bars, opt=opt) + previous = math.nan + for index, (timestamp, observed) in enumerate(completed): + # The copy precedes this candle's possible store to the integer. + if math.isnan(previous): + assert math.isnan(observed), (timestamp, observed) + else: + assert observed == previous, (timestamp, observed, previous) + group = bars[index * 4:(index + 1) * 4] + if group[-1][3] > group[0][0]: + previous = float(index) + + +@pytest.mark.parametrize("source, source_reads_level", [ + pytest.param(LATE_LOOP_UPDATE, False, id="helper-reads-chart"), + pytest.param(LATE_SOURCE_INPUT_UPDATE, True, id="source-update-reads-chart"), +]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_later_loop_update_keeps_its_existing_chart_value_boundary( + tmp_path, source, source_reads_level, opt, +): + skip_if_no_engine_lib() + base = reference_codegen("ecbbfe392f993e8ce6d4cf12067f44b1b95dcf51") + assert base is not None, "reference commit is not in this checkout's history" + pine = tmp_path / "value-copy.pine" + pine.write_text(source) + result = transpile_json(pine, root=base) + assert result["ok"], result.get("diagnostics") + _, base_completed = observe_requested( + source, cpp=result["cpp"], opt=opt, + ) + bars, completed = observe_requested(source, opt=opt) + accumulator = 0.0 + assert [timestamp for timestamp, _ in base_completed] == [ + timestamp for timestamp, _ in completed + ] + for index, ((timestamp, observed), (_, base_value)) in enumerate( + zip(completed, base_completed) + ): + # The mutable source is replayed in the request. The separate helper + # retains the chart value from the preceding chart evaluation, before + # this requested close's on_source_bar runs. + group = bars[index * 4:(index + 1) * 4] + preceding_chart = bars[index * 4 + 2] + chart_value = preceding_chart[3] + 2.0 + delta = chart_value if source_reads_level else group[-1][3] - group[0][0] + chart_operand = 0.0 if source_reads_level else chart_value + expected = accumulator + chart_operand + assert timestamp == index * 4 * HOUR_MS + assert observed == expected, (timestamp, observed, expected) + # Base expanded the copy after its source's update. Preserve its + # separate chart contribution while correcting the copy chronology. + assert base_value == accumulator + delta + chart_operand + accumulator += delta + + +def assert_requested_values(rows, expected): + assert len(rows) == len(expected) + for (timestamp, actual), wanted in zip(rows, expected): + if math.isnan(wanted): + assert math.isnan(actual), (timestamp, actual, wanted) + else: + assert actual == pytest.approx(wanted), (timestamp, actual, wanted) + + +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +@pytest.mark.parametrize("chain", [False, True], ids=["copy", "chain"]) +def test_late_copied_source_loop_keeps_the_original_boundary(tmp_path, chain, opt): + skip_if_no_engine_lib() + source = LATE_COPIED_SOURCE_UPDATE + if chain: + source = source.replace("accAlias = acc\n", "accAlias = acc\nnextAlias = accAlias\n") + source = source.replace('"240", accAlias)', '"240", nextAlias)') + base = reference_codegen("ecbbfe392f993e8ce6d4cf12067f44b1b95dcf51") + assert base is not None, "reference commit is not in this checkout's history" + pine = tmp_path / "late-source-copy.pine" + pine.write_text(source) + result = transpile_json(pine, root=base) + assert result["ok"], result.get("diagnostics") + bars, observed = observe_requested(source, opt=opt) + _, previous = observe_requested(source, cpp=result["cpp"], bars=bars, opt=opt) + control = source.replace('"240", nextAlias)', '"240", acc)').replace( + '"240", accAlias)', '"240", acc)', + ) + _, direct = observe_requested(control, bars=bars, opt=opt) + expected = [bars[start + 2][3] + 2.0 for start in range(0, len(bars), 4)] + assert_requested_values(observed, expected) + assert_requested_values(previous, expected) + assert_requested_values(direct, expected) diff --git a/tests/test_security_var_alias_history.py b/tests/test_security_var_alias_history.py new file mode 100644 index 00000000..7d35bd57 --- /dev/null +++ b/tests/test_security_var_alias_history.py @@ -0,0 +1,261 @@ +"""A copied accumulator's history belongs to the requested clock. + +The reporter strategy is complete and unchanged. Variants change only the +binding or call form under test. Native observations use the repository's +compile-and-run harness and real engine aggregation with changing chart-candle +directions. The arithmetic oracle uses the input candles, not emitted C++. +""" + +from __future__ import annotations + +import math + +import pytest + +from pineforge_codegen import transpile +from tests._compile import run_emitted_tu, skip_if_no_engine_lib + + +REPORTER = """//@version=6 +strategy("security history of a var alias", overlay = true, initial_capital = 10000) +fast = ta.ema(close, 10) +slow = ta.ema(close, 30) +var float acc = 0.0 +acc += close > open ? volume : -volume +accAlias = acc +impulse = accAlias - accAlias[1] +htf = request.security(syminfo.tickerid, "240", impulse) +plot(htf) +if ta.crossover(fast, slow) + strategy.entry("Long", strategy.long, qty = 0.01) +if ta.crossunder(fast, slow) + strategy.close("Long") +""" + +DIRECT = REPORTER.replace( + "accAlias = acc\nimpulse = accAlias - accAlias[1]\n", + "impulse = acc - acc[1]\n", +) +OUTSIDE = REPORTER.replace( + 'htf = request.security(syminfo.tickerid, "240", impulse)\nplot(htf)', + "plot(accAlias - accAlias[1])", +) +TWO_ALIASES = REPORTER.replace( + "impulse = accAlias - accAlias[1]", + "secondAlias = accAlias\nimpulse = secondAlias - secondAlias[1]", +) +THREE_ALIASES = TWO_ALIASES.replace( + "impulse = secondAlias - secondAlias[1]", + "thirdAlias = secondAlias\nimpulse = thirdAlias - thirdAlias[1]", +) +LOCAL_SHADOW = REPORTER.replace( + "fast = ta.ema", + "shadow() =>\n" + " accAlias = close * 2.0\n" + " accAlias - accAlias[1]\n" + "fast = ta.ema", +).replace('"240", impulse)', '"240", impulse + shadow())') +PARAMETER_SHADOW = REPORTER.replace( + "fast = ta.ema", + "shadow(float accAlias) => accAlias - accAlias[1]\nfast = ta.ema", +).replace('"240", impulse)', '"240", impulse + shadow(close))') +NO_HISTORY = REPORTER.replace( + "impulse = accAlias - accAlias[1]", "impulse = accAlias" +) +INPUT_UPDATE = REPORTER.replace( + "var float acc", 'stepScale = input.float(1.75, "Step scale")\nvar float acc' +).replace( + "acc += close > open ? volume : -volume", + "acc += (close > open ? volume : -volume) * stepScale", +) +NAMED = REPORTER.replace( + 'request.security(syminfo.tickerid, "240", impulse)', + 'request.security(expression = impulse, timeframe = "240", symbol = syminfo.tickerid)', +) +DIRECT_PAYLOAD = REPORTER.replace( + 'request.security(syminfo.tickerid, "240", impulse)', + 'request.security(syminfo.tickerid, "240", accAlias - accAlias[1])', +) +COPY_BEFORE_UPDATE = REPORTER.replace( + "accAlias = acc\n", "accAlias = acc\nacc += close - open\n", +) +COPY_VALUE_BEFORE_UPDATE = COPY_BEFORE_UPDATE.replace( + "impulse = accAlias - accAlias[1]", "impulse = accAlias", +) +COPY_CHAIN_BEFORE_UPDATE = TWO_ALIASES.replace( + "accAlias = acc\n", "accAlias = acc\nacc += close - open\n", +) + +# Each group is one requested 4h candle, made from four 1h chart candles. +# The completed signs alternate, and developing candles can change sign. +CHILD_CLOSES = ( + (105.0, 110.0, 109.0, 108.0), + (110.0, 100.0, 95.0, 90.0), + (95.0, 100.0, 109.0, 110.0), + (115.0, 104.0, 103.0, 103.0), + (107.0, 120.0, 122.0, 123.0), + (125.0, 110.0, 100.0, 95.0), + (96.0, 98.0, 102.0, 102.0), + (104.0, 100.0, 98.0, 100.0), +) +HOUR_MS = 60 * 60 * 1000 + + +def chart_bars(): + open_ = 100.0 + for i, close in enumerate(v for group in CHILD_CLOSES for v in group): + yield (open_, max(open_, close) + 1.0, min(open_, close) - 1.0, + close, float(i + 1), i * HOUR_MS) + open_ = close + + +def native_driver(*, outside: bool = False, override: float | None = None) -> str: + bars = list(chart_bars()) + literals = ",\n".join( + " Bar{" + ", ".join(repr(v) for v in row) + "}" for row in bars + ) + if outside: + observer = """ + void on_source_bar(const Bar& bar) override { + GeneratedStrategy::on_source_bar(bar); + std::cout << "chart " << bar.timestamp << " 1 " + << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << static_cast(impulse) << '\\n'; + } +""" + else: + observer = """ + void evaluate_security(int sec_id, const Bar& bar, bool complete) override { + GeneratedStrategy::evaluate_security(sec_id, bar, complete); + if (sec_id == 0) { + std::cout << "requested " << bar.timestamp << ' ' << complete + << ' ' << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << _req_sec_0 << '\\n'; + } + } + + void on_source_bar(const Bar& bar) override { + GeneratedStrategy::on_source_bar(bar); + std::cout << "chart " << bar.timestamp << " 1 " + << bar.open << ' ' << bar.close << ' ' << bar.volume + << ' ' << static_cast(impulse) + << ' ' << static_cast(htf) << '\\n'; + } +""" + setting = "" if override is None else f' strategy.set_input("Step scale", "{override}");\n' + return ( + "#include \n#include \n" + "struct AliasHistoryObserver : GeneratedStrategy {\n" + observer + "};\n" + "int main() {\n std::cout << std::setprecision(17);\n" + " AliasHistoryObserver strategy;\n" + setting + + " Bar bars[] = {\n" + literals + "\n };\n" + + f' strategy.run(bars, {len(bars)}, "60", "60");\n' + + " return 0;\n}\n" + ) + + +CASES = ( + pytest.param(REPORTER, "difference", 1.0, None, id="R01-reporter"), + pytest.param(DIRECT, "difference", 1.0, None, id="C01-direct"), + pytest.param(OUTSIDE, "outside", 1.0, None, id="C02-outside"), + pytest.param(TWO_ALIASES, "difference", 1.0, None, id="A01-two-aliases"), + pytest.param(THREE_ALIASES, "difference", 1.0, None, id="A02-three-aliases"), + pytest.param(LOCAL_SHADOW, "local-shadow", 1.0, None, id="S01-local-shadow"), + pytest.param(PARAMETER_SHADOW, "parameter-shadow", 1.0, None, id="S02-parameter-shadow"), + pytest.param(NO_HISTORY, "cumulative", 1.0, None, id="N01-no-history"), + pytest.param(INPUT_UPDATE, "difference", 1.75, None, id="I01-input-default"), + pytest.param(INPUT_UPDATE, "difference", 2.5, 2.5, id="I02-input-override"), + pytest.param(NAMED, "difference", 1.0, None, id="K01-named"), + pytest.param(DIRECT_PAYLOAD, "difference", 1.0, None, id="H02-direct-payload"), + pytest.param(COPY_BEFORE_UPDATE, "snapshot-history", 1.0, None, id="M01-copy-before-update"), + pytest.param(COPY_VALUE_BEFORE_UPDATE, "snapshot-value", 1.0, None, id="M02-copy-value"), + pytest.param(COPY_CHAIN_BEFORE_UPDATE, "snapshot-history", 1.0, None, id="M03-copy-chain"), +) + + +def expected_values(groups, mode, scale): + """Pine value copies from the input candles, including subsequent stores.""" + result = [] + accumulator = 0.0 + previous_copy = previous_close = None + for group in groups: + open_, close = group[0][0], group[-1][3] + volume = sum(bar[4] for bar in group) + accumulator += (volume if close > open_ else -volume) * scale + copied = accumulator + if mode.startswith("snapshot-"): + accumulator += close - open_ + if mode in {"cumulative", "snapshot-value"}: + value = copied + else: + value = math.nan if previous_copy is None else copied - previous_copy + if mode == "local-shadow" and previous_close is not None: + value += 2.0 * (close - previous_close) + if mode == "parameter-shadow" and previous_close is not None: + value += close - previous_close + result.append(value) + previous_copy, previous_close = copied, close + return result + + +@pytest.mark.parametrize("source, mode, scale, override", CASES) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_alias_history_native_values(source, mode, scale, override, opt): + skip_if_no_engine_lib() + cpp = transpile(source) + output = run_emitted_tu( + cpp, native_driver(outside=mode == "outside", override=override), + opt=opt, label="requested accumulator alias history", + ) + print(output, end="") + rows = [] + chart_rows = [] + for line in output.splitlines(): + parts = line.split() + if mode != "outside" and parts and parts[0] == "chart": + assert len(parts) == 8 and parts[2] == "1", line + chart_rows.append((int(parts[1]), *(float(v) for v in parts[3:]))) + continue + assert len(parts) == 7, line + context, timestamp, complete, open_, close, volume, value = parts + assert context == ("chart" if mode == "outside" else "requested"), line + assert complete in {"0", "1"}, line + rows.append((int(timestamp), complete == "1", float(open_), float(close), + float(volume), float(value))) + + bars = list(chart_bars()) + groups = [bars[i:i + 4] for i in range(0, len(bars), 4)] + if mode == "outside": + groups = [[bar] for bar in bars] + completed = [row for row in rows if row[1]] + assert [row[0] for row in completed] == [group[0][-1] for group in groups] + assert len(rows) >= len(groups) > 1 + + expected_requested = expected_values(groups, mode, scale) + for i, (row, group, wanted) in enumerate(zip(completed, groups, expected_requested)): + open_, close = group[0][0], group[-1][3] + volume = sum(bar[4] for bar in group) + assert row[2:5] == pytest.approx((open_, close, volume)) + if math.isnan(wanted): + assert math.isnan(row[5]), row + else: + assert row[5] == pytest.approx(wanted), (i, row, wanted) + + if mode != "outside": + assert [row[0] for row in chart_rows] == [bar[-1] for bar in bars] + chart_mode = "difference" if mode.endswith("-shadow") else mode + impulses = expected_values([[bar] for bar in bars], chart_mode, scale) + for i, (row, bar, chart_impulse) in enumerate(zip(chart_rows, bars, impulses)): + open_, _, _, close, volume, _ = bar + assert row[1:4] == pytest.approx((open_, close, volume)) + completed_slot = (i + 1) // 4 - 1 + held = (expected_requested[completed_slot] + if completed_slot >= 0 else math.nan) + for actual, wanted in ( + (row[4], chart_impulse), + (row[5], held), + ): + if math.isnan(wanted): + assert math.isnan(actual), (i, row, wanted) + else: + assert actual == pytest.approx(wanted), (i, row, wanted)