From b80da3a64493527f22c9abb13307af0c38b34f32 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:59:47 +0800 Subject: [PATCH 1/8] Integrate string lowering and string result fixes Preserve UTF-16 units across concatenation, search and string-array joins; retain complete literal bytes in input and metadata consumers; preserve string result types and lexical bindings. Add native regression coverage and document the fixture's measured and derived expectations. Unicode semantic refinements and exact-head verification remain pending. --- CHANGELOG.md | 50 ++ pineforge_codegen/analyzer/base.py | 16 +- pineforge_codegen/analyzer/operators.py | 3 + pineforge_codegen/analyzer/types.py | 30 +- pineforge_codegen/codegen/base.py | 5 +- pineforge_codegen/codegen/checked_settings.py | 32 +- pineforge_codegen/codegen/emit_top.py | 26 +- pineforge_codegen/codegen/helpers.py | 39 +- pineforge_codegen/codegen/input.py | 13 +- pineforge_codegen/codegen/security.py | 128 +++-- pineforge_codegen/codegen/tables.py | 77 ++- pineforge_codegen/codegen/tv_string_units.py | 79 +++- pineforge_codegen/codegen/types.py | 77 +-- pineforge_codegen/codegen/unicode_case.py | 213 +++++++++ pineforge_codegen/codegen/unicode_regex.py | 418 +++++++++++++++++ pineforge_codegen/codegen/visit_call.py | 56 ++- pineforge_codegen/codegen/visit_expr.py | 66 ++- pineforge_codegen/codegen/visit_stmt.py | 20 +- tests/_older_codegen.py | 8 +- tests/fixtures/string_pos_na/README.md | 143 ++++++ tests/fixtures/string_pos_na/strategy.pine | 35 ++ tests/test_array_terminal_returns.py | 142 +++++- tests/test_binop_operand_order.py | 16 +- tests/test_builtin_name_shadow.py | 111 +++++ tests/test_codegen_new.py | 88 +++- tests/test_cpp_allowlist.py | 4 +- tests/test_foreign_footprint.py | 116 ++++- tests/test_input_metadata_receipt.py | 181 ++++++- tests/test_method_receiver_types.py | 25 + tests/test_na_and_compare_refusals.py | 9 +- tests/test_security_helper_local_na.py | 79 ++++ tests/test_string_local_history_concat.py | 20 +- tests/test_string_nul_values.py | 63 ++- tests/test_string_split_empty_separator.py | 44 +- tests/test_string_units.py | 442 +++++++++++++++++- 35 files changed, 2649 insertions(+), 225 deletions(-) create mode 100644 pineforge_codegen/codegen/unicode_case.py create mode 100644 pineforge_codegen/codegen/unicode_regex.py create mode 100644 tests/fixtures/string_pos_na/README.md create mode 100644 tests/fixtures/string_pos_na/strategy.pine diff --git a/CHANGELOG.md b/CHANGELOG.md index f526bbd3..1907656c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,56 @@ supported as exact pairs; on the 0.x line they are independent. See the ## Unreleased (next release) +- Requested helpers preserve declared string argument types when inferring + local variables, including string `na` values, so these expressions emit + string storage and string missing values. + +- String `+`, `+=` and string-array joins recompose adjacent UTF-16 surrogate + halves, including across array parts and separators. Unpaired halves are + preserved; string search predicates compare UTF-16 units. Regenerate C++ + and relink affected strategies. +- Persistent string locals initialized from helper parameters retain their + storage type in compound assignments and `request.security()` expressions. +- Array joins retain string return types through helper and method forwarding. + Named, mixed and reordered receiver/separator arguments use their declared + slots; an omitted separator keeps the default. +- `str.upper` and `str.lower` use full Unicode default mappings, including + supplementary letters, expansions and contextual final sigma. Generated + tables record the translating Python's Unicode version and do not depend + on the native locale. Locale tailoring and normalization are not applied. +- `str.match` lowers its literals, character classes, ranges and repeated + atoms as Unicode characters, including complete supplementary characters, + so matches no longer depend on UTF-8 byte lengths. Word classes remain ASCII; + string lengths and substring offsets still count UTF-16 units. + Regenerate C++ and relink affected strategies to use this correction. +- String input defaults, options and titles preserve embedded NUL bytes in + getters, metadata and checked receipts. Internal timeframe and footprint + column consumers carry literal byte lengths; their validation is unchanged. + Public C input setters retain their NUL-terminated contract. +- `str.split` applies string context to missing arguments, including literal + `na` in positional, named and helper calls. + +- `str.pos` returns integer `na` when the substring is absent, so `na()` detects + the missing match and `nz()` uses its replacement (#382, item 5). It returned + `-1`; found positions retain their UTF-16 offsets. The TradingView expectation + comes from the reporter's readings. Regenerate C++ and relink affected + strategies. + +- `str.substring` checks an explicit `na` end before boundary arithmetic and + returns the engine's canonical string `na` (stored as the empty string). + An `na` start becomes zero, including a wide integer `na` propagated from + `str.pos(...) + 1`, so the two-argument form returns the whole text. Finite + endpoints are ordered before computing the count; an end before the start + stops the run. UTF-16 slicing, end clamping and invalid-begin stops use the + existing helper. The oracle is the reporter's #382 item 9 readings. +- `str.trim` additionally removes VT (U+000B), FF (U+000C), U+00A0, U+1680, + U+2003, U+2028, U+3000 and U+FEFF from the ends. Interior bytes and kept + characters are preserved; other Unicode spaces remain unchanged (#382, item 8). + +- Reads of authored variables named like bar fields use their lexical + declarations and types. A string binding passed to `array.from()` previously + read the numeric bar field, which could fail to compile or produce a wrong + value. Leaving a block that shadows a builtin restores the builtin. - A zone name the engine rejects now makes calendar-form `timestamp()` stop the run at its first evaluation, instead of silently calculating a UTC value. The paired engine reports a typed timezone error identifying the failed bar. diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index 772e2426..7eab1bf7 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -6336,24 +6336,26 @@ def _visit_MethodDef(self, node) -> PineType: def _top_level_branch_needs_lexical_scope( self, body: list[ASTNode] ) -> bool: - """Whether a top-level branch declares over a direct script binding. + """Whether a top-level branch declares over a script binding or builtin. Functions already own a symbol-table scope and ``for``/``for in`` create one explicitly. Top-level if/while/switch branches historically reused the global Scope, so a local declaration could overwrite the analyzer's outer Symbol even though codegen emits a C++ lexical local. - Add a child scope only for the collision shape handled by this change; - every unrelated program keeps the established analysis path. + Builtins need the same child scope as authored outer bindings, so + leaving a branch restores the builtin and its type. """ if self._collection_scope_stack: return False for stmt in body: if (isinstance(stmt, VarDecl) - and stmt.name in self._direct_program_binding_names): + and (stmt.name in self._direct_program_binding_names + or stmt.name in BUILTIN_VARS)): return True if (isinstance(stmt, TupleAssign) and any( name in self._direct_program_binding_names + or name in BUILTIN_VARS for name in stmt.names if name != "_" )): @@ -7837,6 +7839,12 @@ def _visit_FuncCall(self, node: FuncCall) -> PineType: ) return PineType.UNKNOWN return receiver_pine_type + # Preserve fixed scalar array results in the coarse return-type + # visitor as well as TypeSpec inference. Authored methods above + # retain precedence over builtin names. + fixed_array_result = self._fixed_array_call_result(node, receiver_spec) + if fixed_array_result is not None: + return self._element_pine_type(fixed_array_result) # Matrix method dispatch: ``m.get(0, 0)`` on ``matrix`` must # type as INT, not VOID, so ``v = m.get(...)`` propagates the # element PineType. ``_type_spec_from_expr`` already carries the diff --git a/pineforge_codegen/analyzer/operators.py b/pineforge_codegen/analyzer/operators.py index d5497b51..b2cedcca 100644 --- a/pineforge_codegen/analyzer/operators.py +++ b/pineforge_codegen/analyzer/operators.py @@ -6,6 +6,7 @@ # Only argument-independent scalar results belong here. The signature # registry also contains storage-oriented approximations of polymorphic calls. _FIXED_RESULT_CALLS = { + "array": frozenset({"join"}), "str": frozenset({ "tostring", "format", "format_time", "length", "contains", "startswith", "endswith", "pos", "substring", "replace", "replace_all", "lower", @@ -53,6 +54,8 @@ def fixed_builtin_result(namespace: str, name: str, arity: int) -> TypeSpec | No else "int" if name in {"barssince", "highestbars", "lowestbars"} else "float" ) + elif namespace == "array": + result = "string" elif namespace == "str": result = ( "bool" if name in {"contains", "startswith", "endswith"} diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index f65daf91..2f969b47 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -54,7 +54,7 @@ method_receiver_type_name, unify_type_specs, ) -from .operators import join_value_types, reference_value_type +from .operators import fixed_builtin_result, join_value_types, reference_value_type # Drawing-objects-as-data type names (spec §4.1). Defined locally — the # analyzer must not import from ``codegen`` (codegen imports analyzer, so the @@ -173,6 +173,29 @@ def _template_args_from_call(self, node: FuncCall) -> list[str]: raw = ann.get("template_args") or [] return [str(x).replace(" ", "") for x in raw] + def _fixed_array_call_result( + self, call: FuncCall, receiver_spec: TypeSpec | None, + ) -> TypeSpec | None: + """Read a fixed array result after authored-method resolution.""" + callee = call.callee + if not isinstance(callee, MemberAccess): + return None + namespace_call = ( + isinstance(callee.object, Identifier) + and callee.object.name == "array" + and self._symbols.resolve("array") is None + and self._value_bindings.declaration_bindings.get( + id(callee.object) + ) is None + ) + if not namespace_call and ( + receiver_spec is None or receiver_spec.kind != "array" + ): + return None + return fixed_builtin_result( + "array", callee.member, len(call.args) + len(call.kwargs), + ) + def _array_from_element_spec(self, value: ASTNode | None) -> TypeSpec | None: """Exact scalar element type for one ``array.from`` argument. @@ -579,6 +602,11 @@ def direct_user_udt_ctor_name(node: ASTNode) -> str | None: # Its body-derived return type is not available yet, but a # same-named builtin must not lend it a false type. return None + fixed_array_result = self._fixed_array_call_result( + value, typed_receiver_spec, + ) + if fixed_array_result is not None: + return fixed_array_result if ( typed_receiver_spec is not None and typed_receiver_spec.kind == "udt" diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index 6bd0a8ad..f26952cf 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -5211,6 +5211,7 @@ def _prepare_closed_trade_folds(self) -> None: def _generate_cpp(self) -> str: self._uses_enum_string_lookup = False self._uses_tv_string_units = False + self._uses_unicode_regex = False # Titles already carry assigned declaration keys. Refuse an unproved # title first, then flag any remaining shared fallback/suffix keys. self._check_input_titles() @@ -6533,7 +6534,7 @@ def _resolve_known(self, arg_str: str) -> str: if isinstance(val, (int, float)): return str(val) if isinstance(val, str): - return f'std::string("{self._cpp_string_escape(val)}")' + return self._cpp_std_string(val) # Also resolve bar field references if arg_str in BAR_FIELDS: return BAR_FIELDS[arg_str] @@ -6900,7 +6901,7 @@ def _sub(p: re.Match) -> str: return "true" if val else "false" if isinstance(val, (int, float)): return str(val) - return f'std::string("{self._cpp_string_escape(val)}")' + return self._cpp_std_string(val) return name # Identifier tokens only: a string literal's text is not identifiers diff --git a/pineforge_codegen/codegen/checked_settings.py b/pineforge_codegen/codegen/checked_settings.py index ce2b9bb9..79239101 100644 --- a/pineforge_codegen/codegen/checked_settings.py +++ b/pineforge_codegen/codegen/checked_settings.py @@ -9,7 +9,10 @@ _QUOTED_STRING = r'"(?:[^"\\\x00\r\n]|\\(?:["\\nrt]|000))*"' -_STRING_LITERAL = re.compile(rf'(?:std::string\({_QUOTED_STRING}\)|{_QUOTED_STRING})') +_STRING_LITERAL = re.compile( + rf'(?:std::string\((?P{_QUOTED_STRING})(?:, (?P0|[1-9][0-9]*))?\)' + rf'|(?P{_QUOTED_STRING}))' +) _STRING_ESCAPE = re.compile(r'\\(?:["\\nrt]|000)') _STRING_ESCAPE_VALUES = { r'\"': '"', r'\\': '\\', r'\n': '\n', r'\r': '\r', r'\t': '\t', r'\000': '\0', @@ -37,7 +40,11 @@ def _string_setting_arg(expr, lowered: str) -> str | None: ) if not known: return None - return lowered if _STRING_LITERAL.fullmatch(lowered) else None + try: + _string_setting_value(lowered) + except ValueError: + return None + return lowered def _visit_setting_arg(emitter, expr) -> str: @@ -53,16 +60,22 @@ def _visit_setting_arg(emitter, expr) -> str: def _string_setting_value(lowered: str) -> str: - """Decode the emitter's literals as native std::string(const char*) values. + """Decode counted and uncounted literals with their native string semantics. Only the serializer's escape grammar is accepted; Unicode stays literal. - The receipt's string constructors stop at the first NUL, including one - followed by digits, while an escaped backslash remains ordinary data. + A counted literal retains NUL and must carry its exact UTF-8 byte count. + Uncounted C-string conversions still stop at the first NUL. An escaped + backslash remains ordinary data in either form. """ - if not _STRING_LITERAL.fullmatch(lowered): + match = _STRING_LITERAL.fullmatch(lowered) + if match is None: raise ValueError("setting value is not an emitted C++ string literal") - literal = lowered[len("std::string("):-1] if lowered.startswith("std::string(") else lowered + literal = match["wrapped"] or match["bare"] decoded = _STRING_ESCAPE.sub(lambda match: _STRING_ESCAPE_VALUES[match.group()], literal[1:-1]) + if match["count"] is not None: + if int(match["count"]) != len(decoded.encode("utf-8")): + raise ValueError("setting value is not an emitted C++ string literal") + return decoded return decoded.partition("\0")[0] @@ -240,10 +253,7 @@ def emit_settings_members(emitter, lines: list[str], constructor: list[str]) -> option = emitter._string_input_argument(option, refuse_unproven=False) option_cpp = _visit_setting_arg(emitter, option) if getter == "get_input_string": - # An input's options stay C strings: they stop at the - # first NUL, as the default does. - option_cpp = _string_setting_arg( - option, emitter._unsized_std_string(option_cpp)) + option_cpp = _string_setting_arg(option, option_cpp) if option_cpp is None: supported = "false" else: diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index d9c67b33..9c97fce6 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -110,10 +110,14 @@ from .drawing import DRAWING_LIFETIME_CPP from .tv_number_format import TV_NUMBER_FORMAT_CPP from .tv_string_units import TV_STRING_UNITS_CPP, TV_STRING_UNITS_MARKER +from .unicode_regex import UNICODE_REGEX_CPP # Keep the late-selected include beside the headers the C++ allowlist reads. ENUM_STRING_INCLUDE = '#include ' +UNICODE_REGEX_INCLUDES = ( + "#include ", "#include ", "#include ", +) _TRADE_HISTORY_POLICY_MARKER = " /* __PINEFORGE_TRADE_HISTORY_POLICY__ */" @@ -1562,7 +1566,7 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " // invalid (na under ignore_invalid_symbol, a stopped run without it).", " bool _pf_symbol_data_installed(const std::string& symbol,", " const std::string& timeframe,", - " const char* column = nullptr) const {", + " const std::string* column = nullptr) const {", " const auto facts = symbol_facts_.find(symbol);", " if (facts != symbol_facts_.end() && facts->second.valid && !*facts->second.valid)", " return true;", @@ -1573,11 +1577,16 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " if (feed.instrument != symbol || feed.tf != tf) continue;", " if (column == nullptr) return true;", " for (const auto& named : feed.columns) {", - " if (named.name == column) return true;", + " if (named.name == *column) return true;", " }", " }", " return false;", " }", + " bool _pf_symbol_data_installed(const std::string& symbol,", + " const std::string& timeframe,", + " const std::string& column) const {", + " return _pf_symbol_data_installed(symbol, timeframe, &column);", + " }", " // Its payload seeds ta.ema as TradingView does in the requested context:", " // na until `length` values, then their mean (EmaSeeding::SimpleAverage).", " struct _PFForeignEmaSeeding {", @@ -1588,14 +1597,15 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " _PFForeignEmaSeeding& operator=(const _PFForeignEmaSeeding&) = delete;", " };", ]) - if any(self._security_footprint_column(info["sec_id"]) for info in self._security_eval_info): + if any(self._security_footprint_column(info["sec_id"]) is not None + for info in self._security_eval_info): lines.extend([ " // A footprint's delta: the named column of the requested bar.", - " double _pf_symbol_column(int sec_id, const char* name) const {", + " double _pf_symbol_column(int sec_id, const std::string& name) const {", " return security_column_value(sec_id, name);", " }", "#else", - " double _pf_symbol_column(int, const char*) const { return na(); }", + " double _pf_symbol_column(int, const std::string&) const { return na(); }", ]) lines.append("#endif") @@ -1609,7 +1619,7 @@ def _emit_foreign_security_registration( sec_id = info["sec_id"] symbol = self._security_tf_runtime_expr(info["symbol_node"]) column = self._security_footprint_column(sec_id) - column = f', "{self._cpp_string_escape(column)}"' if column else "" + column = f', {self._cpp_string_literal(column)}' if column is not None else "" ignore_node = info.get("ignore_invalid_node") if ignore_node is None: ignore = "false" @@ -2179,6 +2189,10 @@ def _settle_tv_string_units(self, lines: list[str]) -> None: ) from exc if self._uses_tv_string_units: lines[marker] = TV_STRING_UNITS_CPP + if self._uses_unicode_regex: + lines[marker] += "\n".join( + header for header in UNICODE_REGEX_INCLUDES if header not in lines + ) + "\n" + UNICODE_REGEX_CPP else: del lines[marker:marker + 2] diff --git a/pineforge_codegen/codegen/helpers.py b/pineforge_codegen/codegen/helpers.py index d0d666cc..cf65dc1e 100644 --- a/pineforge_codegen/codegen/helpers.py +++ b/pineforge_codegen/codegen/helpers.py @@ -461,6 +461,9 @@ def cpp_code_only(cpp: str) -> str: _pf_tv_trim _pf_tv_increment_digits _pf_tv_decimal _pf_tv_pattern _pf_tv_number_style _pf_u16_length _pf_u16_pos _pf_u16_decode _pf_u16_encode _pf_u16_substr + _pf_u16_find _pf_u16_contains _pf_u16_startswith _pf_u16_endswith _pf_u16_join + _pf_u16_normalize _pf_u16_concat + _pf_regex _security_helper_series_ _security_helper_series_str_ _use_precalc _var_initialized _ta_initialized_ _inputs_initialized_ _pf_lines_ _pf_boxes_ _pf_labels_ _pf_linefills_ @@ -541,6 +544,18 @@ def cpp_code_only(cpp: str) -> str: __pf_idx __pf_occ _pf_number_text _pf_substring_text _pf_substring_begin _pf_substring_end _pf_substring_error + _pf_substring_start _pf_substring_count + _pf_match_source _pf_match_pattern + _pf_search_source _pf_search_needle _pf_join_parts _pf_join_separator + _pf_StringArray _pf_text _pf_needle _pf_text_units _pf_needle_units _pf_at + _pf_parts _pf_separator _pf_separator_units _pf_join_units _pf_part_units _pf_i + _pf_case_text _pf_case_properties _pf_case_flags _pf_case_code + _pf_case_lo _pf_case_hi _pf_case_mid _pf_case_after _pf_case_codes + _pf_case_next_cased _pf_case_i _pf_case_props _pf_case_before _pf_case_lead + _pf_case_width _pf_case_valid _pf_case_k _pf_case_byte _pf_case_min + _pf_case_out _pf_case_append _pf_toupper_map _pf_tolower_map + _pf_trim_text _pf_trim_size _pf_trim_first _pf_trim_last + _pf_trim_i _pf_trim_c _pf_trim_width _pf_trim_second _pf_trim_third _pf_array_new_size _pf_array_new_value _pf_array_new_size_type _pf_array_new_error _pf_read _pf_v5_l _pf_v5_r _pf_wide_l _pf_wide_r _pf_symbol _pf_colon _pf_color _pf_bar_back _pf_ema_seeding @@ -646,20 +661,18 @@ def _cpp_std_string(s: str) -> str: return f'std::string("{escaped}", {len(s.encode("utf-8"))})' return f'std::string("{escaped}")' - # ``_cpp_std_string``'s spelling of a literal that holds a NUL; group 1 is - # the quoted literal. - _SIZED_STD_STRING = re.compile(r'std::string\(("(?:[^"\\\r\n]|\\.)*"), \d+\)') - @staticmethod - def _unsized_std_string(cpp: str) -> str: - """``cpp`` with the byte count of a lowered NUL literal left out. - - The default and the options of an input stay C strings: the settings - receipt and the getter's default stop at the first NUL, as they are - documented to, and the manifest must equal both. A text that is not a - counted literal is returned as it is.""" - sized = NamingHelper._SIZED_STD_STRING.fullmatch(cpp.strip()) - return f"std::string({sized.group(1)})" if sized else cpp + def _cpp_string_literal(s: str) -> str: + """A literal for a length-aware C++ string consumer. + + Keep the bare literal spelling when C-string conversion is lossless; + otherwise carry the UTF-8 byte count through a std::string. Consumers + must accept that string directly, without a const-char intermediate. + This does not change a public NUL-terminated C interface. + """ + if "\0" in s: + return NamingHelper._cpp_std_string(s) + return f'"{NamingHelper._cpp_string_escape(s)}"' @staticmethod def _cpp_comment_escape(s: str) -> str: diff --git a/pineforge_codegen/codegen/input.py b/pineforge_codegen/codegen/input.py index e9b84508..1ab60a09 100644 --- a/pineforge_codegen/codegen/input.py +++ b/pineforge_codegen/codegen/input.py @@ -11,7 +11,7 @@ - ``self._resolve_callee`` (``NamingHelper``). - ``self._visit_expr`` (visitor mixin, currently on ``base.py``; used to render the title-arg fallback expression). -- ``self._cpp_string_escape`` (``NamingHelper``). +- ``self._cpp_string_literal`` (``NamingHelper``). """ from __future__ import annotations @@ -405,10 +405,6 @@ def _input_default_expression(self, node: FuncCall) -> tuple[str, int | None]: if proof is not None: return f"0x{proof.value:08X}LL", proof.value cpp = self._visit_expr(default) if default is not None else "0" - if getter == "get_input_string": - # An input's default stays a C string, cut at its first NUL as the - # settings receipt documents; the manifest equals both. - cpp = self._unsized_std_string(cpp) color = (self._literal_color_value(default) if namespace == "input" and func_name == "color" else None) return cpp, color @@ -482,13 +478,14 @@ def _generic_input_cpp_type(self, node: FuncCall) -> str: return self._GETTER_CPP_TYPE[self._input_getter_for_call(node, "input", None)] def _input_key_literal(self, title: str) -> str: - """An assigned key (or option value) as a C++ string literal. + """An assigned key (or option value) as a C++ string expression. Key assignment happens once before lowering, never during escaping. A key is arbitrary Pine string text; pasted verbatim, a quote in it ends the literal early and a backslash escapes the next character, so - the TU no longer compiles.""" - return f'"{self._cpp_string_escape(title)}"' + the TU no longer compiles. A NUL-bearing key also needs its byte count + when converted to the getter's or descriptor's std::string.""" + return self._cpp_string_literal(title) def _coerce_string_input_default(self, getter: str, default_cpp: str) -> str: """String getters take a ``std::string`` default. An unresolved default diff --git a/pineforge_codegen/codegen/security.py b/pineforge_codegen/codegen/security.py index 056756ab..ae524a4e 100644 --- a/pineforge_codegen/codegen/security.py +++ b/pineforge_codegen/codegen/security.py @@ -209,6 +209,9 @@ def __init__(self, values: dict, caller_stack: tuple[dict, ...], self.caller_stack = caller_stack self.call = call self.float_parameters: frozenset[str] = frozenset() + # Declared parameter types survive expansion to caller-authored ASTs, + # including bare na, whose own expression type is numeric. + self.parameter_types: dict[str, str] = {} # Runtime metadata only: the dict's values remain the authored ASTs, # which key TA discovery and binding signatures. self.samples: dict[str, _SecurityArgumentSample] = {} @@ -917,7 +920,7 @@ def result(sec_id): for sec_id in sites: info = self._security_eval_info[sec_id] tf = info["tf"] - arm = (f'"{self._cpp_string_escape(tf)}"' if tf is not None + arm = (self._security_tf_literal(tf) if tf is not None else info["tf_expr"]) # Missing string values denote the chart timeframe in a selection. condition = f"{target} == {arm}" @@ -925,10 +928,10 @@ def result(sec_id): condition = f"({target}.empty() || {condition})" invalid = info.get("series_tf_invalid") if invalid is not None: - message = self._cpp_string_escape(invalid) + message = self._cpp_std_string(invalid) lines.extend(( f" if ({condition}) {{", - f' pine_runtime_error(std::string("{message}"));', + f' pine_runtime_error({message});', # Retain the expression's scalar/tuple/array return type # even when every inventoried value is invalid. f" return {result(sec_id)};", @@ -1107,9 +1110,8 @@ def check(value) -> None: else substituted) def _security_tf_literal(self, tf: str) -> str: - """A timeframe string as the C++ string literal registration and the - evaluator's ``timeframe.*`` reads spell it.""" - return f'"{self._cpp_string_escape(tf)}"' + """A complete timeframe string for registration and evaluator reads.""" + return self._cpp_string_literal(tf) def _security_lookahead_value(self, node): """A computed lookahead's value before on_bar initializes its aliases.""" @@ -2682,6 +2684,14 @@ def _security_float_parameter(name, bindings) -> bool: return name in getattr(frame, "float_parameters", ()) return False + @staticmethod + def _security_parameter_cpp_type(name, bindings) -> str | None: + for frame in reversed(bindings or ()): + types = getattr(frame, "parameter_types", {}) + if name in frame or name in types: + return types.get(name) + return None + def _security_index_reads_helper_local( self, index, @@ -4958,6 +4968,13 @@ def _security_local_value( ) return expr_cpp + def _security_type_for_decl(self, node, bindings=(), body=()) -> str: + """Type an evaluator local in its requested lexical bindings.""" + if (not node.type_hint + and self._security_value_is_string(node.value, bindings, body)): + return "std::string" + return self._type_for_decl(node) + def _security_float_cpp(self, rendered, node, bindings=()) -> str: """Adapt a builder-owned value without re-evaluating it on the chart.""" if not self._security_integer_na_operand(node, bindings): @@ -4989,6 +5006,11 @@ def _security_integer_na_operand(self, node, bindings=(), depth=0) -> bool: value, stack = context if isinstance(value, ASTNode): return self._security_integer_na_operand(value, stack, depth + 1) + series_name = self._security_series_binding_target(value) + if series_name in self._security_string_series: + # The requested slot's family owns the read even when + # the chart has no active local type for this name. + return False if value in getattr(self, "_security_counted_loop_bindings", ()): # The binding identifies the counter even if an inner # declaration later shadows its authored spelling. @@ -5088,7 +5110,7 @@ def unproved(name: str) -> str: continue if not isinstance(stmt, VarDecl): continue - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, stack + (frame,)) persistent = stmt.is_var or stmt.is_varip if persistent or stmt.name in local_series: numeric = (cpp_type != "std::string" @@ -5192,11 +5214,10 @@ def _security_needs_string_series(self) -> bool: seen: set[str] = set() global_expr_map = getattr(self.ctx, "global_expr_map", {}) or {} - def is_string_decl(stmt: VarDecl) -> bool: - # The emitter's own rule (``_type_for_decl``), read in the same - # class-scope context it runs in. + def is_string_decl(stmt: VarDecl, bindings=(), body=()) -> bool: + # Use the emitter's lexical rule before declaring its series map. try: - return self._type_for_decl(stmt) == "std::string" + return self._security_type_for_decl(stmt, bindings, body) == "std::string" except Exception: return stmt.type_hint == "string" or isinstance(stmt.value, StringLiteral) @@ -5247,23 +5268,30 @@ def scan_expr(node) -> bool: if info is None or info.node is None: continue series = set(self.ctx.func_series_vars.get(info.name, set())) - if scan_body(info.node.body, series): + declarations = _SecurityHelperArgumentFrame({}, ()) + declarations.parameter_types = self._security_declared_parameter_types(info) + if scan_body(info.node.body, series, (declarations,)): return True return False - def scan_body(body, series: set[str]) -> bool: + def scan_body(body, series: set[str], bindings=(), declarations=()) -> bool: + preceding = list(declarations) for stmt in body or []: + if self._budget is not None: + self._budget.check(stmt.loc, Phase.CODEGEN) if isinstance(stmt, VarDecl): persistent = stmt.is_var or ( stmt.is_varip and id(stmt) in self._security_series_tf_local_decls) - if (persistent or stmt.name in series) and is_string_decl(stmt): + if ((persistent or stmt.name in series) + and is_string_decl(stmt, bindings, preceding)): return True if scan_expr(stmt.value): return True + preceding.append(stmt) elif isinstance(stmt, IfStmt): if (scan_expr(stmt.condition) - or scan_body(stmt.body, series) - or scan_body(stmt.else_body, series)): + or scan_body(stmt.body, series, bindings, preceding) + or scan_body(stmt.else_body, series, bindings, preceding)): return True elif scan_expr(stmt): return True @@ -5415,7 +5443,7 @@ def activate_decl() -> None: self._security_persistent_helper_series.add(series_name) cpp_type = None if is_persistent_var or self._security_string_series_declared: - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, runtime_stack_local) is_color_state = ( is_persistent_var and self._security_helper_var_is_color(stmt) ) @@ -5483,7 +5511,7 @@ def activate_decl() -> None: plan["func_info"].name, stmt.name, ) - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, runtime_stack_local) expr_cpp = self._build_security_expr( sec_id, stmt.value, @@ -5544,9 +5572,9 @@ def activate_decl() -> None: ) series_name = self._security_series_binding_target(binding) if series_name is not None: - if series_name in self._security_string_series and isinstance( - stmt.value, NaLiteral): - expr_cpp = "na()" + if series_name in self._security_string_series: + expr_cpp = self._security_local_value( + expr_cpp, stmt.value, "std::string", runtime_stack_local) elif series_name in self._security_color_series: expr_cpp = self._security_color_series_store(expr_cpp) else: @@ -5569,6 +5597,9 @@ def activate_decl() -> None: current = _series_expr(series_name, "0") rhs = self._compound_assign_rhs( current, stmt.op, expr_cpp, stmt.target, stmt.value, + target_cpp_type=( + "std::string" + if series_name in self._security_string_series else None), target_is_double=( series_name not in self._security_color_series and series_name not in self._security_string_series), @@ -5593,7 +5624,8 @@ def activate_decl() -> None: rhs = None else: rhs = self._compound_assign_rhs( - binding, stmt.op, expr_cpp, stmt.target, stmt.value) + binding, stmt.op, expr_cpp, stmt.target, stmt.value, + target_cpp_type=self._security_local_cpp_types.get(binding)) if rhs is None: lines.append(f"{pad}{binding} {stmt.op} {expr_cpp};") else: @@ -6344,6 +6376,14 @@ def _security_warn_chart_call(self, node, reason: str | None = None) -> None: "TradingView evaluates it on the requested bar.", ) + def _security_declared_parameter_types(self, info) -> dict[str, str]: + specs = self.ctx.func_declared_param_type_specs.get(info.name, ()) + return { + name: self._type_spec_to_cpp(spec) + for name, spec in zip(info.node.params, specs) + if spec is not None + } + def _security_bind_helper_args( self, node: FuncCall, @@ -6382,6 +6422,7 @@ def _security_bind_helper_args( fi, index, self._callable_target_callsite_idx(fi, node)) == "double") for frame in frames: frame.float_parameters = float_parameters + frame.parameter_types = self._security_declared_parameter_types(fi) return fi, base_stack + frames if not isinstance(node.callee, Identifier): self._codegen_error( @@ -6433,6 +6474,7 @@ def _security_bind_helper_args( param for index, param in enumerate(params) if self._func_param_int_cpp_type( fi, index, self._callable_target_callsite_idx(fi, node)) == "double") + new_frame.parameter_types = self._security_declared_parameter_types(fi) return fi, base_stack + (new_frame,) def _security_helper_call_plan( @@ -6611,6 +6653,9 @@ def _security_value_is_string( nxt = depth + 1 if isinstance(node, StringLiteral): return True + if isinstance(node, Subscript): + return self._security_value_is_string( + node.object, helper_binding_stack, body, nxt) if isinstance(node, Identifier) and not self._security_identifier_is_global_binding(node): for stmt in reversed(body or []): if isinstance(stmt, VarDecl) and stmt.name == node.name: @@ -6619,6 +6664,10 @@ def _security_value_is_string( return self._security_value_is_string( stmt.value, helper_binding_stack, body, nxt ) + parameter_type = self._security_parameter_cpp_type( + node.name, helper_binding_stack) + if parameter_type is not None: + return parameter_type == "std::string" binding = self._security_lookup_helper_binding_context( node.name, helper_binding_stack ) @@ -7814,15 +7863,19 @@ def store(cpp: str) -> str: # (``/`` in double, ``%`` through std::fmod: C++ ``%`` takes no # double) and narrows its result. wide = node.op != ":=" and store(value_cpp) != value_cpp + slot_type = self._security_cpp_type_for_mutable(target_name, info) + string_concat = node.op == "+=" and slot_type == "std::string" def combined(target_read: str) -> str: - return (self._compound_assign_rhs(target_read, node.op, value_cpp) + operand = store(value_cpp) if string_concat else value_cpp + return (self._compound_assign_rhs( + target_read, node.op, operand, target_cpp_type=slot_type) or f"({target_read} {node.op[0]} {value_cpp})") if getattr(info, "is_series", False): if node.op == ":=": lines.append(f"{pad}{state_name}.update({store(value_cpp)});") - elif wide: + elif wide or string_concat: lines.append(f"{pad}{state_name}.update(" f"{store(combined(f'{state_name}[0]'))});") else: @@ -7832,7 +7885,7 @@ def combined(target_read: str) -> str: if node.op == ":=": lines.append(f"{pad}{state_name} = {store(value_cpp)};") - elif wide: + elif wide or string_concat: lines.append(f"{pad}{state_name} = {store(combined(state_name))};") else: lines.append(f"{pad}{state_name} {node.op} {value_cpp};") @@ -9275,7 +9328,7 @@ def _build_security_expr( if column is not None and self._security_foreign(sec_id): # request.footprint(...) of another symbol: the delta its feed # records for the requested bar (the value its delta() reads). - return f'_pf_symbol_column({sec_id}, "{self._cpp_string_escape(column)}")' + return f'_pf_symbol_column({sec_id}, {self._cpp_string_literal(column)})' if resolving is None: resolving = set() @@ -9319,6 +9372,10 @@ def _build_security_expr( ) if self._security_float_parameter(expr_node.name, helper_binding_stack): rendered = self._security_float_cpp(rendered, bound, bound_stack) + if self._security_parameter_cpp_type( + expr_node.name, helper_binding_stack) == "std::string": + rendered = self._security_local_value( + rendered, bound, "std::string", bound_stack) return rendered bar_fields = { **SECURITY_BAR_FIELD_EXPRS, @@ -9401,6 +9458,9 @@ def _build_security_expr( ) resolving.remove(expr_node.name) symbol = self._variable_symbol(expr_node.name) + if self._infer_type(expr_node) == "std::string": + resolved = self._security_local_value( + resolved, value, "std::string", ()) if (symbol is not None and self._global_float_na_selection(expr_node.name, symbol.pine_type)): resolved = self._security_float_cpp(resolved, value) @@ -9985,6 +10045,12 @@ def _build_security_expr( def lower(left: str, right: str) -> str: cpp_ops = {"and": "&&", "or": "||"} op = cpp_ops.get(expr_node.op, expr_node.op) + if (expr_node.op == "+" + and self._security_value_is_string( + expr_node, helper_binding_stack, [])): + # The builder has resolved helper bindings to their + # requested string values, rather than chart locals. + return self._string_concat_cpp(left, right) if expr_node.op in ("and", "or"): left = self._coerce_bool_expr(left, expr_node.left) right = self._coerce_bool_expr(right, expr_node.right) @@ -10208,6 +10274,9 @@ def argument(value): argument = self._build_security_expr( sec_id, argument_node, ta_range, ta_results, resolving, security_mutable_names, frame.caller_stack, emitted_lines) + if frame.parameter_types.get(parameter) == "std::string": + argument = self._security_local_value( + argument, argument_node, "std::string", frame.caller_stack) name = self._security_next_inline_name(sec_id, func_name, parameter) emitted_lines.append(f" const auto {name} = {argument};") frame.samples[parameter] = _SecurityArgumentSample(name, None) @@ -10943,11 +11012,14 @@ def build(value): return True if isinstance(node, Assignment) and id(node.target) in context["reads"]: name = context["reads"][id(node.target)] + cpp_type = context["types"].get(name) value = build(node.value) + if node.op == "+=" and cpp_type == "std::string": + value = self._security_local_value(value, node.value, cpp_type, args[5]) rhs = (value if node.op == ":=" - else self._compound_assign_rhs(name, node.op, value)) + else self._compound_assign_rhs( + name, node.op, value, target_cpp_type=cpp_type)) if rhs is not None: - cpp_type = context["types"].get(name) if node.op == ":=": rhs = self._security_local_value(rhs, node.value, cpp_type, args[5]) elif cpp_type in ("int", "int64_t"): diff --git a/pineforge_codegen/codegen/tables.py b/pineforge_codegen/codegen/tables.py index deae2117..57ea87f4 100644 --- a/pineforge_codegen/codegen/tables.py +++ b/pineforge_codegen/codegen/tables.py @@ -26,6 +26,8 @@ evaluate_args_once, na_preserving_int_cast, pine_index_int_cast, pine_truth_cast, ) +from .unicode_case import _unicode_case_expr +from .unicode_regex import _str_match_expr _PROVEN_INT_ARG = re.compile( @@ -1196,7 +1198,7 @@ def journal_array_method(a: str, method: str, args: list[str], journal: str, "concat": (0,), } -# Pine parameter order for the checked-index subset. Keeping the receiver +# Pine parameter order for keyword-aware array lowerings. Keeping the receiver # (``id``) separate lets method syntax merge ``a.get(index = i)`` while the # namespace-functional path merges ``array.get(id = a, index = i)``. CHECKED_ARRAY_METHOD_KWARGS: dict[str, list[str]] = { @@ -1215,6 +1217,7 @@ def journal_array_method(a: str, method: str, args: list[str], journal: str, "sort_indices": ["order"], "indexof": ["value"], "unshift": ["value"], + "join": ["separator"], } MAP_METHODS = { @@ -1579,6 +1582,59 @@ def _str_split_expr(args: list[str]) -> str: ) +def _str_trim_expr(args: list[str]) -> str: + """Trim the measured whitespace at the ends, keeping the original bytes.""" + # Match only complete admitted UTF-8 sequences. Every other byte, + # including NUL or malformed UTF-8, extends the retained byte bounds. + return ( + "[](const std::string _pf_trim_text) { " + "const std::size_t _pf_trim_size = _pf_trim_text.size(); " + "std::size_t _pf_trim_first = _pf_trim_size; " + "std::size_t _pf_trim_last = 0; " + "for (std::size_t _pf_trim_i = 0; _pf_trim_i < _pf_trim_size;) { " + "const unsigned char _pf_trim_c = " + "static_cast(_pf_trim_text[_pf_trim_i]); " + "std::size_t _pf_trim_width = 0; " + "if (_pf_trim_c == 0x20u || " + "(_pf_trim_c >= 0x09u && _pf_trim_c <= 0x0Du)) { " + "_pf_trim_width = 1; " + "} else if (_pf_trim_size - _pf_trim_i >= 2 && " + "_pf_trim_c == 0xC2u && " + "static_cast(_pf_trim_text[_pf_trim_i + 1]) == 0xA0u) { " + "_pf_trim_width = 2; " + "} else if (_pf_trim_size - _pf_trim_i >= 3) { " + "const unsigned char _pf_trim_second = " + "static_cast(_pf_trim_text[_pf_trim_i + 1]); " + "const unsigned char _pf_trim_third = " + "static_cast(_pf_trim_text[_pf_trim_i + 2]); " + "if ((_pf_trim_c == 0xE1u && _pf_trim_second == 0x9Au && " + "_pf_trim_third == 0x80u) || " + "(_pf_trim_c == 0xE2u && _pf_trim_second == 0x80u && " + "(_pf_trim_third == 0x83u || _pf_trim_third == 0xA8u)) || " + "(_pf_trim_c == 0xE3u && _pf_trim_second == 0x80u && " + "_pf_trim_third == 0x80u) || " + "(_pf_trim_c == 0xEFu && _pf_trim_second == 0xBBu && " + "_pf_trim_third == 0xBFu)) { " + "_pf_trim_width = 3; } } " + "if (_pf_trim_width != 0) { _pf_trim_i += _pf_trim_width; } " + "else { " + "if (_pf_trim_first == _pf_trim_size) _pf_trim_first = _pf_trim_i; " + "_pf_trim_last = ++_pf_trim_i; } } " + "return _pf_trim_first == _pf_trim_size ? std::string() " + ": _pf_trim_text.substr(_pf_trim_first, _pf_trim_last - _pf_trim_first); " + f"}}(({args[0]}))" + ) + + +def _str_u16_search_expr(function: str, args: list[str]) -> str: + """Bind source, then needle, once each in parameter order.""" + return ( + f"[&](){{ const std::string _pf_search_source = ({args[0]}); " + f"const std::string _pf_search_needle = ({args[1]}); " + f"return _pf_u16_{function}(_pf_search_source, _pf_search_needle); }}()" + ) + + STR_FUNC_MAP = { "tostring": None, # handled separately (already works) "tonumber": lambda args: ( @@ -1589,31 +1645,28 @@ def _str_split_expr(args: list[str]) -> str: f"}}(({args[0]}))" ), "length": lambda args: f"_pf_u16_length({args[0]})", - "contains": lambda args: f"({args[0]}.find({args[1]}) != std::string::npos)", - "startswith": lambda args: f"({args[0]}.substr(0, {args[1]}.length()) == {args[1]})", - "endswith": lambda args: f"({args[0]}.length() >= {args[1]}.length() && {args[0]}.compare({args[0]}.length() - {args[1]}.length(), {args[1]}.length(), {args[1]}) == 0)", - # The two statements keep the source before the needle in evaluation order. - "pos": lambda args: f"[&](){{ const std::string __pf_s = ({args[0]}); const std::string __pf_t = ({args[1]}); return _pf_u16_pos(__pf_s, __pf_t); }}()", + "contains": lambda args: _str_u16_search_expr("contains", args), + "startswith": lambda args: _str_u16_search_expr("startswith", args), + "endswith": lambda args: _str_u16_search_expr("endswith", args), + "pos": lambda args: _str_u16_search_expr("pos", args), "substring": None, # handled separately (2 vs 3 args) "replace_all": lambda args: f"[&](){{ std::string __pf_s={args[0]}; size_t __pf_p=0; while((__pf_p=__pf_s.find({args[1]},__pf_p))!=std::string::npos){{ __pf_s.replace(__pf_p,{args[1]}.length(),{args[2]}); __pf_p+={args[2]}.length(); }} return __pf_s; }}()", "replace": None, # handled separately (3 vs 4 args) - "lower": lambda args: f"[&](){{ std::string __pf_s={args[0]}; std::transform(__pf_s.begin(),__pf_s.end(),__pf_s.begin(),::tolower); return __pf_s; }}()", - "upper": lambda args: f"[&](){{ std::string __pf_s={args[0]}; std::transform(__pf_s.begin(),__pf_s.end(),__pf_s.begin(),::toupper); return __pf_s; }}()", - "trim": lambda args: f'[&](){{ std::string __pf_s={args[0]}; __pf_s.erase(0,__pf_s.find_first_not_of(" \\t\\n\\r")); __pf_s.erase(__pf_s.find_last_not_of(" \\t\\n\\r")+1); return __pf_s; }}()', + "lower": lambda args: _unicode_case_expr(args[0], upper=False), + "upper": lambda args: _unicode_case_expr(args[0], upper=True), + "trim": _str_trim_expr, "repeat": lambda args: ( f"[&](){{ std::string __pf_r; for(int __pf_i=0;__pf_i<{na_preserving_int_cast(args[1])};__pf_i++) __pf_r+={args[0]}; return __pf_r; }}()" if len(args) < 3 else f"[&](){{ std::string __pf_r; for(int __pf_i=0;__pf_i<{na_preserving_int_cast(args[1])};__pf_i++){{ if(__pf_i>0) __pf_r+={args[2]}; __pf_r+={args[0]}; }} return __pf_r; }}()" ), - "match": lambda args: f'pine_str_match({args[0]}, {args[1]})', + "match": _str_match_expr, "split": _str_split_expr, "format": None, # handled separately } # The STR_FUNC_MAP arguments a template reads more than once, or once per # loop iteration: evaluate_args_once binds each that is not a plain read. STR_ARGS_READ_REPEATEDLY = { - "startswith": (1,), - "endswith": (0, 1), "replace_all": (1, 2), "repeat": (0, 1, 2), } diff --git a/pineforge_codegen/codegen/tv_string_units.py b/pineforge_codegen/codegen/tv_string_units.py index 32a3192b..6ef291c5 100644 --- a/pineforge_codegen/codegen/tv_string_units.py +++ b/pineforge_codegen/codegen/tv_string_units.py @@ -4,9 +4,11 @@ the Basic Multilingual Plane (an emoji) is two units, every other character is one. The generated code keeps text as UTF-8, whose byte counts differ from the unit counts for every character that is not plain ASCII (an accent is two -bytes, a CJK character three, an emoji four). ``str.length``, ``str.pos`` and -``str.substring`` count and cut through these translation-unit-local helpers; -the engine is not involved. +bytes, a CJK character three, an emoji four). String measurement, search and +substring use these translation-unit-local helpers. String-array joins also +use the same units, so adjacent surrogate halves recompose across parts and +separators. Concatenation uses the same decode/re-encode path, preserving +unpaired halves; the engine is not involved. A cut between the two units of an emoji leaves a lone surrogate. UTF-8 has no form for it, so each half is kept as its own three-byte sequence, the form @@ -35,12 +37,6 @@ return static_cast(units); } -inline int _pf_u16_pos(const std::string& text, const std::string& needle) { - const std::size_t at = text.find(needle); - if (at == std::string::npos) return -1; - return _pf_u16_length(text.substr(0, at)); -} - inline std::vector _pf_u16_decode(const std::string& text) { std::vector units; units.reserve(text.size()); @@ -68,6 +64,46 @@ return units; } +inline std::size_t _pf_u16_find(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + if (_pf_needle_units.empty()) return 0; + const auto _pf_at = std::search( + _pf_text_units.begin(), _pf_text_units.end(), + _pf_needle_units.begin(), _pf_needle_units.end()); + if (_pf_at == _pf_text_units.end()) return std::string::npos; + return static_cast(_pf_at - _pf_text_units.begin()); +} + +inline int _pf_u16_pos(const std::string& _pf_text, const std::string& _pf_needle) { + const std::size_t _pf_at = _pf_u16_find(_pf_text, _pf_needle); + return _pf_at == std::string::npos ? na() : static_cast(_pf_at); +} + +inline bool _pf_u16_contains(const std::string& _pf_text, + const std::string& _pf_needle) { + return _pf_u16_find(_pf_text, _pf_needle) != std::string::npos; +} + +inline bool _pf_u16_startswith(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + return _pf_needle_units.size() <= _pf_text_units.size() + && std::equal(_pf_needle_units.begin(), _pf_needle_units.end(), + _pf_text_units.begin()); +} + +inline bool _pf_u16_endswith(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + return _pf_needle_units.size() <= _pf_text_units.size() + && std::equal(_pf_needle_units.rbegin(), _pf_needle_units.rend(), + _pf_text_units.rbegin()); +} + // The units from begin to end, as UTF-8. A high surrogate that the cut leaves // without its low one (or the reverse) is a lone half, kept as three bytes. inline std::string _pf_u16_encode(const std::vector& units, std::size_t begin, @@ -102,6 +138,31 @@ return text; } +inline std::string _pf_u16_normalize(const std::string& text) { + const std::vector units = _pf_u16_decode(text); + return _pf_u16_encode(units, 0, units.size()); +} + +inline std::string _pf_u16_concat(const std::string& left, const std::string& right) { + return _pf_u16_normalize(left + right); +} + +template +inline std::string _pf_u16_join(const _pf_StringArray& _pf_parts, + const std::string& _pf_separator) { + const auto _pf_separator_units = _pf_u16_decode(_pf_separator); + std::vector _pf_join_units; + for (std::size_t _pf_i = 0; _pf_i < _pf_parts.size(); ++_pf_i) { + if (_pf_i != 0) + _pf_join_units.insert(_pf_join_units.end(), + _pf_separator_units.begin(), _pf_separator_units.end()); + const auto _pf_part_units = _pf_u16_decode(_pf_parts[_pf_i]); + _pf_join_units.insert(_pf_join_units.end(), + _pf_part_units.begin(), _pf_part_units.end()); + } + return _pf_u16_encode(_pf_join_units, 0, _pf_join_units.size()); +} + // std::string::substr in units: past the end it throws std::out_of_range with // the text substr gives (the units are counted), and a count beyond the end is // cut to the end. diff --git a/pineforge_codegen/codegen/types.py b/pineforge_codegen/codegen/types.py index abc3fb2b..05935a74 100644 --- a/pineforge_codegen/codegen/types.py +++ b/pineforge_codegen/codegen/types.py @@ -3547,11 +3547,15 @@ def _array_method_expr( view_function=view_function, ) elif method == "join" and elem_cpp == "std::string": - join_args = list(args) or ['std::string(",")'] - lower_receiver = lambda recv: evaluate_args_once( - join_args, ARRAY_ARGS_READ_REPEATEDLY["join"], - lambda a: f"[&](){{ std::string __pf_r; for(size_t __pf_i=0;__pf_i<{recv}.size();__pf_i++){{ if(__pf_i>0)__pf_r+={a[0]}; __pf_r+={recv}[__pf_i]; }} return __pf_r; }}()", - "_pf_array_a", + self._uses_tv_string_units = True + separator = args[0] if args else 'std::string(",")' + # Bind the receiver before the separator even when the shared + # receiver wrapper sees only one use. A const reference also + # keeps a temporary array alive throughout the join. + lower_receiver = lambda recv: ( + f"[&](){{ const auto& _pf_join_parts = ({recv}); " + f"const std::string _pf_join_separator = ({separator}); " + "return _pf_u16_join(_pf_join_parts, _pf_join_separator); }()" ) else: numeric_only = { @@ -3942,12 +3946,12 @@ def promote_wide_int(cpp_type: str) -> str: _dret = self._drawing_call_return_cpp(node.value) if _dret is not None: return _dret - local = self._builtin_local_symbol(node) - if local is not None: - if local.type_spec is not None: - return promote_wide_int(self._type_spec_to_cpp(local.type_spec)) - if local.pine_type not in (PineType.UNKNOWN, PineType.VOID): - return promote_wide_int(PINE_TYPE_TO_CPP[local.pine_type]) + declared = self._builtin_declaration_symbol(node) + if declared is not None: + if declared.type_spec is not None: + return promote_wide_int(self._type_spec_to_cpp(declared.type_spec)) + if declared.pine_type not in (PineType.UNKNOWN, PineType.VOID): + return promote_wide_int(PINE_TYPE_TO_CPP[declared.pine_type]) # Analyzer scopes are popped before codegen, so ctx.symbols.resolve may # find a same-named global instead of this active callable's plain # local. The local binding is known from the emitted body inventory; @@ -4052,32 +4056,45 @@ def _series_handle_target(self, name: str) -> str | None: return None return DRAWING_TYPE_TO_CPP.get(udt, udt) - def _builtin_local_symbol(self, node): - """Recover a builtin-shadowing declaration from its own callable scope.""" + def _builtin_declaration_symbol(self, node): + """Recover this builtin-named variable's exact authored declaration.""" if node.name not in sigs.BUILTIN_VARIABLES: return None - declaration = ( - node if isinstance(node, VarDecl) - else self.ctx.identifier_binding_nodes.get(id(node)) - ) - if declaration is None: - return None - symbols = getattr(self, "_builtin_local_symbols", None) + if isinstance(node, VarDecl): + declaration = id(node) + else: + binding = ( + self.ctx.identifier_decl_bindings[id(node)] + if id(node) in self.ctx.identifier_decl_bindings + else ValueBindings.declaration_of(node) + ) + if binding is None: + return None + declaration = binding[0] + symbols = getattr(self, "_builtin_declaration_symbols", None) if symbols is None: - symbols = self._builtin_local_symbols = {} + symbols = self._builtin_declaration_symbols = {} + local_symbols = self._builtin_local_symbols = {} for scope in self.ctx.symbols.all_scopes: owner = scope while (owner is not None and not owner.name.startswith(("func_", "method_"))): owner = owner.parent - if owner is None: - continue for symbol in scope.symbols.values(): declared = getattr(symbol, "_pf_decl_node", None) if (symbol.name in sigs.BUILTIN_VARIABLES and isinstance(declared, VarDecl)): symbols[id(declared)] = symbol - return symbols.get(id(declaration)) + if owner is not None: + local_symbols[id(declared)] = symbol + return symbols.get(declaration) + + def _builtin_local_symbol(self, node): + """Recover a builtin-shadowing declaration from its own callable scope.""" + symbol = self._builtin_declaration_symbol(node) + if symbol is None: + return None + return self._builtin_local_symbols.get(id(symbol._pf_decl_node)) def _variable_symbol(self, name: str): """The symbol the variable ``name`` resolves to. @@ -6276,9 +6293,9 @@ def _infer_type( lexical_tuple_type = self._lexical_tuple_cpp_type(node) if lexical_tuple_type is not None: return lexical_tuple_type - local = self._builtin_local_symbol(node) - if local is not None: - return self._type_for_decl(local._pf_decl_node) + declared = self._builtin_declaration_symbol(node) + if declared is not None: + return self._type_for_decl(declared._pf_decl_node) binding = self.ctx.identifier_decl_bindings.get(id(node)) if (binding is not None and binding[0] in self.ctx.primitive_block_shadow_types): @@ -6292,10 +6309,10 @@ def _infer_type( ) if tuple_type is not None: return tuple_type - if not tuple_binding and node.name in ("time", "time_close", "timenow"): + builtin_read = self._reads_builtin_variable(node) + if builtin_read and node.name in ("time", "time_close", "timenow"): return "int64_t" - if (node.name in BAR_FIELDS - or (node.name in BAR_BUILTINS and not tuple_binding)): + if builtin_read: return "double" if (node.name in self._known_vars and not self._known_var_is_lexically_shadowed(node.name)): diff --git a/pineforge_codegen/codegen/unicode_case.py b/pineforge_codegen/codegen/unicode_case.py new file mode 100644 index 00000000..45bdba4a --- /dev/null +++ b/pineforge_codegen/codegen/unicode_case.py @@ -0,0 +1,213 @@ +"""Emit Unicode default casing over runtime UTF-8 strings. + +Generate complete, sorted case tables from Python's standard Unicode database, +including expansions and supplementary letters. The emitted comment records +that database's version; the compiled strategy needs no Unicode library. +Locale-specific tailoring and normalization are not default casing. + +Default lowercase has one contextual rule, Final_Sigma. Cased is the union of +Lowercase, Uppercase and General_Category=Lt. Obtain Case_Ignorable from the +same database through that rule: after A+SIGMA, a cased character is ignorable +iff SIGMA stays final; an uncased character is ignorable iff following it with +A makes SIGMA nonfinal. Ignoring takes precedence for overlapping properties. + +The captureless lambda receives its argument outside its local scope, once. +Adjacent WTF-8 high/low surrogates form one code point before casing, just as +they do in UTF-16. NUL, isolated surrogate pieces and malformed bytes survive. +""" + +from __future__ import annotations + +from functools import lru_cache +import unicodedata + + +@lru_cache(maxsize=1) +def _context_ranges() -> tuple[tuple[int, int, int], ...]: + ranges: list[tuple[int, int, int]] = [] + for code in range(0x110000): + char = chr(code) + cased = char.islower() or char.isupper() or char.istitle() + if cased: + ignorable = ("A\u03a3" + char).lower()[1] == "\u03c2" + else: + ignorable = ("A\u03a3" + char + "A").lower()[1] == "\u03c3" + flags = int(cased) | (int(ignorable) << 1) + if flags: + if ranges and ranges[-1][1] + 1 == code and ranges[-1][2] == flags: + start, _, _ = ranges[-1] + ranges[-1] = (start, code, flags) + else: + ranges.append((code, code, flags)) + return tuple(ranges) + + +_CONTEXT_CPP = r""" + static constexpr uint32_t _pf_case_properties[][3] = { +@PROPERTIES@ + }; + const auto _pf_case_flags = [](uint32_t _pf_case_code) -> unsigned { + std::size_t _pf_case_lo = 0; + std::size_t _pf_case_hi = sizeof(_pf_case_properties) / sizeof(_pf_case_properties[0]); + while (_pf_case_lo < _pf_case_hi) { + const std::size_t _pf_case_mid = _pf_case_lo + (_pf_case_hi - _pf_case_lo) / 2; + if (_pf_case_properties[_pf_case_mid][1] < _pf_case_code) + _pf_case_lo = _pf_case_mid + 1; + else + _pf_case_hi = _pf_case_mid; + } + if (_pf_case_lo < sizeof(_pf_case_properties) / sizeof(_pf_case_properties[0]) + && _pf_case_properties[_pf_case_lo][0] <= _pf_case_code) + return _pf_case_properties[_pf_case_lo][2]; + return 0; + }; + std::vector _pf_case_after(_pf_case_codes.size()); + bool _pf_case_next_cased = false; + for (std::size_t _pf_case_i = _pf_case_codes.size(); _pf_case_i > 0;) { + --_pf_case_i; + _pf_case_after[_pf_case_i] = static_cast(_pf_case_next_cased); + const unsigned _pf_case_props = _pf_case_flags(_pf_case_codes[_pf_case_i]); + if (!(_pf_case_props & 2u)) _pf_case_next_cased = (_pf_case_props & 1u) != 0; + } + bool _pf_case_before = false; +""" + +_CONTEXT_SELECT_CPP = r""" + if (_pf_case_code == 0x3A3u && _pf_case_before && !_pf_case_after[_pf_case_i]) { + _pf_case_append(0x3C2u); + } else +""" + +_CONTEXT_ADVANCE_CPP = r""" + const unsigned _pf_case_props = _pf_case_flags(_pf_case_code); + if (!(_pf_case_props & 2u)) _pf_case_before = (_pf_case_props & 1u) != 0; +""" + +_BODY_CPP = r"""[](const std::string& _pf_case_text) { + // Unicode @VERSION@ default @DIRECTION@, independent of the C locale. + static constexpr uint32_t @TABLE@[][@WIDTH@] = { +@MAPPINGS@ + }; + std::vector _pf_case_codes; + _pf_case_codes.reserve(_pf_case_text.size()); + for (std::size_t _pf_case_i = 0; _pf_case_i < _pf_case_text.size();) { + const unsigned char _pf_case_lead = static_cast(_pf_case_text[_pf_case_i]); + std::size_t _pf_case_width = _pf_case_lead < 0x80u ? 1 + : _pf_case_lead >= 0xC2u && _pf_case_lead <= 0xDFu ? 2 + : _pf_case_lead >= 0xE0u && _pf_case_lead <= 0xEFu ? 3 + : _pf_case_lead >= 0xF0u && _pf_case_lead <= 0xF4u ? 4 : 0; + uint32_t _pf_case_code = _pf_case_width == 1 ? _pf_case_lead + : _pf_case_lead & (0x7Fu >> _pf_case_width); + bool _pf_case_valid = _pf_case_width && _pf_case_width <= _pf_case_text.size() - _pf_case_i; + if (_pf_case_valid) { + for (std::size_t _pf_case_k = 1; _pf_case_k < _pf_case_width; ++_pf_case_k) { + const unsigned char _pf_case_byte = + static_cast(_pf_case_text[_pf_case_i + _pf_case_k]); + if ((_pf_case_byte & 0xC0u) != 0x80u) { _pf_case_valid = false; break; } + _pf_case_code = (_pf_case_code << 6) | (_pf_case_byte & 0x3Fu); + } + const uint32_t _pf_case_min = _pf_case_width == 1 ? 0 + : _pf_case_width == 2 ? 0x80u : _pf_case_width == 3 ? 0x800u : 0x10000u; + _pf_case_valid = _pf_case_valid && _pf_case_code >= _pf_case_min && _pf_case_code <= 0x10FFFFu; + } + // WTF-8 encodes isolated surrogate units as three bytes; keep them. + // Preserve each malformed byte with a value outside the Unicode range. + if (!_pf_case_valid) { + _pf_case_code = 0x110000u + _pf_case_lead; + _pf_case_width = 1; + } + if (_pf_case_code >= 0xDC00u && _pf_case_code <= 0xDFFFu + && !_pf_case_codes.empty() + && _pf_case_codes.back() >= 0xD800u && _pf_case_codes.back() <= 0xDBFFu) { + _pf_case_codes.back() = 0x10000u + + ((_pf_case_codes.back() - 0xD800u) << 10) + (_pf_case_code - 0xDC00u); + } else { + _pf_case_codes.push_back(_pf_case_code); + } + _pf_case_i += _pf_case_width; + } +@CONTEXT@ + std::string _pf_case_out; + _pf_case_out.reserve(_pf_case_text.size()); + const auto _pf_case_append = [&_pf_case_out](uint32_t _pf_case_code) { + if (_pf_case_code >= 0x110000u) { + _pf_case_out.push_back(static_cast(_pf_case_code - 0x110000u)); + } else if (_pf_case_code < 0x80u) { + _pf_case_out.push_back(static_cast(_pf_case_code)); + } else if (_pf_case_code < 0x800u) { + _pf_case_out.push_back(static_cast(0xC0u | (_pf_case_code >> 6))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } else if (_pf_case_code < 0x10000u) { + _pf_case_out.push_back(static_cast(0xE0u | (_pf_case_code >> 12))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 6) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } else { + _pf_case_out.push_back(static_cast(0xF0u | (_pf_case_code >> 18))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 12) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 6) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } + }; + for (std::size_t _pf_case_i = 0; _pf_case_i < _pf_case_codes.size(); ++_pf_case_i) { + const uint32_t _pf_case_code = _pf_case_codes[_pf_case_i]; +@CONTEXT_SELECT@ + { + std::size_t _pf_case_lo = 0; + std::size_t _pf_case_hi = sizeof(@TABLE@) / sizeof(@TABLE@[0]); + while (_pf_case_lo < _pf_case_hi) { + const std::size_t _pf_case_mid = _pf_case_lo + (_pf_case_hi - _pf_case_lo) / 2; + if (@TABLE@[_pf_case_mid][0] < _pf_case_code) _pf_case_lo = _pf_case_mid + 1; + else _pf_case_hi = _pf_case_mid; + } + if (_pf_case_lo < sizeof(@TABLE@) / sizeof(@TABLE@[0]) + && @TABLE@[_pf_case_lo][0] == _pf_case_code) { + for (uint32_t _pf_case_k = 0; _pf_case_k < @TABLE@[_pf_case_lo][1]; ++_pf_case_k) + _pf_case_append(@TABLE@[_pf_case_lo][_pf_case_k + 2]); + } else { + _pf_case_append(_pf_case_code); + } + } +@CONTEXT_ADVANCE@ + } + return _pf_case_out; +}""" + + +@lru_cache(maxsize=2) +def _case_lambda(upper: bool) -> str: + mapping = str.upper if upper else str.lower + rows = [] + for code in range(0x110000): + char = chr(code) + converted = mapping(char) + if converted != char: + rows.append((code, len(converted), *(ord(part) for part in converted))) + width = max(len(row) for row in rows) + mappings = "\n".join( + " {" + ", ".join(f"0x{value:X}u" for value in row) + "}," + for row in rows + ) + context = "" + if not upper: + properties = "\n".join( + f" {{0x{start:X}u, 0x{end:X}u, {flags}u}}," + for start, end, flags in _context_ranges() + ) + context = _CONTEXT_CPP.replace("@PROPERTIES@", properties) + result = _BODY_CPP + for key, value in ( + ("@VERSION@", unicodedata.unidata_version), + ("@DIRECTION@", "uppercase" if upper else "lowercase"), + ("@TABLE@", "_pf_toupper_map" if upper else "_pf_tolower_map"), + ("@WIDTH@", str(width)), + ("@MAPPINGS@", mappings), + ("@CONTEXT@", context), + ("@CONTEXT_SELECT@", "" if upper else _CONTEXT_SELECT_CPP), + ("@CONTEXT_ADVANCE@", "" if upper else _CONTEXT_ADVANCE_CPP), + ): + result = result.replace(key, value) + return result + + +def _unicode_case_expr(text: str, *, upper: bool) -> str: + return f"({_case_lambda(upper)})(({text}))" diff --git a/pineforge_codegen/codegen/unicode_regex.py b/pineforge_codegen/codegen/unicode_regex.py new file mode 100644 index 00000000..8016ebb3 --- /dev/null +++ b/pineforge_codegen/codegen/unicode_regex.py @@ -0,0 +1,418 @@ +"""Lower Unicode-character regex atoms to ordinary byte-regex expressions. + +The runtime regex accepts bytes. A Unicode character is represented by its one +to four byte encoding, retaining an isolated surrogate as three bytes. Each +regex atom is a noncapturing group over those encodings, so a quantifier +repeats characters and new groups do not renumber captures. Class unions, ranges and +complements use the entire character domain, not an inventory of Unicode letters. + +Matching still goes through the runtime's string helper, retaining its capture +selection and coded regex errors. Only the matched units are joined back into +ordinary UTF-8. Valid surrogate pairs form one regex character; string lengths +and substring offsets continue to use the unchanged UTF-16 codec. +""" + + +def _str_match_expr(args: list[str]) -> str: + """Bind both argument expressions once in declared parameter order.""" + return ( + "[&]() { " + f"const std::string _pf_match_source = ({args[0]}); " + f"const std::string _pf_match_pattern = ({args[1]}); " + "return ::_pf_regex::_pf_match(" + "_pf_match_source, _pf_match_pattern, bar_index_); }()" + ) + + +UNICODE_REGEX_CPP = r""" +namespace _pf_regex { +using _pf_ranges = std::vector>; +struct _pf_invalid {}; + +inline std::vector _pf_characters(const std::string& _pf_text) { + const auto _pf_decoded = _pf_u16_decode(_pf_text); + std::vector _pf_result; + for (std::size_t _pf_i = 0; _pf_i < _pf_decoded.size(); ++_pf_i) { + uint32_t _pf_code = _pf_decoded[_pf_i]; + if (_pf_code >= 0xD800u && _pf_code <= 0xDBFFu + && _pf_i + 1 < _pf_decoded.size() + && _pf_decoded[_pf_i + 1] >= 0xDC00u + && _pf_decoded[_pf_i + 1] <= 0xDFFFu) { + _pf_code = 0x10000u + ((_pf_code - 0xD800u) << 10) + + (_pf_decoded[++_pf_i] - 0xDC00u); + } + _pf_result.push_back(_pf_code); + } + return _pf_result; +} + +inline _pf_ranges _pf_normalize(_pf_ranges _pf_values) { + std::sort(_pf_values.begin(), _pf_values.end()); + _pf_ranges _pf_result; + for (const auto& _pf_value : _pf_values) { + if (!_pf_result.empty() && _pf_value.first <= _pf_result.back().second + 1) + _pf_result.back().second = std::max(_pf_result.back().second, _pf_value.second); + else + _pf_result.push_back(_pf_value); + } + return _pf_result; +} + +inline _pf_ranges _pf_complement(_pf_ranges _pf_values) { + _pf_ranges _pf_result; + uint32_t _pf_next = 0; + for (const auto& _pf_value : _pf_normalize(std::move(_pf_values))) { + if (_pf_next < _pf_value.first) + _pf_result.emplace_back(_pf_next, _pf_value.first - 1); + _pf_next = _pf_value.second + 1; + } + if (_pf_next <= 0x10FFFFu) _pf_result.emplace_back(_pf_next, 0x10FFFFu); + return _pf_result; +} + +inline std::string _pf_hex(uint32_t _pf_value) { + const char* _pf_digits = "0123456789abcdef"; + std::string _pf_result = "\\x"; + _pf_result += _pf_digits[(_pf_value >> 4) & 15]; + _pf_result += _pf_digits[_pf_value & 15]; + return _pf_result; +} + +inline std::string _pf_byte_range(uint32_t _pf_low, uint32_t _pf_high) { + if (_pf_low == _pf_high) return _pf_hex(_pf_low); + return "[" + _pf_hex(_pf_low) + "-" + _pf_hex(_pf_high) + "]"; +} + +inline std::vector _pf_bytes(uint32_t _pf_unit) { + if (_pf_unit < 0x80u) return {_pf_unit}; + if (_pf_unit < 0x800u) + return {0xC0u | (_pf_unit >> 6), 0x80u | (_pf_unit & 63)}; + if (_pf_unit < 0x10000u) + return {0xE0u | (_pf_unit >> 12), 0x80u | ((_pf_unit >> 6) & 63), + 0x80u | (_pf_unit & 63)}; + return {0xF0u | (_pf_unit >> 18), 0x80u | ((_pf_unit >> 12) & 63), + 0x80u | ((_pf_unit >> 6) & 63), 0x80u | (_pf_unit & 63)}; +} + +// The endpoints have the same encoding width. Split at differing prefixes; +// complete intervening prefixes share all their continuation-byte ranges. +// This builds a bounded trie for a range instead of enumerating its members. +inline void _pf_byte_trie(const std::vector& _pf_low, + const std::vector& _pf_high, + std::size_t _pf_at, const std::string& _pf_prefix, + std::vector& _pf_alternatives) { + if (_pf_at + 1 == _pf_low.size()) { + _pf_alternatives.push_back( + _pf_prefix + _pf_byte_range(_pf_low[_pf_at], _pf_high[_pf_at])); + return; + } + if (_pf_low[_pf_at] == _pf_high[_pf_at]) { + _pf_byte_trie(_pf_low, _pf_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_low[_pf_at]), _pf_alternatives); + return; + } + auto _pf_first_high = _pf_low; + auto _pf_last_low = _pf_high; + for (std::size_t _pf_i = _pf_at + 1; _pf_i < _pf_low.size(); ++_pf_i) { + _pf_first_high[_pf_i] = 0xBFu; + _pf_last_low[_pf_i] = 0x80u; + } + _pf_byte_trie(_pf_low, _pf_first_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_low[_pf_at]), _pf_alternatives); + if (_pf_low[_pf_at] + 1 < _pf_high[_pf_at]) { + std::string _pf_middle = _pf_prefix + + _pf_byte_range(_pf_low[_pf_at] + 1, _pf_high[_pf_at] - 1); + for (std::size_t _pf_i = _pf_at + 1; _pf_i < _pf_low.size(); ++_pf_i) + _pf_middle += "[\\x80-\\xbf]"; + _pf_alternatives.push_back(std::move(_pf_middle)); + } + _pf_byte_trie(_pf_last_low, _pf_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_high[_pf_at]), _pf_alternatives); +} + +inline std::string _pf_atom(_pf_ranges _pf_values) { + std::vector _pf_alternatives; + for (const auto& _pf_value : _pf_normalize(std::move(_pf_values))) { + uint32_t _pf_low = _pf_value.first; + for (uint32_t _pf_limit : {0x7Fu, 0x7FFu, 0xFFFFu, 0x10FFFFu}) { + if (_pf_low > _pf_value.second || _pf_low > _pf_limit) continue; + const auto _pf_high = std::min(_pf_value.second, _pf_limit); + _pf_byte_trie(_pf_bytes(_pf_low), _pf_bytes(_pf_high), + 0, "", _pf_alternatives); + _pf_low = _pf_high + 1; + } + } + if (_pf_alternatives.empty()) return "(?!)"; + std::string _pf_result = "(?:"; + for (const auto& _pf_alternative : _pf_alternatives) { + if (_pf_result.size() > 3) _pf_result += "|"; + _pf_result += _pf_alternative; + } + return _pf_result + ")"; +} + +inline _pf_ranges _pf_class_escape(uint32_t _pf_code) { + _pf_ranges _pf_result; + switch (_pf_code) { + case 'd': case 'D': + _pf_result = {{'0', '9'}}; + break; + case 'w': case 'W': + _pf_result = {{'0', '9'}, {'A', 'Z'}, {'_', '_'}, {'a', 'z'}}; + break; + case 's': case 'S': + _pf_result = {{9, 13}, {32, 32}}; + break; + default: throw _pf_invalid{}; + } + if (_pf_code == 'D' || _pf_code == 'W' || _pf_code == 'S') + return _pf_complement(std::move(_pf_result)); + return _pf_result; +} + +inline bool _pf_is_class_escape(uint32_t _pf_code) { + return _pf_code == 'd' || _pf_code == 'D' || _pf_code == 'w' + || _pf_code == 'W' || _pf_code == 's' || _pf_code == 'S'; +} + +struct _pf_member { + _pf_ranges _pf_values; + bool _pf_single; + uint32_t _pf_unit; +}; + +class _pf_pattern { + const std::vector& _pf_units; + std::size_t _pf_at = 0; + + uint32_t _pf_take() { + if (_pf_at == _pf_units.size()) throw _pf_invalid{}; + return _pf_units[_pf_at++]; + } + + uint32_t _pf_hex_unit(unsigned _pf_count) { + uint32_t _pf_value = 0; + while (_pf_count--) { + const auto _pf_code = _pf_take(); + const int _pf_digit = _pf_code >= '0' && _pf_code <= '9' + ? static_cast(_pf_code - '0') + : _pf_code >= 'a' && _pf_code <= 'f' ? static_cast(_pf_code - 'a') + 10 + : _pf_code >= 'A' && _pf_code <= 'F' ? static_cast(_pf_code - 'A') + 10 : -1; + if (_pf_digit < 0) throw _pf_invalid{}; + _pf_value = (_pf_value << 4) | static_cast(_pf_digit); + } + return _pf_value; + } + + uint32_t _pf_character_escape(uint32_t _pf_code) { + switch (_pf_code) { + case 'u': return _pf_hex_unit(4); + case 'x': return _pf_hex_unit(2); + case 'b': return 8; + case 'f': return 12; + case 'n': return 10; + case 'r': return 13; + case 't': return 9; + case 'v': return 11; + case '0': return 0; + case 'c': { + const auto _pf_letter = _pf_take(); + if (!((_pf_letter >= 'A' && _pf_letter <= 'Z') + || (_pf_letter >= 'a' && _pf_letter <= 'z'))) + throw _pf_invalid{}; + return _pf_letter & 31; + } + default: return _pf_code; + } + } + + uint32_t _pf_pair(uint32_t _pf_high) { + if (_pf_high < 0xD800u || _pf_high > 0xDBFFu) return _pf_high; + const auto _pf_saved = _pf_at; + uint32_t _pf_low; + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= 0xDC00u + && _pf_units[_pf_at] <= 0xDFFFu) { + _pf_low = _pf_take(); + } else if (_pf_at + 1 < _pf_units.size() && _pf_units[_pf_at] == '\\' + && _pf_units[_pf_at + 1] == 'u') { + _pf_at += 2; + _pf_low = _pf_hex_unit(4); + } else { + return _pf_high; + } + if (_pf_low < 0xDC00u || _pf_low > 0xDFFFu) { + _pf_at = _pf_saved; + return _pf_high; + } + return 0x10000u + ((_pf_high - 0xD800u) << 10) + (_pf_low - 0xDC00u); + } + + // Retain the existing byte engine's named/collating class syntax too. + // Its classic-locale classes are sets of ASCII units. + _pf_member _pf_named_member(uint32_t _pf_delimiter) { + std::string _pf_name; + for (;;) { + const auto _pf_code = _pf_take(); + if (_pf_code == _pf_delimiter && _pf_at < _pf_units.size() + && _pf_units[_pf_at] == ']') { + ++_pf_at; + break; + } + if (_pf_code > 127) throw _pf_invalid{}; + _pf_name += static_cast(_pf_code); + } + std::regex_traits _pf_traits; + _pf_traits.imbue(std::locale::classic()); + if (_pf_delimiter != ':') { + const auto _pf_value = _pf_traits.lookup_collatename( + _pf_name.begin(), _pf_name.end()); + if (_pf_value.size() != 1) throw _pf_invalid{}; + const auto _pf_unit = static_cast(_pf_value[0]); + if (_pf_delimiter == '=') { + const auto _pf_primary = _pf_traits.transform_primary( + _pf_value.begin(), _pf_value.end()); + _pf_ranges _pf_values; + for (uint32_t _pf_candidate = 0; _pf_candidate < 128; ++_pf_candidate) { + const std::string _pf_byte(1, static_cast(_pf_candidate)); + if (_pf_traits.transform_primary(_pf_byte.begin(), _pf_byte.end()) + == _pf_primary) + _pf_values.emplace_back(_pf_candidate, _pf_candidate); + } + return {std::move(_pf_values), false, 0}; + } + return {{{_pf_unit, _pf_unit}}, true, _pf_unit}; + } + const auto _pf_mask = _pf_traits.lookup_classname(_pf_name.begin(), _pf_name.end()); + if (_pf_mask == decltype(_pf_mask)()) throw _pf_invalid{}; + _pf_ranges _pf_values; + for (uint32_t _pf_unit = 0; _pf_unit < 128; ++_pf_unit) + if (_pf_traits.isctype(static_cast(_pf_unit), _pf_mask)) + _pf_values.emplace_back(_pf_unit, _pf_unit); + return {std::move(_pf_values), false, 0}; + } + + _pf_member _pf_class_member() { + auto _pf_code = _pf_take(); + if (_pf_code == '[' && _pf_at < _pf_units.size() + && (_pf_units[_pf_at] == ':' || _pf_units[_pf_at] == '.' + || _pf_units[_pf_at] == '=')) { + return _pf_named_member(_pf_take()); + } + uint32_t _pf_unit = _pf_code; + if (_pf_code == '\\') { + _pf_code = _pf_take(); + if (_pf_is_class_escape(_pf_code)) + return {_pf_class_escape(_pf_code), false, 0}; + _pf_unit = _pf_character_escape(_pf_code); + } + _pf_unit = _pf_pair(_pf_unit); + return {{{_pf_unit, _pf_unit}}, true, _pf_unit}; + } + + std::string _pf_bracket() { + const bool _pf_negate = _pf_at < _pf_units.size() && _pf_units[_pf_at] == '^'; + if (_pf_negate) ++_pf_at; + _pf_ranges _pf_values; + while (_pf_at < _pf_units.size() && _pf_units[_pf_at] != ']') { + auto _pf_left = _pf_class_member(); + if (_pf_at + 1 < _pf_units.size() && _pf_units[_pf_at] == '-' + && _pf_units[_pf_at + 1] != ']') { + ++_pf_at; + const auto _pf_right = _pf_class_member(); + if (!_pf_left._pf_single || !_pf_right._pf_single + || _pf_left._pf_unit > _pf_right._pf_unit) + throw _pf_invalid{}; + _pf_values.emplace_back(_pf_left._pf_unit, _pf_right._pf_unit); + } else { + _pf_values.insert(_pf_values.end(), _pf_left._pf_values.begin(), + _pf_left._pf_values.end()); + } + } + if (_pf_take() != ']') throw _pf_invalid{}; + return _pf_atom(_pf_negate ? _pf_complement(std::move(_pf_values)) + : std::move(_pf_values)); + } + +public: + explicit _pf_pattern(const std::vector& _pf_value) : _pf_units(_pf_value) {} + + std::string _pf_compile() { + std::string _pf_result; + while (_pf_at < _pf_units.size()) { + auto _pf_code = _pf_take(); + if (_pf_code == '[') { + _pf_result += _pf_bracket(); + } else if (_pf_code == '.') { + _pf_result += _pf_atom(_pf_complement({{10, 10}, {13, 13}, + {0x2028, 0x2029}})); + } else if (_pf_code == '\\') { + _pf_code = _pf_take(); + if (_pf_is_class_escape(_pf_code)) { + _pf_result += _pf_atom(_pf_class_escape(_pf_code)); + } else if (_pf_code == 'u' || _pf_code == 'x' || _pf_code > 127) { + const auto _pf_unit = _pf_pair(_pf_character_escape(_pf_code)); + _pf_result += _pf_atom({{_pf_unit, _pf_unit}}); + } else { + // Assertions, backreferences and ASCII character escapes + // retain the byte engine's grammar and capture numbering. + _pf_result += '\\'; + _pf_result += static_cast(_pf_code); + if (_pf_code == 'c') _pf_result += static_cast(_pf_take()); + if (_pf_code >= '1' && _pf_code <= '9') + while (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= '0' + && _pf_units[_pf_at] <= '9') + _pf_result += static_cast(_pf_take()); + } + } else if (_pf_code == '(') { + _pf_result += '('; + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '?') { + _pf_result += static_cast(_pf_take()); + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] < 128) + _pf_result += static_cast(_pf_take()); + } + } else if (_pf_code == '{') { + _pf_result += '{'; + while (_pf_at < _pf_units.size() + && ((_pf_units[_pf_at] >= '0' && _pf_units[_pf_at] <= '9') + || _pf_units[_pf_at] == ',')) + _pf_result += static_cast(_pf_take()); + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '}') + _pf_result += static_cast(_pf_take()); + } else if (_pf_code == ')' || _pf_code == '|' || _pf_code == '^' + || _pf_code == '$' || _pf_code == '*' || _pf_code == '+' + || _pf_code == '?' || _pf_code == '}') { + _pf_result += static_cast(_pf_code); + } else { + _pf_code = _pf_pair(_pf_code); + _pf_result += _pf_atom({{_pf_code, _pf_code}}); + } + } + return _pf_result; + } +}; + +inline std::string _pf_match(const std::string& _pf_source, + const std::string& _pf_regex, int64_t _pf_bar) { + const auto _pf_pattern_units = _pf_characters(_pf_regex); + std::string _pf_pattern_bytes; + try { + _pf_pattern_bytes = _pf_pattern(_pf_pattern_units)._pf_compile(); + } catch (const _pf_invalid&) { + // If the byte grammar already rejects this spelling, retain its + // exact runtime reason/context. A range invalid only in characters still + // needs the same coded invalid-pattern stop. + (void)::pineforge::pine_str_match(_pf_source, _pf_regex); + const std::string _pf_message = "Error on bar " + std::to_string(_pf_bar) + + ": Invalid regex pattern in the 'str.match' function."; + _PF_STRING_STOP("invalid_regex_pattern", _pf_message.c_str()); + } + std::string _pf_source_bytes; + for (const uint32_t _pf_unit : _pf_characters(_pf_source)) + for (const uint32_t _pf_byte : _pf_bytes(_pf_unit)) + _pf_source_bytes += static_cast(_pf_byte); + const auto _pf_matched = ::pineforge::pine_str_match(_pf_source_bytes, _pf_pattern_bytes); + const auto _pf_matched_units = _pf_u16_decode(_pf_matched); + return _pf_u16_encode(_pf_matched_units, 0, _pf_matched_units.size()); +} +} // namespace _pf_regex +""" diff --git a/pineforge_codegen/codegen/visit_call.py b/pineforge_codegen/codegen/visit_call.py index efb19c5c..be03c690 100644 --- a/pineforge_codegen/codegen/visit_call.py +++ b/pineforge_codegen/codegen/visit_call.py @@ -864,11 +864,11 @@ def _array_method_args( ] def _array_method_arg_nodes(self, method: str, node: FuncCall) -> list: - """Merge checked-array method kwargs into Pine signature order.""" + """Merge registered array method kwargs into Pine signature order.""" return self._collection_arg_nodes("array", method, node, False) def _collection_arg_nodes(self, namespace, method, node, functional): - """The arguments of a matrix or checked-array method call in Pine's + """The arguments of a matrix or registered array method call in Pine's parameter order (the receiver first in the namespace form). A keyword argument binds to its parameter's own slot, never shifted @@ -877,7 +877,7 @@ def _collection_arg_nodes(self, namespace, method, node, functional): receiver's own extent, which the method's template reads). A keyword the lowering does not consume -- an unknown name, a parameter given twice -- is refused, as is a gap before a required parameter. A call - without keywords, and an array method outside the checked-index + without keywords, and an array method outside the keyword signature table, keep their arguments as written.""" if not node.kwargs: return list(node.args) @@ -4212,8 +4212,14 @@ def slot(index: int): def _visit_str_call(self, func_name: str, node) -> str: if func_name == "format_time": return self._visit_format_time_call(node) + visit_arg = self._visit_expr + if func_name == "split": + # Both parameters are strings; use their contextual type for na + # just as for a string declaration or assignment. + visit_arg = lambda arg: self._visit_rhs_value( + arg, target_cpp_type="std::string") args = _merge_kwargs(node.args, node.kwargs, - sigs.get_param_names("str", func_name), self._visit_expr, + sigs.get_param_names("str", func_name), visit_arg, intrinsic=sigs.get_intrinsic("str", func_name)) arg_nodes = _merge_kwargs( node.args, node.kwargs, @@ -4280,12 +4286,44 @@ def _visit_str_call(self, func_name: str, node) -> str: if func_name == "substring": if len(args) in (2, 3): end_parameter = ", auto _pf_substring_end" if len(args) == 3 else "" - count = ", _pf_substring_end - _pf_substring_begin" if len(args) == 3 else "" + # Test sentinels in their original types: arithmetic can carry + # an int64_t na even when str.pos itself returned an int na. + end_guard = ( + "if (is_na(_pf_substring_end)) return na(); " + if len(args) == 3 else "" + ) + begin = na_preserving_int_cast("_pf_substring_begin", "int64_t") + count = "" + count_guard = "" + if len(args) == 3: + # Byte length bounds UTF-16 length from above. Beyond it, + # use the existing helper's end clamp without converting + # the end. Otherwise ordered nonnegative size_t operands + # give a count without signed overflow or unsigned wrap. + count_guard = ( + "if (_pf_substring_end < _pf_substring_start) " + 'throw std::out_of_range("str.substring: end_pos is less than begin_pos"); ' + "const std::size_t _pf_substring_count = " + "_pf_substring_end > _pf_substring_text.size() ? std::string::npos : " + "static_cast(_pf_substring_end) - " + "static_cast(_pf_substring_start); " + ) + count = ", _pf_substring_count" def checked_substring(bound): return ( f"[&](const std::string& _pf_substring_text, auto _pf_substring_begin{end_parameter}){{ " - "try { return _pf_u16_substr(_pf_substring_text, _pf_substring_begin" + f"{end_guard}" + "const int64_t _pf_substring_start = " + f"is_na(_pf_substring_begin) ? int64_t{{0}} : {begin}; " + "try { " + # Preserve the helper's existing invalid-begin stop, + # including its unsigned position and UTF-16 size. + "if (_pf_substring_start < 0) return _pf_u16_substr(" + "_pf_substring_text, static_cast(_pf_substring_start)); " + f"{count_guard}" + "return _pf_u16_substr(_pf_substring_text, " + "static_cast(_pf_substring_start)" f"{count}); }} catch (const std::out_of_range& _pf_substring_error) {{ " '_PF_STRING_STOP("substring_out_of_range", _pf_substring_error.what()); ' "return std::string(); } " @@ -4327,9 +4365,11 @@ def checked_substring(bound): return 'std::string("")' if func_name in STR_FUNC_MAP and STR_FUNC_MAP[func_name] is not None: - if func_name in ("length", "pos"): - # These two count in UTF-16 units through tv_string_units. + if func_name in ("length", "pos", "contains", "startswith", "endswith", "match"): + # Measurement/searches use units; matching decodes characters. self._uses_tv_string_units = True + if func_name == "match": + self._uses_unicode_regex = True return evaluate_args_once( args, STR_ARGS_READ_REPEATEDLY.get(func_name, ()), STR_FUNC_MAP[func_name], "_pf_str_a") diff --git a/pineforge_codegen/codegen/visit_expr.py b/pineforge_codegen/codegen/visit_expr.py index da1f99a1..6f30b5e4 100644 --- a/pineforge_codegen/codegen/visit_expr.py +++ b/pineforge_codegen/codegen/visit_expr.py @@ -91,6 +91,7 @@ from .run_stops import checked_history_offset, history_offset_guard, request_stop from .capacities import literal_history_offset +from ..analyzer.value_bindings import ValueBindings from ..errors import CodegenInvariantError, Phase from ..symbols import TypeSpec from ..external_requests import UNPINNED_ANNOTATION @@ -768,20 +769,30 @@ def _identifier_reads_series(self, node: Identifier) -> bool: if name in self._current_func_series_params: return True if (name in self._current_func_param_types - or (name in BAR_FIELDS and self._builtin_local_symbol(node) is None) or self._reads_builtin_variable(node)): return False return self._binding_is_series( name, self._call_site_var_name(node, self._safe_name(name))) def _reads_builtin_variable(self, node: Identifier) -> bool: - """Whether ``node`` reads a builtin variable (``second``, ``hour``, - ``bar_index``...) and not a user variable of the same name - (``float second = na``). A read the analyzer bound to a user - declaration in scope is the user's: it carries the annotation a - tuple-destructured name does, or ``builtin_shadow_binding``.""" + """Whether this read resolves to a builtin rather than an authored value. + + Bar fields follow the same lexical rule as other builtins. Preserve + the analyzer's exact binding, including an explicit builtin binding + before a later declaration. Copied reads retain authored-use metadata. + """ + name = node.name + if ((name not in BAR_FIELDS and name not in BAR_BUILTINS) + or name in self._current_func_param_types + or name in self._current_func_series_params): + return False + binding = ( + self.ctx.identifier_decl_bindings[id(node)] + if id(node) in self.ctx.identifier_decl_bindings + else ValueBindings.declaration_of(node) + ) annotations = node.annotations or {} - return (node.name in BAR_BUILTINS + return (binding is None and not annotations.get("tuple_builtin_binding") and not annotations.get("builtin_shadow_binding")) @@ -803,10 +814,8 @@ def _visit_ident(self, node: Identifier) -> str: if (retained and name in retained.aliases and not self._known_var_is_lexically_shadowed(name)): return f"(*{self._safe_name(name)})" - if name in BAR_FIELDS and self._builtin_local_symbol(node) is None: - return BAR_FIELDS[name] if self._reads_builtin_variable(node): - return BAR_BUILTINS[name] + return BAR_FIELDS[name] if name in BAR_FIELDS else BAR_BUILTINS[name] binding = self.ctx.identifier_decl_bindings.get(id(node)) primitive_shadow_read = ( binding is not None @@ -1966,6 +1975,24 @@ def _callable_has_ordered_effect(self, key: str) -> bool: ) return memo[key] + def _concat_type(self, operand) -> str: + """The scalar type of a concatenation operand, including series locals.""" + inferred = self._infer_type(operand) + if (inferred == "double" + and isinstance(operand, Identifier) + and operand.name in self._current_func_locals + and self._identifier_reads_series(operand) + and self._series_type_for(operand.name) == "std::string"): + # Series locals bypass the scalar local-type map. Their current + # slot has the same type as their history reads. + return "std::string" + return inferred + + def _string_concat_cpp(self, left: str, right: str) -> str: + """Join two rendered string values through their UTF-16 units.""" + self._uses_tv_string_units = True + return f"_pf_u16_concat({left}, {right})" + def _lower_binop(self, node: BinOp, left: str, right: str, widened: bool = False, na_checked: bool = False) -> str: """The C++ of ``node`` over its rendered operands; ``widened`` when @@ -1986,20 +2013,8 @@ def _lower_binop(self, node: BinOp, left: str, right: str, f"const bool _pf_v5_r = ({right}); " f"return _pf_v5_l {op} _pf_v5_r; }}())") if node.op == "+": - def _concat_type(operand): - inferred = self._infer_type(operand) - if (inferred == "double" - and isinstance(operand, Identifier) - and operand.name in self._current_func_locals - and self._identifier_reads_series(operand) - and self._series_type_for(operand.name) == "std::string"): - # Series locals bypass the scalar local-type map. Their - # current slot has the same type as their history reads. - return "std::string" - return inferred - - lt = _concat_type(node.left) - rt = _concat_type(node.right) + lt = self._concat_type(node.left) + rt = self._concat_type(node.right) if lt == "std::string" or rt == "std::string": def _as_string(rendered, inferred): if inferred == "std::string": @@ -2008,7 +2023,8 @@ def _as_string(rendered, inferred): return f'(({rendered}) ? std::string("true") : std::string("false"))' return f"std::to_string({rendered})" - return f"({_as_string(left, lt)} + {_as_string(right, rt)})" + return self._string_concat_cpp( + _as_string(left, lt), _as_string(right, rt)) if (node.op in ("+", "-", "*", "/", "%") and not widened and not na_checked and any(self._integer_na_operand(side) diff --git a/pineforge_codegen/codegen/visit_stmt.py b/pineforge_codegen/codegen/visit_stmt.py index ffdb77e0..4ef3d101 100644 --- a/pineforge_codegen/codegen/visit_stmt.py +++ b/pineforge_codegen/codegen/visit_stmt.py @@ -263,6 +263,11 @@ def _visit_stmt(self, node: ASTNode, lines: list[str], indent: int) -> None: ).get(id(node)) if metadata is not None: member_name = metadata[1] + if metadata[4] and metadata[2] == PineType.STRING: + # Persistent callable locals bypass ordinary scalar + # declarations. Activate their member's string family only + # after the initializer, for later reads and write targets. + self._current_func_local_types[node.name] = "std::string" if member_name != node.name: # The declaration RHS was emitted against the inherited # binding. Only now activate the collision-safe member for @@ -1152,6 +1157,7 @@ def remember_local_type(cpp_type: str | None) -> None: def _compound_assign_rhs(self, target_read: str, op: str, val_cpp: str, target_node=None, value_node=None, *, + target_cpp_type: str | None = None, target_is_double: bool = False, value_is_double: bool = False) -> str | None: """RHS for a compound assignment that must NOT lower to the C++ @@ -1161,10 +1167,18 @@ def _compound_assign_rhs(self, target_read: str, op: str, val_cpp: str, fmod-like — the binary-operator lowering in visit_expr casts both sides to double / uses std::fmod. ``a /= b`` and ``a %= b`` must match those semantics (C++ ``/=`` would do integer division on int - operands; ``%=`` does not even compile for doubles). Returns None - for operators where the native C++ compound form is correct - (+=, -=, *=). + operands; ``%=`` does not even compile for doubles). String ``+=`` + rejoins UTF-16 halves like string ``+``. The caller can supply its + emitted storage type when a requested binding differs from the chart + AST's type. Returns None when the native compound form is correct. """ + if (op == "+=" + and (target_cpp_type or self._concat_type(target_node)) == "std::string"): + # A bare na takes the string slot's type; other values keep the + # assignment writer's already-rendered RHS. + if self._is_na_expr(value_node): + val_cpp = "na()" + return self._string_concat_cpp(target_read, val_cpp) # Requested helper state already stores numeric values as doubles. # Its caller supplies the emitted operand families after converting # the RHS; the chart AST's int hint cannot type those series reads. diff --git a/tests/_older_codegen.py b/tests/_older_codegen.py index e123ba47..37ba7323 100644 --- a/tests/_older_codegen.py +++ b/tests/_older_codegen.py @@ -73,9 +73,13 @@ def with_utf16_units(cpp: str) -> str: The older emission counted ``str.length`` through ``(int)text.length()`` and carried no unit helpers. The calls become calls of the helper and the helper block is added before the syminfo helpers, where the current emitter puts - it. A text without such a call is returned as it is.""" + it. A repaired string concatenation also needs the block. A text that + already carries the current block is not given a second copy.""" spelled, sites = _spell_lengths(cpp) - if not sites: + if not sites and "_pf_u16_concat(" not in spelled: return cpp + if "inline int _pf_u16_length(" in spelled: + assert TV_STRING_UNITS_CPP in spelled, spelled + return spelled assert spelled.count(_SYMINFO_HELPERS) == 1, spelled return spelled.replace(_SYMINFO_HELPERS, TV_STRING_UNITS_CPP + "\n\n" + _SYMINFO_HELPERS) diff --git a/tests/fixtures/string_pos_na/README.md b/tests/fixtures/string_pos_na/README.md new file mode 100644 index 00000000..d96a76c1 --- /dev/null +++ b/tests/fixtures/string_pos_na/README.md @@ -0,0 +1,143 @@ +# String position and boundary fixture + +`strategy.pine` checks `str.pos` when a needle is absent, found, empty, or +longer than its text. It checks missing results through `na()` and `nz()`, +using both literal arguments and arguments that change on alternate bars. + +The public source is the reporter's observations in +[issue #382](https://github.com/pineforge-4pass/pineforge-codegen-oss/issues/382): +item 5 for positions, item 9 for substring boundaries, and item 8 for trimming. +**`strategy.pine` has no captured TradingView trade tape.** Its expected report +is derived from those public observations, not a new TradingView capture. +Related substring and trim cases live in `tests/test_string_units.py`; the +strategy itself checks positions only. + +## Expected position values + +| Expression | Expected result | +| --- | --- | +| `str.pos("abc", "x")` | integer `na` | +| `na(str.pos("abc", "x"))` | `true` | +| `nz(str.pos("abc", "x"), 0)` | `0` | +| `str.tostring(str.pos("abc", "x"))` | `"NaN"`: generated-string contract, without a separate reported TradingView measurement | + +The related native tests include these runtime inputs. The Unicode positions +are derived from UTF-16 units, where an emoji occupies two units; they are not +separate TradingView measurements of these exact inputs. The `"NaN"` entries +all use the generated-string contract above. + +| Text | Needle | `na(p)` | `nz(p, 0)` | `str.tostring(p)` | +| --- | --- | --- | ---: | --- | +| `"abc"` | `"x"` | `true` | 0 | `"NaN"` | +| `"seed42"` | `"?"` | `true` | 0 | `"NaN"` | +| `"abc"` | `"b"` | `false` | 1 | `"1"` | +| `"abc"` | `"a"` | `false` | 0 | `"0"` | +| `"abc"` | `""` | `false` | 0 | `"0"` | +| `""` | `"x"` | `true` | 0 | `"NaN"` | +| `""` | `""` | `false` | 0 | `"0"` | +| `"ab"` | `"abc"` | `true` | 0 | `"NaN"` | +| `"a😀b"` | `"b"` | `false` | 3 | `"3"` | +| `"中😀tail"` | `"l"` | `false` | 6 | `"6"` | + +The strategy opens a position on its first bar and closes it on its second. +The exact expected exit-comment report is: + +```text +literal_na=true|literal_nz=0|variable_na=true|variable_nz=0|long_na=true|long_nz=0|start=0|found=1|empty=0 +``` + +For comparison, the report derived from affected-version behavior is: + +```text +literal_na=false|literal_nz=-1|variable_na=false|variable_nz=-1|long_na=false|long_nz=-1|start=0|found=1|empty=0 +``` + +## Substring observations and controls + +| Form | Expected result | +| --- | --- | +| `str.substring("abc", str.pos("abc", ":") + 1)` | `"abc"`, completed run | +| Substring with `na` start and omitted end | Whole text, completed run | +| Substring with `na` end | String `na`, completed run | +| `str.substring("abc", 2, j)` with finite runtime `j = 1` | Run-time stop; detail `end_pos is less than begin_pos` | + +Item 9 reports undefined affected-version behavior that depends on platform +and compiled context. With runtime positions, releases 1.10.0 and 1.11.0 +produced: + +| Form on `"abc"` | Affected-version x86-64 output | Affected-version ARM64 output | TradingView observation | +| --- | --- | --- | --- | +| Start 2, end 1 | `"c"` | Empty text | Run-time error | +| Start -1, omitted end | Run-time error | `"abc"` | Run-time error | +| Start 0, end `na` | `"abc"` | Empty text, reported as `na()` | String `na` | +| Start `na`, omitted end | `"abc"` | `"abc"` | `"abc"` | + +The affected-version results involving invalid or missing positions are not +guaranteed behavior. In particular, matching a TradingView value through an +undefined conversion does not make the result reliable. + +Finite valid slices must also complete successfully: + +| Expression | Expected result | +| --- | --- | +| `str.substring("abc", 1, 2)` | `"b"`, UTF-8 hex `62` | +| `str.substring("a😀b", 1, 3)` | Whole emoji, UTF-8 hex `f09f9880` | +| `str.substring("abc", 1, 9)` | Clamped `"bc"`, length 2 | +| `str.substring("abc", 3, 3)` | Empty text, length 0 | + +The native representation stores string `na` as an empty string, so it cannot +distinguish a missing string from an authored empty string. + +## Trim observations + +Item 8 reports these exact TradingView results: + +| Text passed to `str.trim` | Expected result | +| --- | --- | +| U+00A0, `a`, U+00A0 | `"a"` | +| U+2003, `a`, U+2003 | `"a"` | +| U+FEFF, `a` | `"a"` | +| Vertical tab U+000B, `a`, form feed U+000C | `"a"` | + +Reported as removed at the ends: no-break space U+00A0, em space U+2003, +ideographic space U+3000, Ogham space mark U+1680, line separator U+2028, +zero-width no-break space U+FEFF, vertical tab U+000B, and form feed U+000C, +in addition to space, tab, line feed, and carriage return. + +Reported as kept: next-line U+0085, unit separator U+001F, zero-width space +U+200B, and Mongolian vowel separator U+180E. Interior characters are kept. +The tests compose these characters into literal and runtime-input cases, +including mixed boundaries and empty text; those compositions are derived +contracts, not separate TradingView measurements or a full Unicode whitespace +inventory. + +## How to run + +Open `strategy.pine` in TradingView's Pine Editor and add it to a chart with +at least two bars. In the strategy's trade list, inspect the closing order's +comment and compare it with the expected report above. + +To run the related native checks, configure the engine headers, generated +headers, built runtime, Eigen, and compiler as described in the repository +setup, then run: + +```sh +PINEFORGE_REQUIRE_NATIVE=1 python -m pytest -q -ra tests/test_string_units.py +``` + +`PINEFORGE_REQUIRE_NATIVE=1` makes missing native inputs fail instead of skip. + +## Unmeasured limits + +- A substring with an `na` start and an explicit finite end has no reported + TradingView observation or native case here. +- The variable idiom `p = str.pos(s, ":")` followed by + `str.substring(s, p + 1)` has no native case here. +- Recorded native runs used ARM64. Finite slicing under UBSan, a sanitizer + control demonstrating the old overflow, a sanitized runtime archive, and + native behavior on x86-64 have not been established by these checks. The + existing four UBSan cases return at the `na`-end guard before finite arithmetic. +- Boundary and consumer cases use `-O0` and `-O2`; the original position cases + use `-O0`. Broader `-O2` coverage, `request.security`, array-index and plot + compositions, surrogate-half search, and other issue #382 string functions + are outside this fixture's coverage. diff --git a/tests/fixtures/string_pos_na/strategy.pine b/tests/fixtures/string_pos_na/strategy.pine new file mode 100644 index 00000000..21718cf6 --- /dev/null +++ b/tests/fixtures/string_pos_na/strategy.pine @@ -0,0 +1,35 @@ +//@version=6 +// Issue #382 item 5. Candidate public parity probe; no captured TV tape yet. +strategy("382 str.pos missing match", overlay=true, process_orders_on_close=true) + +// The literal forms are the public expressions reported in the issue packet. +literalNa = na(str.pos("abc", "x")) +literalNz = nz(str.pos("abc", "x"), 0) + +// Both arguments vary at run time; neither possible needle occurs in its source. +source = bar_index % 2 == 0 ? "abc" : "ab" +needle = bar_index % 2 == 0 ? "x" : "z" +variablePos = str.pos(source, needle) +variableNa = na(variablePos) +variableNz = nz(variablePos, 0) + +longPos = str.pos("abc", "abcdef") +longNa = na(longPos) +longNz = nz(longPos, 0) + +// ASCII controls: a match at zero, another found position and an empty needle. +startPos = str.pos("abc", "a") +foundPos = str.pos("abc", "b") +emptyPos = str.pos("abc", "") + +report = "literal_na=" + str.tostring(literalNa) + "|literal_nz=" + str.tostring(literalNz) +report += "|variable_na=" + str.tostring(variableNa) + "|variable_nz=" + str.tostring(variableNz) +report += "|long_na=" + str.tostring(longNa) + "|long_nz=" + str.tostring(longNz) +report += "|start=" + str.tostring(startPos) + "|found=" + str.tostring(foundPos) + "|empty=" + str.tostring(emptyPos) + +if bar_index == 0 + strategy.entry("probe", strategy.long, qty=1) +if bar_index == 1 + strategy.close("probe", comment=report) + +plot(close) diff --git a/tests/test_array_terminal_returns.py b/tests/test_array_terminal_returns.py index df220933..327e7047 100644 --- a/tests/test_array_terminal_returns.py +++ b/tests/test_array_terminal_returns.py @@ -1,8 +1,9 @@ -"""Exact primitive UDF return typing for direct terminal array ``get`` calls.""" +"""Primitive UDF return typing for terminal array calls and forwarding.""" from __future__ import annotations from itertools import product +import re import pytest @@ -15,6 +16,145 @@ from tests._legacy_cpp import without_source_rename +@pytest.mark.parametrize( + "expression, separator_cpp", + [ + pytest.param("values.join(separator)", "separator", id="method-positional"), + pytest.param( + "values.join(separator=separator)", "separator", id="method-named", + ), + pytest.param("values.join()", 'std::string(",")', id="method-default"), + pytest.param( + "array.join(values, separator)", "separator", id="namespace-positional", + ), + pytest.param( + "array.join(values, separator=separator)", "separator", + id="namespace-mixed", + ), + pytest.param( + "array.join(id=values, separator=separator)", "separator", + id="namespace-named", + ), + pytest.param( + "array.join(separator=separator, id=values)", "separator", + id="namespace-reordered", + ), + pytest.param( + "values.copy().join(separator)", "separator", id="temporary-receiver", + ), + pytest.param( + "array.join(values)", 'std::string(",")', id="namespace-default", + ), + pytest.param( + "array.join(id=values)", 'std::string(",")', id="namespace-named-default", + ), + ], +) +@pytest.mark.parametrize( + "element_type, elements", + [("string", '"alpha", "beta"'), ("int", "2, 7")], +) +def test_join_helpers_and_forwarded_results_keep_string_signatures( + expression, separator_cpp, element_type, elements, +): + source = f'''//@version=6 +strategy("array join return types") +combine(array<{element_type}> values, string separator) => + {expression} +relay(array<{element_type}> values, string separator) => + combine(values=values, separator=separator) +parts = array.from({elements}) +first_result = relay(parts, "|") +second_result = relay(separator=":", values=parts) +plot(close) +''' + cpp = transpile(source) + for name in ("combine", "relay"): + assert f"std::string {name}(" in cpp, cpp + for name in ("first_result", "second_result"): + assert re.search(rf"\bstd::string {name}\s*[=;]", cpp), cpp + # A string signature alone still compiles when a named join emits + # ``return 0`` (a null C string), or silently takes the default separator. + if element_type == "string": + assert f"const std::string _pf_join_separator = ({separator_cpp});" in cpp, cpp + assert "_pf_u16_join(_pf_join_parts, _pf_join_separator)" in cpp, cpp + else: + assert f"__pf_r+={separator_cpp};" in cpp, cpp + compile_cpp(cpp, label="array-join-return-types") + + +@pytest.mark.parametrize( + "expression", + [ + pytest.param( + "array.join(make_parts(), make_separator())", id="namespace-positional", + ), + pytest.param( + "array.join(make_parts(), separator=make_separator())", + id="namespace-mixed", + ), + pytest.param( + "array.join(id=make_parts(), separator=make_separator())", + id="namespace-named", + ), + pytest.param( + "array.join(separator=make_separator(), id=make_parts())", + id="namespace-reordered", + ), + pytest.param( + "make_parts().join(make_separator())", id="method-positional", + ), + pytest.param( + "make_parts().join(separator=make_separator())", id="method-named", + ), + ], +) +def test_join_argument_emission_binds_receiver_then_separator_once(expression): + source = f'''//@version=6 +strategy("array join argument bindings") +events = array.new_int() +make_parts() => + events.push(1) + array.from("alpha", "beta") +make_separator() => + events.push(2) + "|" +result = {expression} +plot(close) +''' + cpp = transpile(source) + assignment, = ( + line for line in cpp.splitlines() + if line.strip().startswith("result =") + ) + assert assignment.count("make_parts()") == 1, assignment + assert assignment.count("make_separator()") == 1, assignment + assert assignment.index("make_parts()") < assignment.index("make_separator()") + assert "const auto& _pf_join_parts = (" in assignment, assignment + assert "const std::string _pf_join_separator = (make_separator());" in assignment, assignment + assert "_pf_u16_join(_pf_join_parts, _pf_join_separator)" in assignment, assignment + compile_cpp(cpp, label="array-join-argument-bindings") + + +def test_join_result_keeps_string_type_through_local_and_method_returns(): + source = '''//@version=6 +strategy("array join local and method returns") +method combine(array values, string separator) => + joined_value = values.join(separator) + joined_value +relay(array values, string separator) => + values.combine(separator) +parts = array.from("alpha", "beta") +result = relay(parts, "|") +plot(close) +''' + cpp = transpile(source) + assert re.search(r"\bstd::string _udt_array_string_+combine\(", cpp), cpp + assert "std::string relay(" in cpp + assert re.search(r"\bstd::string result\s*[=;]", cpp), cpp + compile_cpp(cpp, label="array-join-local-method-return") + + _GET_SHAPES_SOURCE = r'''//@version=6 strategy("Terminal array get shapes") diff --git a/tests/test_binop_operand_order.py b/tests/test_binop_operand_order.py index 68327fee..b23ec68f 100644 --- a/tests/test_binop_operand_order.py +++ b/tests/test_binop_operand_order.py @@ -47,20 +47,20 @@ def test_mutating_operand_is_evaluated_after_the_left_operand_is_bound(): s6 = _assignment(cpp, "s6") bound = _BINDING.findall(s6) # The + that adds str.tostring(a.remove(-1)) and the + that adds - # array.join(a, ","); the + of a literal "," and the effect-free + of the - # two reads before them keep their spelling. + # array.join(a, ","); literal separators and the effect-free join of the + # two reads before them need no operand-order binding. assert len(bound) == 2, s6 # The inner binding's right operand comes first in the text. before_remove, before_join = sorted( - s6.index(f"return ({token} + ") for token in bound + s6.index(f"return _pf_u16_concat({token}, ") for token in bound ) # a.get(-1) is read into the inner binding, then a.remove(-1) erases. assert s6.index("return (__pf_array[(size_t)__pf_array_index]);") < before_remove assert s6.index(".erase(") > before_remove # array.join(a, ",") reads the array after the outer binding. assert s6.index("std::to_string(a[") > before_join - # Operands no other operand changes keep the plain C++ operator: s4's - # two reads, and every entry id's concatenation. + # Operands no other operand changes need no order binding: s4's two + # reads, and every entry id's concatenation. assert "__pf_binop_lhs_" not in _assignment(cpp, "s4") assert cpp.count("__pf_binop_lhs_") == s6.count("__pf_binop_lhs_") @@ -111,7 +111,9 @@ def test_user_calls_methods_and_payloads_bind_their_left_operand(): ): line = _assignment(cpp, target) (token,) = _BINDING.findall(line) - assert (line.index(bound) < line.index(f"return ({token} ") + returned = (f"return ({token} " if target == "through_call" + else f"return _pf_u16_concat({token}, ") + assert (line.index(bound) < line.index(returned) < line.index(after)), line # A bool operand is bound as a bool: a std::vector element is a # proxy of the bit, which the remove moves (a bound proxy stays right only @@ -123,7 +125,7 @@ def test_user_calls_methods_and_payloads_bind_their_left_operand(): # A request.security payload binds its left operand the same way. payload = _assignment(cpp, "_req_sec_0") (token,) = _BINDING.findall(payload) - assert payload.index(".erase(") > payload.index(f"return ({token} + ") + assert payload.index(".erase(") > payload.index(f"return _pf_u16_concat({token}, ") def test_operands_are_evaluated_left_to_right_at_run_time(): diff --git a/tests/test_builtin_name_shadow.py b/tests/test_builtin_name_shadow.py index 2835f73c..276754a7 100644 --- a/tests/test_builtin_name_shadow.py +++ b/tests/test_builtin_name_shadow.py @@ -6,7 +6,12 @@ of the builtin itself keeps its emitted text. """ +import re + +import pytest + from pineforge_codegen import transpile +from pineforge_codegen.codegen.tables import BAR_BUILTINS, BAR_FIELDS HEADER = '//@version=6\nstrategy("builtin name shadow", initial_capital = 100000)\n' @@ -36,3 +41,109 @@ def test_read_of_the_builtin_variable_keeps_its_emitted_text(): cpp = transpile(HEADER + "float s = second + minute\nplot(s)\n") assert "pine_second(current_bar_.timestamp, syminfo_.timezone)" in cpp assert "pine_minute(current_bar_.timestamp, syminfo_.timezone)" in cpp + + +@pytest.mark.parametrize("name", sorted(BAR_FIELDS.keys() | BAR_BUILTINS.keys())) +def test_string_binding_types_repeated_constructor_and_scalar_operands(name): + cpp = transpile( + HEADER + + 'parts = str.split("first|second", "|")\n' + + f"{name} = array.get(parts, bar_index % 2)\n" + + f"values = array.from({name}, {name})\n" + + f'joined = {name} + ":"\n' + + "plot(1)\n" + ) + # Both elements must read the inferred string declaration, including + # builtin names whose C++ spelling needs escaping. + initializer = re.search( + r"values = std::vector\{([A-Za-z_]\w*), \1\};", + cpp, + ) + assert initializer is not None, cpp + binding = initializer.group(1) + assert f"std::string {binding}" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("joined = ") + ] + assert len(assignments) == 1 + assert binding in assignments[0] + # Correct spelling with a numeric inferred type would still emit an + # invalid std::to_string call around the string operand. + assert "std::to_string(" not in assignments[0] + + +@pytest.mark.parametrize("name, builtin", sorted(BAR_FIELDS.items())) +def test_bar_read_before_global_declaration_keeps_its_source_binding(name, builtin): + cpp = transpile( + HEADER + + f"before = {name}\n" + + f"{name} = str.tostring(bar_index)\n" + + f'after = {name} + ":"\n' + + "plot(before)\n" + ) + assert f"before = {builtin};" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("after = ") + ] + assert len(assignments) == 1 + assert builtin not in assignments[0] + assert "std::to_string(" not in assignments[0] + + +@pytest.mark.parametrize("name, builtin", sorted(BAR_FIELDS.items())) +@pytest.mark.parametrize("block", ["if", "while", "switch"]) +def test_bar_builtin_returns_after_a_string_block_binding_expires(name, builtin, block): + body = ( + f" {name} = str.tostring(bar_index)\n" + f' seen := {name} + ":"\n' + ) + if block == "switch": + branch = "switch\n bar_index >= 0 =>\n" + "".join( + " " + line for line in body.splitlines(keepends=True) + ) + elif block == "while": + branch = "while bar_index >= 0\n" + body + " break\n" + else: + branch = "if bar_index >= 0\n" + body + cpp = transpile( + HEADER + + f"before = {name}\n" + + 'seen = ""\n' + + branch + + f"after = {name}\n" + + "plot(before + after)\n" + ) + assert "double before" in cpp + assert "double after" in cpp + assert f"before = {builtin};" in cpp + assert f"after = {builtin};" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("seen = ") and 'std::string("")' not in line + ] + assert len(assignments) == 1 + assert builtin not in assignments[0] + assert "std::to_string(" not in assignments[0] + + +def test_qualified_member_and_other_bar_builtin_keep_their_bindings(): + cpp = transpile( + HEADER + + "type Holder\n string high\n" + + 'holder = Holder.new("member")\n' + + 'parts = str.split("first|second", "|")\n' + + "high = array.get(parts, bar_index % 2)\n" + + "seen = holder.high + high\n" + + "builtin = low\nplot(builtin)\n" + ) + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("seen = ") + ] + assert len(assignments) == 1 + assert ".high" in assignments[0] + assert "current_bar_." not in assignments[0] + assert "std::to_string(" not in assignments[0] + assert "builtin = current_bar_.low;" in cpp diff --git a/tests/test_codegen_new.py b/tests/test_codegen_new.py index f93b26c2..beddec7e 100644 --- a/tests/test_codegen_new.py +++ b/tests/test_codegen_new.py @@ -782,20 +782,27 @@ def test_map_codegen(): def test_str_contains(): src = 'x = str.contains("hello world", "world")\n' cpp = _generate(src) - assert "find" in cpp - assert "npos" in cpp + assert "_pf_u16_contains(_pf_search_source, _pf_search_needle)" in cpp def test_str_lower(): src = 'x = str.lower("HELLO")\n' cpp = _generate(src) - assert "tolower" in cpp + # The lowering carries a complete Unicode table even for an ASCII source. + assert "_pf_tolower_map" in cpp + assert "{0xC9u, 0x1u, 0xE9u}" in cpp + assert "{0x10400u, 0x1u, 0x10428u}" in cpp + assert "::tolower" not in cpp def test_str_upper(): src = 'x = str.upper("hello")\n' cpp = _generate(src) - assert "toupper" in cpp + assert "_pf_toupper_map" in cpp + assert "{0xE9u, 0x1u, 0xC9u}" in cpp + assert "{0xDFu, 0x2u, 0x53u, 0x53u}" in cpp + assert "{0x10428u, 0x1u, 0x10400u}" in cpp + assert "::toupper" not in cpp def test_str_length(): @@ -808,20 +815,19 @@ def test_str_length(): def test_str_startswith(): src = 'x = str.startswith("hello", "he")\n' cpp = _generate(src) - assert "substr" in cpp + assert "_pf_u16_startswith(_pf_search_source, _pf_search_needle)" in cpp def test_str_endswith(): src = 'x = str.endswith("hello", "lo")\n' cpp = _generate(src) - assert "compare" in cpp + assert "_pf_u16_endswith(_pf_search_source, _pf_search_needle)" in cpp def test_str_pos(): src = 'x = str.pos("hello", "ll")\n' cpp = _generate(src) - assert "find" in cpp - assert "npos" in cpp + assert "_pf_u16_pos(_pf_search_source, _pf_search_needle)" in cpp def test_str_substring(): @@ -845,9 +851,40 @@ def test_str_replace(): def test_str_trim(): - src = 'x = str.trim(" hello ")\n' - cpp = _generate(src) - assert "erase" in cpp + from tests._compile import run_emitted_tu + + src = ( + '//@version=6\n' + 'strategy("trim runtime input")\n' + 'source_value = input.string("unused default", "Text")\n' + 'x = str.trim(source_value)\n' + 'plot(close)\n' + ) + cpp = transpile(src) + driver = r''' +#include +#include + +int main() { + GeneratedStrategy strategy; + const char source_bytes[] = "\t \302\240hello world\343\200\200\n"; + strategy.set_input("Text", std::string(source_bytes, sizeof(source_bytes) - 1)); + pineforge::Bar bar{}; + bar.open = bar.high = bar.low = bar.close = 100.0; + bar.volume = 1.0; + bar.timestamp = 1704067200000LL; + strategy.run(&bar, 1, "1", "1"); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << "\n"; + return 14; + } + std::cout << strategy.x << "\n"; + return 0; +} +''' + assert run_emitted_tu( + cpp, driver, opt="-O0", label="str.trim runtime input", + ) == "hello world\n" def test_str_repeat(): @@ -1291,13 +1328,28 @@ def test_str_format_uses_runtime(): assert "pine_str_format" in cpp -def test_str_match_uses_runtime(): - cpp = _generate(""" -//@version=6 -strategy("T") -x = str.match("hello123", "\\\\d+") -""") - assert "pine_str_match" in cpp +@pytest.mark.parametrize("call", [ + 'str.match("hello123", "\\\\d+")', + 'str.match("aéb", "a[é]b")', + 'str.match(regex="a[à-ÿ]b", source="aéb")', + 'str.match("😀", ".")', + 'str.match(regex="a[😀-😇]b", source="a😀b")', +]) +def test_str_match_uses_runtime(call): + cpp = _generate(f'x = {call}\ny = {call}\n') + assert "return ::_pf_regex::_pf_match(" in cpp + assert cpp.count("inline std::string _pf_match(") == 1 + assert "::pineforge::pine_str_match(_pf_source_bytes, _pf_pattern_bytes)" in cpp + + +def test_str_match_evaluates_both_arguments_once_in_parameter_order(): + from pineforge_codegen.codegen.tables import STR_FUNC_MAP + + expression = STR_FUNC_MAP["match"](["next_source()", "next_pattern()"]) + assert expression.count("next_source()") == 1 + assert expression.count("next_pattern()") == 1 + assert expression.index("next_source()") < expression.index("next_pattern()") + assert expression.index("next_pattern()") < expression.index("return ::_pf_regex::_pf_match(") def test_str_split_uses_runtime(): diff --git a/tests/test_cpp_allowlist.py b/tests/test_cpp_allowlist.py index ac8c82ba..9196a3fd 100644 --- a/tests/test_cpp_allowlist.py +++ b/tests/test_cpp_allowlist.py @@ -28,12 +28,14 @@ # predates the unconditional settings and run-stop blocks, so it cannot show # those paths. These fixtures are transpiled here, against the current # emitter, for the positive corpus: settings/inputs (which emit ``<::``), -# timezone use (the sealed timestamp calendar run), drawing and request. +# timezone use (the sealed timestamp calendar run), drawing, request and the +# late-emitted ordinary-byte regex helpers. CURRENT_CODEGEN_FIXTURES = ( "tests/fixtures/matrix_eigen_pca.pine", "tests/fixtures/input_metadata_numeric.pine", "tests/fixtures/drawing_lifetime/w9dg-label-collect/strategy.pine", "tests/fixtures/request_discovery/case_strategy.pine", + "tests/gate-corpus/ok/validation__ta-str-match-regex-filter-01.pine", ) diff --git a/tests/test_foreign_footprint.py b/tests/test_foreign_footprint.py index 4f1ad221..33df4988 100644 --- a/tests/test_foreign_footprint.py +++ b/tests/test_foreign_footprint.py @@ -25,7 +25,8 @@ from pineforge_codegen import transpile, transpile_full from pineforge_codegen.errors import CompileError -from tests._compile import compile_cpp +from tests._compile import compile_cpp, run_emitted_tu +from tests._cpp_tokens import string_values from tests._e2e import chart_feed_head, skip_unless_e2e_env from tests._requests import build, hourly, run, same, write_root @@ -135,3 +136,116 @@ def test_footprint_of_the_charts_own_symbol_reads_its_feed(tmp_path_factory): assert same(result.trace[chart_open]["pf_d"], deltas[visible[-1]]), chart_open result = run(engine, work, feed_path, None, tag="none") assert not result.ok and PINNED in result.error + + +@pytest.mark.parametrize("names", [ + ("fp\0left", "fp\0right", "fp"), + ("fp\x01left", "fp\x01right", "fp"), + ("fpleft", "fpright", "fp"), + ("é\0中😀left", "é\0中😀right", "é"), + ("", "suffix", "missing"), +], ids=["nul", "other-control", "ascii", "unicode-nul", "empty-name"]) +def test_generated_column_helpers_compare_whole_names(names): + """Internal serializer/lookup contract; public Pine produces ASCII names.""" + from pineforge_codegen import CodeGen, _generate + from pineforge_codegen.external_requests import FOOTPRINT_COLUMN_ANNOTATION + + # Supply an internal annotation as serializer input, after normal checking. + # No Pine source or public footprint-name producer is changed or bypassed. + _, context, _, _, _ = _generate(HEAD + REQUEST + "d = fp.delta()\n", True, "") + (site,) = context.security_calls + assert site.expression.annotations[FOOTPRINT_COLUMN_ANNOTATION] == "fp_delta_100_70" + site.expression.annotations[FOOTPRINT_COLUMN_ANNOTATION] = names[0] + cpp = CodeGen(context).generate() + registration = next(line for line in cpp.splitlines() + if "if (_pf_symbol_data_installed(" in line) + payload = next(line for line in cpp.splitlines() if "_req_sec_0 = _pf_symbol_column(" in line) + for line in (registration, payload): + assert names[0] in string_values(line) + if "\0" in names[0]: + counted = re.findall(r'std::string\(("(?:[^"\\]|\\.)*"), (\d+)\)', line) + assert any( + string_values(literal) == [names[0]] + and int(count) == len(names[0].encode("utf-8")) + for literal, count in counted + ), line + + def native_bytes(value): + return "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + + body = "\n".join( + f'check("name{index}", probe._pf_symbol_data_installed("PF:A", "15", ' + f'{native_bytes(name)}));' + for index, name in enumerate(names) + ) + driver = r''' +#include +#include +#include +#include +#include + +class ColumnProbe : public GeneratedStrategy { +public: + using GeneratedStrategy::_pf_symbol_data_installed; + using GeneratedStrategy::_pf_symbol_column; + + void install(const std::string& left, const std::string& right, + const std::vector& bars, + const std::vector& closes) { + if (!set_symbol_feed("PF:A", "15", bars.data(), closes.data(), bars.size())) + throw std::runtime_error("could not install column test feed"); + auto& feed = symbol_feeds_.back(); + feed.columns.emplace_back(); + feed.columns.back().name = left; + feed.columns.back().values = {1.25, -4.5, 19.75}; + feed.columns.emplace_back(); + feed.columns.back().name = right; + feed.columns.back().values = {7.0, 11.0, -23.5}; + } +}; + +static_assert(std::is_invocable_r_v); + +void check(const char* name, bool found) { + std::cout << name << "=" << found << "\n"; +} + +int main() { + std::vector bars(3); + std::vector closes; + for (std::size_t i = 0; i < bars.size(); ++i) { + bars[i].open = bars[i].low = 90.0 + double(i); + bars[i].high = 110.0 + double(i); + bars[i].close = 100.0 + double(i); + bars[i].volume = 1.0; + bars[i].timestamp = 1704067200000LL + static_cast(i) * 900000LL; + closes.push_back(bars[i].timestamp + 900000LL); + } + ColumnProbe probe; + probe.install(LEFT, RIGHT, bars, closes); + // BODY + check("no-column-filter", probe._pf_symbol_data_installed("PF:A", "15")); + check("empty-name", probe._pf_symbol_data_installed("PF:A", "15", std::string{})); + check("other-symbol", probe._pf_symbol_data_installed("PF:B", "15", LEFT)); + check("other-timeframe", probe._pf_symbol_data_installed("PF:A", "60", LEFT)); + check("unknown-context-na", std::isnan(probe._pf_symbol_column(-1, LEFT))); + probe.run(bars.data(), bars.size(), "15", "15"); + if (!probe.last_error().empty()) throw std::runtime_error(probe.last_error()); + std::cout << "payload=" << probe.d << "\n"; + std::cout << "left=" << probe._pf_symbol_column(0, LEFT) << "\n"; + std::cout << "right=" << probe._pf_symbol_column(0, RIGHT) << "\n"; + check("unknown-column-na", std::isnan(probe._pf_symbol_column(0, MISSING))); +} +'''.replace("LEFT", native_bytes(names[0])).replace("RIGHT", native_bytes(names[1])) + driver = driver.replace("MISSING", native_bytes(names[2])).replace("// BODY", body) + output = run_emitted_tu(cpp, driver, opt="-O0", label="complete column names") + assert output.splitlines() == [ + "name0=1", "name1=1", "name2=0", "no-column-filter=1", + f"empty-name={int(names[0] == '')}", "other-symbol=0", "other-timeframe=0", + "unknown-context-na=1", "payload=19.75", "left=19.75", "right=-23.5", + "unknown-column-na=1", + ] diff --git a/tests/test_input_metadata_receipt.py b/tests/test_input_metadata_receipt.py index deb8816c..d59f81b9 100644 --- a/tests/test_input_metadata_receipt.py +++ b/tests/test_input_metadata_receipt.py @@ -206,20 +206,27 @@ def test_unrepresentable_strings_match_receipt_in_both_envelopes(): id="literal-escape-text"), pytest.param("x\\\0" + "789", "x\\", id="backslash-before-nul"), pytest.param("x\0y\0z", "x", id="multiple-nuls"), + pytest.param("é\0中😀", "é", id="unicode-nul"), ] @pytest.mark.parametrize("value,expected", STRING_VALUES) -@pytest.mark.parametrize("wrapped", [False, True], ids=["literal", "std-string"]) +@pytest.mark.parametrize("wrapped", [False, True, "counted"], + ids=["literal", "std-string", "counted"]) def test_emitted_string_setting_decoder(value, expected, wrapped): literal = '"' + NamingHelper._cpp_string_escape(value) + '"' - lowered = f"std::string({literal})" if wrapped else literal + if wrapped == "counted": + lowered = f"std::string({literal}, {len(value.encode('utf-8'))})" + else: + lowered = f"std::string({literal})" if wrapped else literal assert _string_setting_arg(StringLiteral(value=value), lowered) == lowered - assert _string_setting_value(lowered) == expected + assert _string_setting_value(lowered) == (value if wrapped == "counted" else expected) @pytest.mark.parametrize("lowered", [ - "0", "nullptr", 'std::string("x", 1)', '"x" + other', '"x" "y"', + "0", "nullptr", 'std::string("x", 0)', 'std::string("x", 2)', + 'std::string("é\\000z", 3)', 'std::string("é\\000z", 5)', + '"x" + other', '"x" "y"', r'"\u2603"', r'"\x41"', r'"\012"', r'"\0"', r'"\a"', '"x\\"', '"x\ny"', '"x\ry"', '"x\0y"', ]) @@ -229,9 +236,69 @@ def test_string_setting_decoder_refuses_non_emitted_forms(lowered): _string_setting_value(lowered) +INPUT_DRIVER = r''' +#include +#include +#include + +class InputProbe : public GeneratedStrategy { +public: + using GeneratedStrategy::_pf_set_input_checked; + using GeneratedStrategy::_pf_settings_receipt; +}; + +void observe(const char* name, const std::string& value) { + static const char digits[] = "0123456789abcdef"; + std::cout << name << "="; + for (unsigned char byte : value) + std::cout << digits[byte >> 4] << digits[byte & 15]; + std::cout << "\n"; +} + +void execute(InputProbe& strategy) { + std::vector bars(3); + for (std::size_t i = 0; i < bars.size(); ++i) { + bars[i].open = bars[i].low = 90.0 + double(i); + bars[i].high = 110.0 + double(i); + bars[i].close = 100.0 + double(i); + bars[i].volume = 1.0; + bars[i].timestamp = 1704067200000LL + static_cast(i) * 60000LL; + } + strategy.run(bars.data(), bars.size(), "1", "1"); + if (!strategy.last_error().empty()) + throw std::runtime_error(strategy.last_error()); +} + +int main() { + // BODY +} +''' + + +def _bytes_cpp(value): + """Supply a native setter value independently of the product serializer.""" + return "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + + +def _run_input_observer(full, body, opt): + output = run_emitted_tu( + full["cpp"], INPUT_DRIVER.replace("// BODY", body), + opt=opt, label="string input bytes and receipt", + ) + records = {} + for line in output.splitlines(): + key, value = line.split("=", 1) + assert key not in records, output + records[key] = value + return records + + @pytest.mark.parametrize("value,expected", STRING_VALUES) @pytest.mark.parametrize("opt", ["-O0", "-O2"]) -def test_all_string_input_forms_match_native_receipt(value, expected, opt): +@pytest.mark.parametrize("call_style", ["positional", "named"]) +def test_all_string_input_forms_match_native_receipt(value, expected, opt, call_style): literal = pine_string_literal(value) forms = [ ("string", "String default", ""), @@ -242,14 +309,24 @@ def test_all_string_input_forms_match_native_receipt(value, expected, opt): ("string", "String option", f', options=[{literal}, "safe"]'), ] source = '//@version=6\nstrategy("literal settings")\n' - source += "".join(f'choice_{index} = input.{kind}({literal}, "{title}"{options})\n' - for index, (kind, title, options) in enumerate(forms)) + for index, (kind, title, options) in enumerate(forms): + arguments = (f'defval={literal}, title="{title}"' + if call_style == "named" else f'{literal}, "{title}"') + source += f'choice_{index} = input.{kind}({arguments}{options})\n' full = transpile_full(source) envelope = json.loads(GLUE(source)) assert envelope["ok"] is True assert envelope["inputs"] == full["inputs"] - receipt = json.loads(run_emitted_tu(full["cpp"], RECEIPT_DRIVER, opt=opt, - label="literal settings receipt")) + body = "InputProbe strategy;\nexecute(strategy);\n" + body += "".join( + f'observe("choice_{index}", strategy.choice_{index});\n' + for index in range(len(forms)) + ) + body += 'std::cout << "receipt=" << strategy._pf_settings_receipt() << "\\n";\n' + records = _run_input_observer(full, body, opt) + assert set(records) == {*(f"choice_{index}" for index in range(len(forms))), "receipt"} + receipt = json.loads(records.pop("receipt")) + assert list(records.values()) == [value.encode("utf-8").hex()] * len(forms) assert len(full["inputs"]) == len(receipt["inputs"]) == len(forms) for metadata, checked, (kind, title, options) in zip(full["inputs"], receipt["inputs"], forms): assert metadata["title"] == checked["name"] == title @@ -257,6 +334,88 @@ def test_all_string_input_forms_match_native_receipt(value, expected, opt): if kind == "symbol": assert metadata["kind"] == "symbol" assert metadata["supported"] is True - assert metadata["default"] == checked["default"] == checked["effective_value"] == expected - assert metadata.get("options", []) == ([expected, "safe"] if options else []) + # STRING_VALUES' second field describes uncounted C strings only. + # Authored defaults and options retain the complete Pine value. + assert metadata["default"] == checked["default"] == checked["effective_value"] == value + assert metadata.get("options", []) == ([value, "safe"] if options else []) assert _manifest_view(metadata) == _receipt_view(checked) + + +@pytest.mark.parametrize("separator", ["\0", "\x01", ""], + ids=["nul", "other-control", "ascii"]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_string_options_keep_distinct_suffixes_in_native_values(separator, opt): + """The C++ checked setter validates whole option values, including NUL.""" + options = [f"a{separator}b", f"a{separator}c"] + source = '//@version=6\nstrategy("option identity")\n' + source += f'var string FIRST = {pine_string_literal(options[0])}\n' + source += f'const string SECOND = {pine_string_literal(options[1])}\n' + source += 'choice = input.string(FIRST, "Choice", options=[FIRST, SECOND])\n' + full = transpile_full(source) + attempts = [*options, "a"] + body = "" + for index, attempt in enumerate(attempts): + body += f''' +{{ + InputProbe strategy; + try {{ + strategy._pf_set_input_checked("Choice", {_bytes_cpp(attempt)}); + std::cout << "accepted{index}=1\\n"; + }} catch (const ::pineforge::checked_settings::Error&) {{ + std::cout << "accepted{index}=0\\n"; + }} + execute(strategy); + observe("value{index}", strategy.choice); + std::cout << "receipt{index}=" << strategy._pf_settings_receipt() << "\\n"; +}} +''' + records = _run_input_observer(full, body, opt) + assert set(records) == { + f"{field}{index}" for field in ("accepted", "value", "receipt") for index in range(3) + } + assert full["inputs"][0]["supported"] is True + assert full["inputs"][0]["options"] == options + for index, attempt in enumerate(attempts): + accepted = attempt in options + effective = attempt if accepted else options[0] + assert records[f"accepted{index}"] == str(int(accepted)) + assert records[f"value{index}"] == effective.encode("utf-8").hex() + (checked,) = json.loads(records[f"receipt{index}"])["inputs"] + assert checked["supported"] is True + assert checked["options"] == options + assert checked["effective_value"] == effective + + +@pytest.mark.parametrize("separator", ["\0", "\x01", ""], + ids=["nul", "other-control", "ascii"]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_input_keys_keep_distinct_suffixes_for_scalar_and_source_getters(separator, opt): + """Full C++ keys address scalar/source inputs independently of prefixes.""" + keys = [f"Key{separator}left", f"Key{separator}right", "Key", f"Source{separator}value"] + source = '//@version=6\nstrategy("key identity")\n' + source += f'const string KEY = {pine_string_literal("Key" + separator)}\n' + source += 'first = input.int(11, KEY + "left")\n' + source += f'second = input.int(defval=22, title={pine_string_literal(keys[1])})\n' + source += 'prefix = input.int(33, "Key")\n' + source += f'selected = input.source(title={pine_string_literal(keys[3])}, defval=close)\n' + full = transpile_full(source) + overrides = ["41", "42", "43", "high"] + body = "InputProbe strategy;\n" + for key, value in zip(keys, overrides, strict=True): + body += f"strategy._pf_set_input_checked({_bytes_cpp(key)}, {_bytes_cpp(value)});\n" + body += ''' +execute(strategy); +std::cout << "first=" << strategy.first << "\\n"; +std::cout << "second=" << strategy.second << "\\n"; +std::cout << "prefix=" << strategy.prefix << "\\n"; +std::cout << "selected=" << strategy.selected << "\\n"; +std::cout << "receipt=" << strategy._pf_settings_receipt() << "\\n"; +''' + records = _run_input_observer(full, body, opt) + assert set(records) == {"first", "second", "prefix", "selected", "receipt"} + assert [records[name] for name in ("first", "second", "prefix")] == overrides[:3] + assert float(records["selected"]) == 112.0 + assert [entry["title"] for entry in full["inputs"]] == keys + inputs = json.loads(records["receipt"])["inputs"] + assert [entry["name"] for entry in inputs] == keys + assert [entry["effective_value"] for entry in inputs] == overrides diff --git a/tests/test_method_receiver_types.py b/tests/test_method_receiver_types.py index 21a2c3c9..dde316a2 100644 --- a/tests/test_method_receiver_types.py +++ b/tests/test_method_receiver_types.py @@ -100,6 +100,31 @@ def test_array_user_method_precedes_same_named_builtin() -> None: compile_cpp(cpp, label="array-method-builtin-collision") +@pytest.mark.parametrize( + "receiver_type, receiver", + [ + pytest.param("array", 'array.from("alpha", "beta")', id="array"), + pytest.param("int", "7", id="namespace-shadow"), + ], +) +def test_authored_join_return_precedes_fixed_builtin_result(receiver_type, receiver): + source = f'''//@version=6 +strategy("authored join return") +method join({receiver_type} self, string separator) => + self + str.length(separator) +relay({receiver_type} array, string separator) => + array.join(separator) +parts = {receiver} +result = relay(parts, "|") +plot(close) +''' + cpp = transpile(source) + assert "int relay(" in cpp + assert re.search(r"\bint result\s*[=;]", cpp), cpp + compile_cpp(cpp, label="authored-array-join-return") + + def test_map_receiver_keeps_shared_id_value_handle() -> None: source = '''//@version=6 strategy("map builtin collision") diff --git a/tests/test_na_and_compare_refusals.py b/tests/test_na_and_compare_refusals.py index 74b3134a..23f13610 100644 --- a/tests/test_na_and_compare_refusals.py +++ b/tests/test_na_and_compare_refusals.py @@ -330,16 +330,17 @@ def test_grouping_around_consecutive_not_keeps_the_refusal( ) -def test_string_ordering_reports_the_first_string_operand() -> None: +@pytest.mark.parametrize("operator", ["<", "<=", ">", ">="]) +def test_string_ordering_reports_the_first_string_operand(operator: str) -> None: _assert_diagnostic( _script( 'string a = "a"\n' 'string b = "b"\n' - "value = a < b\n" + f"value = a {operator} b\n" ), "PF-E2052", - "Cannot order string values with <.", - {"op": "<"}, + f"Cannot order string values with {operator}.", + {"op": operator}, 5, 9, # a is the first operand of the ordered comparison. ) diff --git a/tests/test_security_helper_local_na.py b/tests/test_security_helper_local_na.py index 15724ee0..1a2e0fff 100644 --- a/tests/test_security_helper_local_na.py +++ b/tests/test_security_helper_local_na.py @@ -15,6 +15,8 @@ import re +import pytest + from pineforge_codegen import transpile from tests import _compile as compile_env @@ -98,3 +100,80 @@ def test_int_locals_read_from_helper_state_narrow_without_losing_na(): body, ), body compile_env.compile_cpp(cpp, label="security-helper-state-narrowing") + + +@pytest.mark.parametrize( + "prefix, arguments, forwarded", + [ + pytest.param("string absent = na\n", '"é", absent', False, id="global-na"), + pytest.param( + "string absent = na\nstring alias = absent\n", + '"中", alias', False, id="aliased-global-na", + ), + pytest.param("", 'na, "suffix"', False, id="parameter-na"), + pytest.param( + "string absent = na\n", 'absent, "尾"', True, id="forwarded-first-na", + ), + pytest.param( + "first = 17\nstring absent = na\n", '"lead", absent', True, + id="parameter-shadows-global", + ), + ], +) +def test_requested_string_arguments_type_inferred_locals(prefix, arguments, forwarded): + # The declaration and missing-value representations are the unit target; + # Unicode/NA value observations belong to the external parity probes. + helper = ( + "combine(string first, string last) =>\n" + " accumulated = first\n" + " accumulated += last\n" + " accumulated\n" + ) + if forwarded: + helper += ( + "forward(string first, string last) =>\n" + " copied = first\n" + " combine(copied, last)\n" + ) + callee = "forward" if forwarded else "combine" + source = ( + '//@version=6\nstrategy("requested string argument storage")\n' + + helper + prefix + + f'joined = request.security(syminfo.tickerid, "1", {callee}({arguments}))\n' + + "plot(str.length(joined))\n" + ) + cpp = transpile(source) + body = _eval_bodies(cpp) + assert re.search(r"std::string _sec0_combine_\d+_accumulated = ", body), body + if forwarded: + assert re.search(r"std::string _sec0_forward_\d+_copied = ", body), body + assert "na()" in body, body + assert "na()" not in body, body + assert "_pf_u16_concat(" in body, body + compile_env.compile_cpp(cpp, label="requested-string-argument-storage") + + +@pytest.mark.parametrize( + "storage", + [pytest.param("var ", id="persistent"), pytest.param("", id="history")], +) +def test_requested_inferred_string_state_declares_string_history(storage): + history = " previous = accumulated[1]\n" if not storage else "" + result = "previous" if history else "accumulated" + source = ( + '//@version=6\nstrategy("requested inferred string history")\n' + "read_value(string first) =>\n" + f" {storage}accumulated = first\n" + + history + f" {result}\n" + + 'joined = request.security(syminfo.tickerid, "1", read_value(na))\n' + + "plot(str.length(joined))\n" + ) + cpp = transpile(source) + body = _eval_bodies(cpp) + assert "Series> _security_helper_series_str_;" in cpp + assert "_security_helper_series_str_" in body, body + assert "na()" in body, body + assert "na()" not in body, body + if history: + assert re.search(r"std::string _sec0_read_value_\d+_previous = ", body), body + compile_env.compile_cpp(cpp, label="requested-inferred-string-history") diff --git a/tests/test_string_local_history_concat.py b/tests/test_string_local_history_concat.py index 28e657e3..4e099955 100644 --- a/tests/test_string_local_history_concat.py +++ b/tests/test_string_local_history_concat.py @@ -7,9 +7,14 @@ from pineforge_codegen import transpile from pineforge_codegen.errors import CompileError, Phase from tests._compile import compile_cpp, run_emitted_tu +from tests._older_codegen import with_utf16_units -_REFERENCE_JOIN = re.compile(r"std::to_string\((\w+)\[0\]\) \+ (\w+)\)") +_REFERENCE_JOIN = re.compile(r"\(std::to_string\((\w+)\[0\]\) \+ (\w+)\)") +_REFERENCE_REQUESTED_JOIN = re.compile( + r'\((\(_security_helper_series_str_\.[A-Za-z_][A-Za-z_0-9]*\(' + r'"(?:\\.|[^"\\])*", [0-9]+\)[^()\n;]*\)\[0\]) \+ (\w+)\)' +) def with_string_join_fixed(cpp): @@ -17,11 +22,15 @@ def with_string_join_fixed(cpp): The older emission wrapped the current value of a string local read with history in std::to_string when joining it with its previous value (issue - #354), which does not compile. A test that compares a new emission with an - older one passes only when that single site is the whole difference. + #354), which does not compile. Both that join and the corresponding + requested join now use UTF-16 concatenation. The comparison still checks + the complete C++, allowing only these joins and the shared helper block. """ assert len(_REFERENCE_JOIN.findall(cpp)) == 1, cpp - return _REFERENCE_JOIN.sub(r"\1[0] + \2)", cpp) + cpp = _REFERENCE_JOIN.sub(r"_pf_u16_concat(\1[0], \2)", cpp) + cpp, requested = _REFERENCE_REQUESTED_JOIN.subn(r"_pf_u16_concat(\1, \2)", cpp) + assert requested <= 1, cpp + return with_utf16_units(cpp) _REPRODUCTION = """//@version=6 @@ -141,7 +150,8 @@ def test_history_string_operands_compile_and_return_the_join( for member in members: assert f"std::to_string({member}" not in cpp expression = ( - f"(bb + {member}[0])" if swapped else f"({member}[0] + bb)" + f"_pf_u16_concat(bb, {member}[0])" if swapped + else f"_pf_u16_concat({member}[0], bb)" ) assert expression in cpp # Bar 0 executes to seed history. Only bars with an existing previous diff --git a/tests/test_string_nul_values.py b/tests/test_string_nul_values.py index 4991c646..f9cf2da4 100644 --- a/tests/test_string_nul_values.py +++ b/tests/test_string_nul_values.py @@ -84,7 +84,7 @@ def _run(script: str, observe: str) -> str: ), pytest.param( f'enum Mode\n fast = "f{NUL}t"\n slow\n' - 'e = Mode.fast\ntext = str.tostring(e)\nn = str.length(text)\nplot(n)\n', + 'e = Mode.fast\ntitle_value = str.tostring(e)\nn = str.length(title_value)\nplot(n)\n', 'std::cout << strategy.n << "\\n";', "3\n", id="length-of-an-enum-title", ), pytest.param( @@ -98,13 +98,11 @@ def _run(script: str, observe: str) -> str: 'std::cout << strategy.n << "\\n";', "3\n", id="length-of-a-history-read-var-set-from-a-literal", ), - # A known limit, pinned so that a change of it is seen: the default of a - # string input is still cut at its first NUL, so the input no longer equals - # the literal it was given (the two were equal only while both were cut). + # The input's getter must preserve the same bytes as its authored default. pytest.param( f'm = input.string("a{NUL}b", "Text")\nu = m == "a{NUL}b" ? 1 : 0\n' 'k = str.length(m)\nplot(u + k)\n', - 'std::cout << strategy.u << " " << strategy.k << "\\n";', "0 1\n", + 'std::cout << strategy.u << " " << strategy.k << "\\n";', "1 3\n", id="known-limit-an-input-default-is-cut-and-differs-from-its-literal", ), ] @@ -180,3 +178,58 @@ def test_the_forms_above_keep_their_lengths_at_run_time(): '<< strategy.viaheld << " " << strategy.joined << " " << strategy.chosen << "\\n";' ) assert _run(script, observe) == "3 3 3 4 3\n" + + +@pytest.mark.parametrize("value", [ + "\0", "x\0" + "789", "x\0y\0z", 'é"\\\0中😀', "a\x01b", "60", "", +], ids=["nul", "digits", "multiple", "utf8-escapes", "other-control", "ascii", "empty"]) +def test_length_aware_literal_consumers_keep_the_authored_bytes(value): + """Serializer units: a timeframe's grammar is a separate validation rule.""" + from pineforge_codegen import Analyzer, CodeGen, Lexer, Parser + from pineforge_codegen.codegen.helpers import NamingHelper + from pineforge_codegen.codegen.input import InputHelper + from pineforge_codegen.codegen.security import SecurityEmitter + + class Consumers(NamingHelper, InputHelper, SecurityEmitter): + pass + + consumer = Consumers() + ast = Parser(Lexer(HEADER).tokenize(), source=HEADER).parse() + generator = CodeGen(Analyzer(ast).analyze()) + generator._known_vars["sample"] = value + expressions = [ + consumer._cpp_string_literal(value), + consumer._input_key_literal(value), + consumer._security_tf_literal(value), + generator._resolve_known("sample"), + ] + expected = "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + body = "\n".join( + f'check("consumer{index}", {expression}, {expected});' + for index, expression in enumerate(expressions) + ) + driver = r''' +#include +#include + +void check(const char* name, const std::string& value, const std::string& expected) { + if (value != expected) throw std::runtime_error("literal consumer changed bytes"); + static const char digits[] = "0123456789abcdef"; + std::cout << name << "="; + for (unsigned char byte : value) + std::cout << digits[byte >> 4] << digits[byte & 15]; + std::cout << "\n"; +} + +int main() { + // BODY +} +'''.replace("// BODY", body) + output = run_emitted_tu( + transpile(HEADER), driver, opt="-O0", label="length-aware literal consumers", + ) + assert output.splitlines() == [ + f"consumer{index}={value.encode('utf-8').hex()}" for index in range(len(expressions)) + ] diff --git a/tests/test_string_split_empty_separator.py b/tests/test_string_split_empty_separator.py index 45b90272..389c6132 100644 --- a/tests/test_string_split_empty_separator.py +++ b/tests/test_string_split_empty_separator.py @@ -12,7 +12,7 @@ import pytest from pineforge_codegen import transpile -from tests._compile import run_emitted_tu +from tests._compile import compile_cpp, run_emitted_tu HEADER = '//@version=6\nstrategy("string split")\n' @@ -133,3 +133,45 @@ def test_the_empty_text_gives_one_part(script, observe, expected): @pytest.mark.parametrize("script, observe, expected", CONTROL_ROWS) def test_a_separator_that_is_not_empty_splits_as_before(script, observe, expected): assert _run(script, observe) == expected + + +MISSING_SEPARATOR_ROWS = [ + pytest.param( + 'n = array.size(str.split("abc", na))\n' + 'parts = str.split("abc", na)\nplot(n)\n', + id="literal-separator-and-nested-consumer", + ), + pytest.param( + 'parts = str.split("", separator = na)\n' + 'n = array.size(parts)\nplot(n)\n', + id="empty-source-with-named-separator", + ), + pytest.param( + 'parts = str.split(separator = na, string = "é中")\n' + 'n = array.size(parts)\nplot(n)\n', + id="bmp-source-with-reversed-keywords", + ), + pytest.param( + 'parts = str.split("a\U0001f600b", na)\n' + 'n = array.size(parts)\nplot(n)\n', + id="astral-source-with-positional-separator", + ), + pytest.param( + 'src = close > open ? "longer" : "xy"\n' + 'parts = str.split(src, na)\nn = array.size(parts)\nplot(n)\n', + id="computed-source", + ), + pytest.param( + 'f(string src) =>\n str.split(src, na)\n' + 'parts = f("aéb")\nn = array.size(parts)\nplot(n)\n', + id="helper-body", + ), +] + + +@pytest.mark.parametrize("script", MISSING_SEPARATOR_ROWS) +def test_a_literal_na_separator_has_the_string_parameter_type(script): + # This target is contextual argument typing: compile the complete emitted + # TU so a numeric na cannot reach a string parameter. Runtime/TV values + # belong to the split probes, including the separate named/Unicode tapes. + compile_cpp(transpile(HEADER + script), label="split string argument type") diff --git a/tests/test_string_units.py b/tests/test_string_units.py index fcdf24e2..de118d6e 100644 --- a/tests/test_string_units.py +++ b/tests/test_string_units.py @@ -22,8 +22,15 @@ units of an emoji keeps each half as its own three-byte sequence, the form str.split gives its parts, so the length of the half is one unit; * ``control-*``: a case in which bytes, code points and UTF-16 units agree. + +For issue #382 item 8, ``measured-*`` trim rows repeat the issue's four exact +examples. ``contract-derived-*`` rows compose only its supplied removed/kept +characters with other text, interior characters, empty text and runtime inputs; +they are not separate TradingView measurements or a full Unicode whitespace set. """ +import json + import pytest from pineforge_codegen import transpile @@ -50,6 +57,7 @@ bars[i].timestamp = 1704067200000LL + static_cast(i) * 60000LL; } GeneratedStrategy strategy; + // INPUTS strategy.run(bars.data(), bars.size(), "1", "1"); if (!strategy.last_error().empty()) { std::cerr << strategy.last_error() << "\n"; @@ -66,10 +74,18 @@ FEFF = "\U0000feff" -def _run(script: str, observe: str) -> str: +def _run(script: str, observe: str, *, opt: str = "-O0", + inputs: dict[str, str] | None = None, setup: str = "") -> str: cpp = transpile(HEADER + script) + input_setup = "\n".join( + f"strategy.set_input({json.dumps(name)}, {json.dumps(value, ensure_ascii=False)});" + for name, value in (inputs or {}).items() + ) + driver = DRIVER.replace( + "// INPUTS", input_setup + ("\n" + setup if setup else ""), + ).replace("// OBSERVE", observe) return run_emitted_tu( - cpp, DRIVER.replace("// OBSERVE", observe), opt="-O0", label="string units", + cpp, driver, opt=opt, label="string units", ) @@ -103,15 +119,159 @@ def test_length_counts_utf16_units(text, expected): pytest.param("aab", "b", 2, id="measured-after-an-ascii-letter"), pytest.param(f"{EMOJI}{EMOJI}b", "b", 4, id="derived-after-two-emoji"), pytest.param(f"a{EMOJI}b", EMOJI, 1, id="derived-the-emoji-itself"), - pytest.param(f"a{EMOJI}b", "x", -1, id="control-not-found"), + # Issue #382 item 5: an absent match is na, including after non-ASCII text. + pytest.param(f"a{EMOJI}b", "x", "na", id="control-not-found"), pytest.param(f"a{CJK}b", CJK, 1, id="control-the-bmp-character-itself"), ] @pytest.mark.parametrize("text, needle, expected", POS_ROWS) def test_pos_counts_utf16_units(text, needle, expected): - script = f'p = str.pos("{text}", "{needle}")\nplot(p)\n' - assert _run(script, 'std::cout << strategy.p << "\\n";') == f"{expected}\n" + script = (f'p = str.pos("{text}", "{needle}")\n' + 'missing = na(p)\nplot(p)\n') + observe = ( + 'if (strategy.missing) std::cout << "na\\n"; ' + 'else std::cout << strategy.p << "\\n";' + ) + assert _run(script, observe) == f"{expected}\n" + + +# Retain the original module selection and run these +# boundary/consumer rows at both optimisation levels. These are native lowering +# checks of the behavior described by the public string fixture. +NATIVE_OPTS = [ + pytest.param("-O0", id="O0"), + pytest.param("-O2", id="O2"), +] + +MISSING_POS_CONSUMERS = [ + pytest.param('nz(str.pos("abc", "x"), 0)', "0\n", id="nz"), + pytest.param('str.tostring(str.pos("abc", "x"))', "NaN\n", id="tostring"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("expression, expected", MISSING_POS_CONSUMERS) +def test_missing_pos_literal_consumers(expression, expected, opt): + # Keep the reported public expressions intact, including direct nesting. + script = f"observed = {expression}\nplot(close)\n" + assert _run(script, 'std::cout << strategy.observed << "\\n";', opt=opt) == expected + + +RUNTIME_POS_ROWS = [ + pytest.param("abc", "x", True, 0, "NaN", id="missing-ascii"), + pytest.param("seed42", "?", True, 0, "NaN", id="missing-distinct"), + pytest.param("abc", "b", False, 1, "1", id="found-middle"), + pytest.param("abc", "a", False, 0, "0", id="found-zero"), + pytest.param("abc", "", False, 0, "0", id="empty-needle"), + pytest.param("", "x", True, 0, "NaN", id="empty-haystack"), + pytest.param("", "", False, 0, "0", id="both-empty"), + pytest.param("ab", "abc", True, 0, "NaN", id="longer-needle"), + pytest.param(f"a{EMOJI}b", "b", False, 3, "3", id="found-utf16"), + pytest.param(f"{CJK}{EMOJI}tail", "l", False, 6, "6", id="found-distinct"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("text, needle, missing, replaced, rendered", RUNTIME_POS_ROWS) +def test_pos_consumers_use_runtime_inputs(text, needle, missing, replaced, rendered, opt): + # Every row emits the same Pine source. Only the native input overrides + # change; expected positions/strings are never injected into that source. + script = ( + 'source_value = input.string("unused text", "Text")\n' + 'needle = input.string("unused needle", "Needle")\n' + 'p = str.pos(source_value, needle)\n' + 'missing = na(p)\n' + 'replaced = nz(p, 0)\n' + 'rendered = str.tostring(p)\n' + 'plot(p)\n' + ) + observe = ( + 'std::cout << strategy.missing << "|" << strategy.replaced ' + '<< "|" << strategy.rendered << "\\n";' + ) + assert _run(script, observe, opt=opt, inputs={"Text": text, "Needle": needle}) == ( + f"{int(missing)}|{replaced}|{rendered}\n" + ) + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +def test_substring_after_absent_delimiter_completes(opt): + # An unexpected run stop remains a failing native execution in DRIVER, + # never an expected-stop assertion. + script = ( + 't = str.substring("abc", str.pos("abc", ":") + 1)\n' + 'missing = na(t)\n' + 'completed = true\n' + 'plot(close)\n' + ) + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << "\\n"; ' + 'hex(strategy.t);' + ) + assert _run(script, observe, opt=opt) == "1 0\n616263\n" + + +MISSING_START_ROWS = [ + pytest.param("abcdef", "int(na)", id="ascii-int-na"), + pytest.param(f"a{EMOJI}b", "int(na) + 1", id="unicode-wide-na"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("text, begin", MISSING_START_ROWS) +def test_substring_missing_start_returns_whole_text(text, begin, opt): + script = ( + 'source_value = input.string("unused text", "Text")\n' + f't = str.substring(source_value, {begin})\n' + 'missing = na(t)\n' + 'completed = true\n' + 'plot(close)\n' + ) + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << "\\n"; ' + 'hex(strategy.t);' + ) + assert _run(script, observe, opt=opt, inputs={"Text": text}) == ( + f"1 0\n{text.encode('utf-8').hex()}\n" + ) + + +MISSING_END_ROWS = [ + pytest.param( + 't = str.substring("abc", 0, str.pos("abc", ":"))\n', {}, + id="literal-zero-begin", + ), + pytest.param( + 't = str.substring("abc", 1, str.pos("abc", ":"))\n', {}, + id="literal-nonzero-begin", + ), + pytest.param( + 'source_value = input.string("unused text", "Text")\n' + 'needle = input.string("unused needle", "Needle")\n' + 'begin = input.int(1, "Begin")\n' + 't = str.substring(source_value, begin, str.pos(source_value, needle))\n', + {"Text": f"r{EMOJI}tail", "Needle": "!", "Begin": "2"}, + id="runtime-utf16-nonzero-begin", + ), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("script, inputs", MISSING_END_ROWS) +def test_substring_missing_end_returns_string_na(script, inputs, opt): + # The zero-begin row isolates the wrong value without signed overflow. + # Check a missing end before subtracting begin. A stop or undefined + # conversion cannot satisfy the value assertion. + script += 'missing = na(t)\ncompleted = true\nplot(close)\n' + # visit_call.py lowers Pine na(string) as .empty(); the engine has no + # is_na(std::string) overload. Compare the actual result with its canonical + # na() too, without hard-coding empty bytes or a numeric NA. + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << " " ' + '<< (strategy.t == na()) << "\\n";' + ) + assert _run(script, observe, opt=opt, inputs=inputs) == "1 1 1\n" TAIL_ROWS = [ @@ -241,19 +401,114 @@ def test_the_arguments_may_be_any_string_expression(script, observe, expected): id="str-substring-alone"), pytest.param('p = str.pos("abc", "b")\nplot(close)\n', True, id="str-pos-alone"), pytest.param('n = str.length("abc")\nplot(close)\n', True, id="str-length-alone"), + pytest.param('t = str.match("aéb", "a.b")\nplot(close)\n', True, + id="str-match-alone"), + pytest.param('n = str.contains("abc", "b")\nplot(close)\n', True, + id="str-contains-alone"), + pytest.param('n = str.startswith("abc", "a")\nplot(close)\n', True, + id="str-startswith-alone"), + pytest.param('n = str.endswith("abc", "c")\nplot(close)\n', True, + id="str-endswith-alone"), + pytest.param('s = "a" + "b"\nplot(close)\n', True, id="string-plus-alone"), + pytest.param('s = "a"\ns += "b"\nplot(close)\n', True, id="string-append-alone"), + pytest.param("n = 1\nn += 2\nplot(close)\n", False, + id="control-numeric-append"), + pytest.param( + 'parts = array.from("a", "b")\n' + 'joined = array.join(parts, "")\nplot(close)\n', + True, id="string-array-namespace-join-alone", + ), + pytest.param( + 'parts = array.from("a", "b")\njoined = parts.join()\nplot(close)\n', + True, id="string-array-method-join-default-separator", + ), + pytest.param( + 'joined = array.from("a", "b").join("")\nplot(close)\n', + True, id="string-array-temporary-join-alone", + ), + pytest.param( + 'collect_join(array parts, array results) =>\n' + ' array.push(results, parts.join(""))\n' + ' array.size(parts)\n' + 'results = array.new_string(0)\n' + 'counted = collect_join(array.from("a", "b"), results)\nplot(close)\n', + True, id="string-array-forwarded-join-alone", + ), pytest.param('t = str.upper("abc")\nplot(close)\n', False, id="control-another-string-function"), pytest.param("plot(close)\n", False, id="control-no-string-function"), # A plot argument is not emitted, so the call emits nothing and needs no helper. pytest.param('plot(str.length("abc"))\n', False, id="control-a-call-that-is-only-a-plot-argument"), + pytest.param('plot(str.match("aéb", "a.b"))\n', False, + id="control-a-match-that-is-only-a-plot-argument"), ] @pytest.mark.parametrize("script, emitted", EMISSION_ROWS) def test_the_helpers_are_emitted_only_for_a_call_that_uses_them(script, emitted): cpp = transpile(HEADER + script) - assert (HELPER_DEFINITION in cpp) is emitted + assert cpp.count(HELPER_DEFINITION) == int(emitted) + + +@pytest.mark.parametrize("operation", ("pos", "contains", "startswith", "endswith")) +@pytest.mark.parametrize("named", (False, True), ids=("positional", "named")) +def test_search_arguments_bind_once_in_parameter_order(operation, named): + arguments = 'read_value("abc", calls, 1), read_value("b", calls, 2)' + if named: + arguments = 'str=read_value("b", calls, 2), source=read_value("abc", calls, 1)' + cpp = transpile( + HEADER + + "read_value(string payload, array calls, int tag) =>\n" + + " array.push(calls, tag)\n payload\n" + + "calls = array.new_int(0)\n" + + f"observed = str.{operation}({arguments})\nplot(close)\n" + ) + line = next(line for line in cpp.splitlines() if line.strip().startswith("observed =")) + source = 'const std::string _pf_search_source = (read_value(std::string("abc"), calls, 1));' + needle = 'const std::string _pf_search_needle = (read_value(std::string("b"), calls, 2));' + searched = f"return _pf_u16_{operation}(_pf_search_source, _pf_search_needle);" + assert line.count(source) == line.count(needle) == line.count(searched) == 1 + assert line.index(source) < line.index(needle) < line.index(searched) + assert line.count("read_value(") == 2 + + +@pytest.mark.parametrize("persistent, history", ((False, False), (True, False), (False, True))) +@pytest.mark.parametrize("string", (False, True), ids=("numeric", "string")) +@pytest.mark.parametrize("parameters", (False, True), ids=("literals", "parameters")) +def test_requested_compound_assignment_uses_its_storage_family( + persistent, history, string, parameters, +): + from tests.test_security_series_timeframe import requested_body + + kind, first, last = ("string", '"a"', '"b"') if string else ("float", "2.5", "1.25") + signature = f"{kind} first, {kind} last" if parameters else "" + initial, appended = ("first", "last") if parameters else (first, last) + arguments = f"{first}, {last}" if parameters else "" + cpp = transpile( + HEADER + f"combine({signature}) =>\n" + + f" {'var ' if persistent else ''}{kind} accumulated = {initial}\n" + + f" accumulated += {appended}\n" + + (" previous = accumulated[1]\n" if history else "") + + " accumulated\n" + + f'joined = request.security(syminfo.tickerid, "60", combine({arguments}))\n' + + "plot(close)\n" + ) + body = requested_body(cpp, 0) + assert ("_pf_u16_concat(" in body) is string + assert (HELPER_DEFINITION in cpp) is string + if string: + assert 'std::string("b")' in body + # The chart helper is emitted too, even when only a request calls it. + # Both bodies must keep the persistent/local string out of arithmetic. + assert "_pf_u16_concat(std::to_string(" not in cpp + for line in body.splitlines(): + if "_pf_u16_concat(" in line: + assert "std::to_string(" not in line + assert "is_na(" not in line + assert "(double)" not in line + else: + assert "1.25" in body STOP_DRIVER = r''' @@ -287,10 +542,179 @@ def test_a_substring_past_the_end_stops_the_run_and_names_the_units(): @pytest.mark.parametrize("begin, detail", REFUSED_ROWS) def test_a_begin_outside_the_text_stops_the_run(begin, detail): - """TradingView refuses these two scripts when it compiles them (their - arguments are constants); the engine stops the run, with the text - std::string::substr gives. Both refuse; the moment is not compared.""" + """Pin the existing native boundary guard and its diagnostic detail. + + A native runtime stop does not establish agreement with a TradingView + compile-time refusal. That acceptance rule needs its separate real record. + """ cpp = transpile(HEADER + f't = str.substring("abc", {begin})\nplot(close)\n') stopped = run_emitted_tu(cpp, STOP_DRIVER, opt="-O2", label="substring begin outside") assert "basic_string::substr" in stopped assert detail in stopped + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +def test_substring_finite_end_before_start_stops_the_run(opt): + # Issue #382 item 9: a finite end before begin stops the run. + # Retain actual native values and errors in stdout for diagnosis. + script = ( + 'j = input.int(1, "End")\n' + 'completed = false\n' + 't = str.substring("abc", 2, j)\n' + 'completed := true\n' + 'plot(close)\n' + ) + cpp = transpile(HEADER + script) + driver = r''' +#include +int main() { + GeneratedStrategy strategy; + strategy.set_input("End", "1"); + Bar bars[3]{ + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067200000LL}, + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067260000LL}, + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067320000LL}, + }; + strategy.run(bars, 3, "1", "1"); + std::cout << "j=" << strategy.j << "\n" + << "completed=" << strategy.completed << "\n" + << "value=" << strategy.t << "\n" + << "error=" << strategy.last_error() << "\n"; + return 0; +} +''' + # Compile/link failures and unexpected process exits propagate unchanged. + # Only the native run's own last_error() can satisfy the stop assertion. + observed = run_emitted_tu( + cpp, driver, opt=opt, label="substring finite end before start", + ) + print(f"native stdout at {opt}:\n{observed}", end="") + position, completed, value, error = observed.split("\n", 3) + assert position == "j=1", observed + assert value.startswith("value="), observed + assert error.startswith("error="), observed + assert "end_pos is less than begin_pos" in error, observed + assert completed == "completed=0", observed + + +def _assert_trim_cases(cases: list[tuple[str, str]], input_shape: str) -> None: + """Observe both the actual Pine source value and its trim result as bytes.""" + script = [] + setup = [] + observe = [] + expected = [] + for index, (text, trimmed) in enumerate(cases): + source = f"trim_source_{index}" + result = f"trim_result_{index}" + if input_shape == "runtime-input": + title = f"Trim text {index}" + # A distinct default makes an ignored override observable. Counted + # UTF-8 bytes keep NUL intact without exercising the known input- + # default truncation defect from item 6. + script.append(f'{source} = input.string("unused default", "{title}")\n') + encoded = text.encode("utf-8") + octets = "".join(f"\\{byte:03o}" for byte in encoded) + setup.append( + f'strategy.set_input("{title}", ' + f'std::string("{octets}", {len(encoded)}));' + ) + else: + assert input_shape == "literal" + # Pine supports \n and \t; \r spells the letter r. Keep a raw CR + # inside the quotes, where the lexer preserves it as string data. + literal = (text.replace("\\", "\\\\").replace('"', '\\"') + .replace("\n", "\\n").replace("\t", "\\t")) + script.append(f'{source} = "{literal}"\n') + script.append(f"{result} = str.trim({source})\n") + observe.append(f"hex(strategy.{source}); hex(strategy.{result});") + expected.extend((text.encode("utf-8").hex() + "\n", + trimmed.encode("utf-8").hex() + "\n")) + script.append("plot(close)\n") + # hex() iterates std::string's bytes, including NUL. Only ASCII reaches + # stdout, so invalid UTF-8 cannot be hidden by decoding or replacement. + # Compare every line, including empty ones and the final newline. + assert _run("".join(script), "\n".join(observe), + setup="\n".join(setup)) == "".join(expected) + + +TRIM_MEASURED_ROWS = [ + pytest.param("\u00a0a\u00a0", "a", id="measured-nbsp-both-ends"), + pytest.param("\u2003a\u2003", "a", id="measured-em-space-both-ends"), + pytest.param("\ufeffa", "a", id="measured-feff-left-end"), + pytest.param("\va\f", "a", id="measured-vertical-tab-and-form-feed"), +] + + +@pytest.mark.parametrize("text, expected", TRIM_MEASURED_ROWS) +def test_trim_measured_end_whitespace(text, expected): + _assert_trim_cases([(text, expected)], "literal") + + +# The issue's explicit classification, not Python's isspace()/strip() or a +# guessed Unicode category. The test shapes below are contract compositions. +TRIM_CHARACTER_ROWS = [ + pytest.param("\u00a0", True, id="removes-u00a0-nbsp"), + pytest.param("\u2003", True, id="removes-u2003-em-space"), + pytest.param("\u3000", True, id="removes-u3000-ideographic-space"), + pytest.param("\u1680", True, id="removes-u1680-ogham-space-mark"), + pytest.param("\u2028", True, id="removes-u2028-line-separator"), + pytest.param("\ufeff", True, id="removes-ufeff-zero-width-no-break-space"), + pytest.param("\v", True, id="removes-u000b-vertical-tab"), + pytest.param("\f", True, id="removes-u000c-form-feed"), + pytest.param(" ", True, id="removes-u0020-space"), + pytest.param("\t", True, id="removes-u0009-tab"), + pytest.param("\n", True, id="removes-u000a-line-feed"), + pytest.param("\r", True, id="removes-u000d-carriage-return"), + pytest.param("\u0085", False, id="keeps-u0085-next-line"), + pytest.param("\u001f", False, id="keeps-u001f-unit-separator"), + pytest.param("\u200b", False, id="keeps-u200b-zero-width-space"), + pytest.param("\u180e", False, id="keeps-u180e-mongolian-vowel-separator"), +] + +TRIM_INPUT_SHAPES = [ + pytest.param("literal", id="contract-derived-literal"), + pytest.param("runtime-input", id="contract-derived-runtime-input"), +] + + +@pytest.mark.parametrize("input_shape", TRIM_INPUT_SHAPES) +@pytest.mark.parametrize("character, remove_at_ends", TRIM_CHARACTER_ROWS) +def test_trim_contract_characters_at_ends_and_inside(character, remove_at_ends, input_shape): + cases = [] + # Two different interiors, one with a NUL followed by an emoji and more + # text: neither a constant "a" nor a C-string prefix can satisfy all rows. + for left, right in (("left", "right"), (f"{CJK}\0", f"{EMOJI}tail")): + body = left + right + interior = left + character + right + for prefix, suffix in ((character * 2, ""), ("", character * 3), + (character * 2, character * 3)): + text = prefix + body + suffix + cases.append((text, body if remove_at_ends else text)) + cases.append((interior, interior)) + text = character + interior + character + cases.append((text, interior if remove_at_ends else text)) + text = " \t" + character + body + character + "\n\r" + cases.append((text, body if remove_at_ends else character + body + character)) + blank_only = character * 3 + cases.append((blank_only, "" if remove_at_ends else blank_only)) + _assert_trim_cases(cases, input_shape) + + +@pytest.mark.parametrize("input_shape", TRIM_INPUT_SHAPES) +def test_trim_contract_mixed_boundaries_and_empty(input_shape): + removed = "\u00a0\u2003\u3000\u1680\u2028\ufeff\v\f \t\n\r" + kept = "\u0085\u001f\u200b\u180e" + body = f"head{removed}tail{CJK}\0{EMOJI}" + cases = [ + ("", ""), + ("plain words", "plain words"), + (removed, ""), + (removed + body + removed, body), + (removed + kept + removed, kept), + ] + for boundary in kept: + # A kept character stops trimming: removable characters behind it are + # interior text and must survive along with every byte after the NUL. + interior = boundary + removed + body + removed + boundary + cases.append((removed + interior + removed, interior)) + _assert_trim_cases(cases, input_shape) From 41b7aec5ff91bd911abe9df074ba9f38bc0b5b93 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:01:11 +0800 Subject: [PATCH 2/8] Validate proven negative substring begin arguments Share immutable integer proofs with argument validation and retain dynamic runtime boundary checks. Add positional, named, alias, missing and runtime controls. Diagnostic catalog generation and exact-head verification pending. --- CHANGELOG.md | 4 + pineforge_codegen/analyzer/v6_forms.py | 62 ++++++++ pineforge_codegen/analyzer/value_bindings.py | 5 +- pineforge_codegen/loop_types.py | 53 +++++-- tests/test_string_units.py | 158 +++++++++++++++++-- 5 files changed, 256 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1907656c..b9c573c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,10 @@ supported as exact pairs; on the 0.x line they are independent. See the endpoints are ordered before computing the count; an end before the start stops the run. UTF-16 slicing, end clamping and invalid-begin stops use the existing helper. The oracle is the reporter's #382 item 9 readings. +- `str.substring` refuses a provably negative constant `begin_pos` during + analysis, including integer arithmetic, constant aliases and named arguments. + Dynamic negative starts retain their runtime error, and missing positions + retain their existing handling. - `str.trim` additionally removes VT (U+000B), FF (U+000C), U+00A0, U+1680, U+2003, U+2028, U+3000 and U+FEFF from the ends. Interior bytes and kept characters are preserved; other Unicode spaces remain unchanged (#382, item 8). diff --git a/pineforge_codegen/analyzer/v6_forms.py b/pineforge_codegen/analyzer/v6_forms.py index 42cd774d..2f93341d 100644 --- a/pineforge_codegen/analyzer/v6_forms.py +++ b/pineforge_codegen/analyzer/v6_forms.py @@ -9,7 +9,9 @@ from ..diagnostic_codes import render_diagnostic from ..errors import CompileError, Diagnostic, Level, Phase, SourceLocation from ..limits import TimeBudget, syntax_children +from ..loop_types import IntegerConstants, LoopTypeSources from ..pine_spelling import expr_start +from ..signatures import STR_FUNCTIONS def _refuse(code: str, node: ASTNode, filename: str, @@ -114,3 +116,63 @@ def validate_v6_forms(program: Program, filename: str, value for value in notes.get("param_defaults", ()) if isinstance(value, ASTNode) ) + + +def _substring_begin(node: FuncCall) -> ASTNode | None: + """Read the begin slot of a call that binds to a declared signature.""" + for signature in STR_FUNCTIONS["substring"].signatures: + names = [parameter.name for parameter in signature.params] + if (len(node.args) > len(names) + or set(node.kwargs) - set(names) + or set(names[:len(node.args)]) & set(node.kwargs)): + continue + bound = dict(zip(names, node.args)) + bound.update(node.kwargs) + if any(parameter.default is None and parameter.name not in bound + for parameter in signature.params): + continue + return bound["begin_pos"] + # Malformed calls retain their existing argument-binding handling. + return None + + +def validate_v6_constant_arguments( + program: Program, bindings: dict[int, tuple[int, str] | None], + filename: str, budget: TimeBudget | None = None, +) -> None: + """Check constant domains after lexical resolution, before call lowering.""" + if program.version != 6: + return + constants = None + stack = [program] + while stack: + node = stack.pop() + if budget is not None: + budget.check(node.loc, Phase.ANALYZER) + notes = node.annotations or {} + if notes.get("pine_version") == 5: + continue + if (isinstance(node, FuncCall) and _callee_name(node) == "str.substring" + and bindings.get(id(node.callee.object)) is None): + begin = _substring_begin(node) + if begin is not None: + if constants is None: + constants = IntegerConstants(LoopTypeSources(program), bindings, budget) + value = constants.value(begin) + if value is not None and value < 0: + raise CompileError([Diagnostic( + level=Level.ERROR, + phase=Phase.ANALYZER, + location=(expr_start(begin).loc or begin.loc or node.loc + or SourceLocation(filename, 1, 1, 1)), + message=( + "str.substring() requires 'begin_pos' " + "to be greater than or equal to 0." + ), + )]) + stack.extend(syntax_children(node)) + if isinstance(node, (FuncDef, MethodDef)): + stack.extend( + value for value in notes.get("param_defaults", ()) + if isinstance(value, ASTNode) + ) diff --git a/pineforge_codegen/analyzer/value_bindings.py b/pineforge_codegen/analyzer/value_bindings.py index 389d5106..66700e5e 100644 --- a/pineforge_codegen/analyzer/value_bindings.py +++ b/pineforge_codegen/analyzer/value_bindings.py @@ -230,7 +230,7 @@ def _missing(self, name: str, scope: _Scope, position: int, def resolve(self) -> None: if self.program.version != 6: return # Analyzer owns missing/unsupported-version diagnostics. - from .v6_forms import validate_v6_forms + from .v6_forms import validate_v6_constant_arguments, validate_v6_forms validate_v6_forms(self.program, self._filename, self.budget) # Import after Analyzer has loaded; reuse its namespace inventory. from .base import _BUILTIN_NAMESPACE_NAMES @@ -266,6 +266,9 @@ def resolve(self) -> None: ) node.annotations = {**(node.annotations or {}), _VALUE_USE: origin} self.check_use(node) + validate_v6_constant_arguments( + self.program, self.declaration_bindings, self._filename, self.budget, + ) def _body(self, body: list, scope: _Scope) -> None: for stmt in body: diff --git a/pineforge_codegen/loop_types.py b/pineforge_codegen/loop_types.py index 85e5ddc5..65b01d9d 100644 --- a/pineforge_codegen/loop_types.py +++ b/pineforge_codegen/loop_types.py @@ -387,23 +387,30 @@ def array_result(name, spec): return None -class _NumericValueQuery(_CounterQuery): - """One value query for storage and counters, including integral proofs.""" +class IntegerConstants: + """Memoized immutable integer arithmetic, shared by value and argument checks. - _use_reassignment_values = True + Missing values and runtime expressions supply no proof. In particular, na + is never converted to the native integer sentinel while checking a value. + """ - def __init__(self, check_query): - super().__init__(check_query.sources, check_query.bindings, check_query.scopes) - self.check_query = check_query - self.integer_constants = {} + def __init__(self, sources, bindings, budget=None): + self.sources = sources + self.bindings = bindings + self.budget = budget + self.values = {} - def _integer_constant(self, node): + def value(self, node): """Prove an integer using only immutable literal arithmetic.""" + if node is None: + return None + if self.budget is not None: + self.budget.check(node.loc, Phase.ANALYZER) key = id(node) - if key in self.integer_constants: - return self.integer_constants[key] + if key in self.values: + return self.values[key] # Memoize failures too; a provisional failure breaks alias cycles. - self.integer_constants[key] = None + self.values[key] = None value = None if isinstance(node, NumberLiteral) and type(node.value) is int: value = node.value @@ -415,14 +422,14 @@ def _integer_constant(self, node): and declaration.name not in self.sources.reassigned_names and declaration.type_hint in (None, "int") and (declaration.annotations or {}).get("qualifier") in (None, "const")): - value = self._integer_constant(declaration.value) + value = self.value(declaration.value) elif isinstance(node, UnaryOp) and node.op in {"+", "-"}: - operand = self._integer_constant(node.operand) + operand = self.value(node.operand) if operand is not None: value = -operand if node.op == "-" else operand elif isinstance(node, BinOp) and node.op in {"+", "-", "*", "/", "%"}: - left = self._integer_constant(node.left) - right = self._integer_constant(node.right) + left = self.value(node.left) + right = self.value(node.right) if left is not None and right is not None: if node.op == "+": value = left + right @@ -436,9 +443,23 @@ def _integer_constant(self, node): elif node.op == "%": remainder = abs(left) % abs(right) value = -remainder if left < 0 else remainder - self.integer_constants[key] = value + self.values[key] = value return value + +class _NumericValueQuery(_CounterQuery): + """One value query for storage and counters, including integral proofs.""" + + _use_reassignment_values = True + + def __init__(self, check_query): + super().__init__(check_query.sources, check_query.bindings, check_query.scopes) + self.check_query = check_query + self._integer_constants = IntegerConstants(self.sources, self.bindings) + + def _integer_constant(self, node): + return self._integer_constants.value(node) + def infer(self, node): if isinstance(node, BinOp) and node.op == "/": return _INT if self._integer_constant(node) is not None else _FLOAT diff --git a/tests/test_string_units.py b/tests/test_string_units.py index de118d6e..d23fc5c9 100644 --- a/tests/test_string_units.py +++ b/tests/test_string_units.py @@ -13,9 +13,10 @@ * ``measured-*``: the script text and the value TradingView returned for it (str.length, str.pos and str.substring on the strings "a" + one character + - "b"; and str.substring at its edges on "abc" and on a, an emoji, b: - TradingView refuses the two begin positions outside the text when it compiles - a script whose arguments are constants, where the engine stops the run); + "b"; and str.substring at its edges on "abc" and on a, an emoji, b). + A constant negative begin has a separate compiler-refusal assertion below; + the positive out-of-range constant still exercises the native runtime guard + and does not establish compiler agreement; * ``derived-*``: arithmetic on the same unit (an emoji counts two), not a separate reading; * ``design-*``: a choice the measurement does not decide. A cut between the two @@ -34,6 +35,11 @@ import pytest from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.diagnostic_codes import UNCATALOGUED +from pineforge_codegen.errors import CompileError, Level, Phase, SourceLocation +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser from tests._compile import run_emitted_tu HEADER = '//@version=6\nstrategy("string units")\n' @@ -532,20 +538,22 @@ def test_a_substring_past_the_end_stops_the_run_and_names_the_units(): assert "(which is 5) > this->size() (which is 4)" in stopped -REFUSED_ROWS = [ +RUNTIME_BEGIN_ROWS = [ pytest.param(4, "(which is 4) > this->size() (which is 3)", id="measured-ascii-begin-beyond-length-stops"), - pytest.param(-1, "(which is 18446744073709551615) > this->size() (which is 3)", - id="measured-ascii-begin-negative-stops"), + pytest.param('input.int(-1, "Begin")', + "(which is 18446744073709551615) > this->size() (which is 3)", + id="dynamic-ascii-begin-negative-stops"), ] -@pytest.mark.parametrize("begin, detail", REFUSED_ROWS) +@pytest.mark.parametrize("begin, detail", RUNTIME_BEGIN_ROWS) def test_a_begin_outside_the_text_stops_the_run(begin, detail): """Pin the existing native boundary guard and its diagnostic detail. - A native runtime stop does not establish agreement with a TradingView - compile-time refusal. That acceptance rule needs its separate real record. + A negative input remains a runtime value. The separate constant-argument + test below checks the compiler refusal; a runtime stop cannot satisfy it. + The positive out-of-range constant retains its existing runtime behavior. """ cpp = transpile(HEADER + f't = str.substring("abc", {begin})\nplot(close)\n') stopped = run_emitted_tu(cpp, STOP_DRIVER, opt="-O2", label="substring begin outside") @@ -553,6 +561,138 @@ def test_a_begin_outside_the_text_stops_the_run(begin, detail): assert detail in stopped +SUBSTRING_CALLS = [ + pytest.param('result_value = str.substring("abc", {begin})', id="positional"), + pytest.param('result_value = str.substring("abc", {begin}, 3)', id="positional-end"), + pytest.param('result_value = str.substring(source="abc", begin_pos={begin})', + id="named"), + pytest.param('result_value = str.substring(end_pos=3, begin_pos={begin}, source="abc")', + id="named-reordered"), + pytest.param('result_value = str.substring("abc", end_pos=3, begin_pos={begin})', + id="mixed-reordered"), +] + + +@pytest.mark.parametrize("check_support", [True, False]) +@pytest.mark.parametrize("call_template", SUBSTRING_CALLS) +@pytest.mark.parametrize("declarations, begin, first_token", [ + pytest.param("", "-1", "-", id="literal"), + pytest.param("", "-(2 + 1)", "-", id="negated-expression"), + pytest.param("", "1 - 4", "1", id="folded-subtraction"), + pytest.param("", "-8 / 2", "-", id="folded-exact-division"), + pytest.param("", "-8 % 3", "-", id="folded-remainder"), + pytest.param("begin = -2\n", "begin", "begin", id="inferred-constant"), + pytest.param("int start = -3\nbegin = start\n", "begin", "begin", + id="typed-alias"), + pytest.param("const int origin = 2\nconst int begin = origin - 5\n", + "begin", "begin", id="const-alias-chain"), +]) +def test_constant_negative_substring_begin_is_refused( + declarations, begin, first_token, call_template, check_support, +): + # TradingView CE10276 refuses the preserved constant-negative source at + # its begin argument. These are unit cases for the general constant rule, + # not additional TradingView measurements or a trade-value oracle. + source = HEADER + declarations + call_template.format(begin=begin) + "\nplot(close)\n" + with pytest.raises(CompileError) as caught: + transpile(source, filename="substring.pine", check_support=check_support) + diagnostic, = caught.value.diagnostics + assert diagnostic.level is Level.ERROR + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = call_template.index("{begin}") + 1 + assert diagnostic.location == SourceLocation( + "substring.pine", 3 + declarations.count("\n"), + column, column + len(first_token), + ) + # Allocation belongs to the catalog generator. Require registration without + # assigning an unreserved code in this test or in the diagnostic emitter. + assert diagnostic.code not in UNCATALOGUED.values() + assert diagnostic.args == {} + + +@pytest.mark.parametrize("body", [ + pytest.param('unused() => str.substring("abc", -1)\nplot(close)\n', + id="unused-body"), + pytest.param('plot(close, title=str.substring("abc", -1))\n', + id="visual-keyword"), +]) +def test_constant_negative_substring_is_checked_before_ignored_calls_are_lowered(body): + source = HEADER + body + with pytest.raises(CompileError) as caught: + transpile(source, filename="substring.pine") + diagnostic, = caught.value.diagnostics + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = body.splitlines()[0].index("-1") + 1 + assert diagnostic.location == SourceLocation("substring.pine", 3, column, column + 1) + + +def test_direct_analyzer_checks_constant_negative_substring_arguments(): + source = HEADER + 'result_value = str.substring("abc", -1)\n' + program = Parser( + Lexer(source, filename="substring.pine").tokenize(), + source=source, filename="substring.pine", + ).parse() + with pytest.raises(CompileError) as caught: + Analyzer(program, filename="substring.pine").analyze() + diagnostic, = caught.value.diagnostics + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = source.splitlines()[2].index("-1") + 1 + assert diagnostic.location == SourceLocation("substring.pine", 3, column, column + 1) + + +@pytest.mark.parametrize("declarations, begin", [ + pytest.param('begin = input.int(-1, "Begin")\n', "begin", id="input"), + pytest.param("begin = bar_index - 1\n", "begin", id="series"), + pytest.param("begin = -1\nbegin := 0\n", "begin", id="reassigned"), + pytest.param("var int begin = -1\n", "begin", id="persistent"), + pytest.param("int begin = na\n", "begin", id="missing-alias"), + pytest.param("", "int(na) + 1", id="missing-expression"), + pytest.param("", "0", id="zero"), + pytest.param("", "2", id="positive"), +]) +def test_substring_begin_without_negative_constant_proof_keeps_its_lowering( + declarations, begin, +): + # Acceptance and emitted runtime checking only; native value/stop assertions + # remain in the existing missing, edge and runtime-boundary tests. + cpp = transpile( + HEADER + declarations + + f'result_value = str.substring(end_pos=3, source="abc", begin_pos={begin})\n' + + "plot(close)\n" + ) + assert '_PF_STRING_STOP("substring_out_of_range"' in cpp + + +def test_substring_constant_proof_respects_parameter_bindings_and_authored_overloads(): + source = HEADER + ( + "const int begin = -2\n" + 'cut(int begin) => str.substring("abc", begin)\n' + "substring(int begin) => begin + 3\n" + "method substring(int self, int begin) => self + begin\n" + "base = 4\n" + "result_value = cut(1)\n" + "n = substring(-2) + base.substring(-3)\n" + "plot(n)\n" + ) + cpp = transpile(source) + assert '_PF_STRING_STOP("substring_out_of_range"' in cpp + + +def test_substring_negative_end_retains_runtime_validation(): + # This change has no compiler evidence for an end-domain refusal. + cpp = transpile(HEADER + 'result_value = str.substring("abc", 0, -1)\nplot(close)\n') + assert "str.substring: end_pos is less than begin_pos" in cpp + + @pytest.mark.parametrize("opt", NATIVE_OPTS) def test_substring_finite_end_before_start_stops_the_run(opt): # Issue #382 item 9: a finite end before begin stops the run. From 640bd7e31a394cfd6215325b12f3b76666e8da38 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:16:39 +0800 Subject: [PATCH 3/8] Register the constant substring boundary diagnostic Carry the generator-written PF-E2094 entry and semantic pin, with its authored presentation. The temporary PF-E2093 seed is excluded. --- pineforge_codegen/diagnostics_catalog.json | 1 + tests/fixtures/diagnostic_codes_pin.json | 1 + 2 files changed, 2 insertions(+) diff --git a/pineforge_codegen/diagnostics_catalog.json b/pineforge_codegen/diagnostics_catalog.json index eb4bc5ff..6c9a808a 100644 --- a/pineforge_codegen/diagnostics_catalog.json +++ b/pineforge_codegen/diagnostics_catalog.json @@ -235,6 +235,7 @@ "PF-E2090": {"severity": "error", "area": "analysis", "message": "input.time() needs a constant default in Pine v6: timestamp() with a time zone and calendar numbers is not a constant. Use the date-string form, for example timestamp(\"01 Jan 2023 00:00 +0000\").", "hint": "Use a single date string in timestamp or an integer timestamp literal for the default.", "explanation": "The time-zone overload of timestamp, which takes a zone and calendar numbers, returns a simple int, while a Pine v6 input.time default requires a const int.", "args": {}, "user_message": "input.time() requires a constant default in Pine v6."}, "PF-E2091": {"severity": "error", "area": "analysis", "message": "plot() cannot have a linewidth below 1 in Pine v6: use a value of 1 or greater.", "hint": "Set the plot linewidth to an integer of at least 1.", "explanation": "A Pine v6 plot call cannot use an integer literal below one for linewidth; variable and calculated widths retain their existing handling.", "args": {}, "user_message": "plot() requires a linewidth of at least 1 in Pine v6."}, "PF-E2092": {"severity": "error", "area": "analysis", "message": "\"range\" cannot be used as a variable or function name in Pine v6: use another name.", "hint": "Rename the variable, function or method declaration.", "explanation": "Pine v6 reserves range as a variable and callable declaration name; parameters, user type fields and member access retain their existing handling.", "args": {}, "user_message": "Choose another variable, function or method name instead of range in Pine v6."}, + "PF-E2094": {"severity": "error", "area": "analysis", "message": "str.substring() requires ''begin_pos'' to be greater than or equal to 0.", "hint": null, "explanation": "Pine v6 refuses a str.substring begin_pos proven to be a negative constant integer. Dynamic values retain runtime boundary checking, and missing positions retain their existing handling.", "args": {}, "user_message": "A constant begin_pos in str.substring() cannot be negative."}, "PF-E3001": {"severity": "error", "area": "request-security", "message": "request.security timeframe ''{name}'' cannot be resolved before the first bar: PineForge registers every requested timeframe before the script runs.", "hint": "Pass a literal, an input or a global to the helper that holds request.security.", "explanation": "The timeframe given to request.security is a name whose value PineForge cannot compute before the first bar; use a literal, an input or a global.", "args": {"name": {"kind": "identifier"}}, "user_message": "The timeframe given to request.security is a name whose value PineForge cannot compute before the first bar."}, "PF-E3002": {"severity": "error", "area": "request-security", "message": "request.security timeframe expression cannot be resolved before the first bar: PineForge registers every requested timeframe before the script runs.", "hint": "Pass a literal, an input or a global to the helper that holds request.security.", "explanation": "The timeframe expression given to request.security cannot be computed before the first bar; use a literal, an input or a global.", "args": {}, "user_message": "The timeframe expression given to request.security cannot be computed before the first bar."}, "PF-E3003": {"severity": "error", "area": "request-security", "message": "request.security timeframe switch arms must contain one expression", "hint": "Compute multi-statement timeframe logic before the request.security call or rewrite it as pure switch arms.", "explanation": "A switch that picks a request.security timeframe must have exactly one expression in each arm; compute multi-statement logic beforehand.", "args": {}, "user_message": "A switch that picks a request.security timeframe must have exactly one expression in each arm."}, diff --git a/tests/fixtures/diagnostic_codes_pin.json b/tests/fixtures/diagnostic_codes_pin.json index 0d4aaa73..05fb9000 100644 --- a/tests/fixtures/diagnostic_codes_pin.json +++ b/tests/fixtures/diagnostic_codes_pin.json @@ -236,6 +236,7 @@ "PF-E2090": "dcc4d05f50b881e5345de8c73ec7b7951bedc86c30a24d40bc0077fca5242a0d", "PF-E2091": "2d1b04fd25a4198619628b94bf8f90b4002f33e4081c2b780fea510c49e0b44a", "PF-E2092": "74b483fe343d239ec254f03e839b257611db5295b0433eabd15072f919950f3d", + "PF-E2094": "b693cc3415fcd4b4738b511967e6cfb2b88fd4aa8bba6f498f0c850bfc32485d", "PF-E3001": "585dad07e797bbe2cfffa9f45e14dee040be5570cc13f8e46fa4c31e953fc92a", "PF-E3002": "db089893825afe5ad1d91934c21107fc58451cadf3d71de2539b9c5805549433", "PF-E3003": "02571beee5163d1596d8a11fe49336dfcf65f86d4fed13254bac6b36023c3c79", From 93da9a9f79ad3bbd0b5682820bc36fb799716b88 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:35:52 +0800 Subject: [PATCH 4/8] Pin Unicode casing data and preserve complete regex matches Use generated Unicode 14 tables from checksum-bound source data, extend casing context across decimal runs, and preserve whole matches with shifted backreferences. Add parser, provenance, context and regex-compiler checks. This checkpoint still requires exact-head native and member verification. --- CHANGELOG.md | 11 +- .../codegen/_unicode_case_data.py | 3657 +++++++++++++++++ pineforge_codegen/codegen/helpers.py | 1 + pineforge_codegen/codegen/unicode_case.py | 133 +- pineforge_codegen/codegen/unicode_regex.py | 66 +- scripts/gen_unicode_case_data.py | 322 ++ tests/test_unicode_case_context.py | 39 + tests/test_unicode_case_data_generation.py | 113 + tests/test_unicode_regex_lowering.py | 71 + 9 files changed, 4339 insertions(+), 74 deletions(-) create mode 100644 pineforge_codegen/codegen/_unicode_case_data.py create mode 100644 scripts/gen_unicode_case_data.py create mode 100644 tests/test_unicode_case_context.py create mode 100644 tests/test_unicode_case_data_generation.py create mode 100644 tests/test_unicode_regex_lowering.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c573c5..f5c9b3b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,14 +35,17 @@ supported as exact pairs; on the 0.x line they are independent. See the - Array joins retain string return types through helper and method forwarding. Named, mixed and reordered receiver/separator arguments use their declared slots; an omitted separator keeps the default. -- `str.upper` and `str.lower` use full Unicode default mappings, including - supplementary letters, expansions and contextual final sigma. Generated - tables record the translating Python's Unicode version and do not depend - on the native locale. Locale tailoring and normalization are not applied. +- `str.upper` and `str.lower` use pinned Unicode 14.0.0 mappings and properties, + including supplementary letters, expansions and contextual final sigma. + Translating Python versions and the native locale do not select the data. + The context crosses decimal runs; locale tailoring and normalization are + not applied. The data pin does not identify TradingView's whole dialect. - `str.match` lowers its literals, character classes, ranges and repeated atoms as Unicode characters, including complete supplementary characters, so matches no longer depend on UTF-8 byte lengths. Word classes remain ASCII; string lengths and substring offsets still count UTF-16 units. + Matches return the complete match while preserving user captures and + backreferences for pattern evaluation. Regenerate C++ and relink affected strategies to use this correction. - String input defaults, options and titles preserve embedded NUL bytes in getters, metadata and checked receipts. Internal timeframe and footprint diff --git a/pineforge_codegen/codegen/_unicode_case_data.py b/pineforge_codegen/codegen/_unicode_case_data.py new file mode 100644 index 00000000..5696c7ec --- /dev/null +++ b/pineforge_codegen/codegen/_unicode_case_data.py @@ -0,0 +1,3657 @@ +"""Generated Unicode 14.0.0 casing data; do not edit table entries. + +Source: https://www.unicode.org/Public/14.0.0/ucd/ +Copyright 2021 Unicode, Inc. +Unicode data terms: https://www.unicode.org/terms_of_use.html +Regenerate with scripts/gen_unicode_case_data.py using pinned inputs.""" + +DATA_FORMAT = 1 +DATA_VERSION = "14.0.0" +SOURCE_SHA256 = ( + ("UnicodeData.txt", "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +SOURCE_BYTE_COUNTS = ( + ("UnicodeData.txt", 1897793), + ("SpecialCasing.txt", 16830), + ("DerivedCoreProperties.txt", 1040794), +) +EXCLUDED_LOCALE_RULES = ( + "0049: az Not_Before_Dot", + "0049: lt More_Above", + "0049: tr Not_Before_Dot", + "004A: lt More_Above", + "0069: az", + "0069: tr", + "00CC: lt", + "00CD: lt", + "0128: lt", + "012E: lt More_Above", + "0130: az", + "0130: tr", + "0307: az After_I", + "0307: lt After_Soft_Dotted", + "0307: tr After_I", +) + +LOWER_ROWS = ( + (0x41, 0x1, 0x61), + (0x42, 0x1, 0x62), + (0x43, 0x1, 0x63), + (0x44, 0x1, 0x64), + (0x45, 0x1, 0x65), + (0x46, 0x1, 0x66), + (0x47, 0x1, 0x67), + (0x48, 0x1, 0x68), + (0x49, 0x1, 0x69), + (0x4A, 0x1, 0x6A), + (0x4B, 0x1, 0x6B), + (0x4C, 0x1, 0x6C), + (0x4D, 0x1, 0x6D), + (0x4E, 0x1, 0x6E), + (0x4F, 0x1, 0x6F), + (0x50, 0x1, 0x70), + (0x51, 0x1, 0x71), + (0x52, 0x1, 0x72), + (0x53, 0x1, 0x73), + (0x54, 0x1, 0x74), + (0x55, 0x1, 0x75), + (0x56, 0x1, 0x76), + (0x57, 0x1, 0x77), + (0x58, 0x1, 0x78), + (0x59, 0x1, 0x79), + (0x5A, 0x1, 0x7A), + (0xC0, 0x1, 0xE0), + (0xC1, 0x1, 0xE1), + (0xC2, 0x1, 0xE2), + (0xC3, 0x1, 0xE3), + (0xC4, 0x1, 0xE4), + (0xC5, 0x1, 0xE5), + (0xC6, 0x1, 0xE6), + (0xC7, 0x1, 0xE7), + (0xC8, 0x1, 0xE8), + (0xC9, 0x1, 0xE9), + (0xCA, 0x1, 0xEA), + (0xCB, 0x1, 0xEB), + (0xCC, 0x1, 0xEC), + (0xCD, 0x1, 0xED), + (0xCE, 0x1, 0xEE), + (0xCF, 0x1, 0xEF), + (0xD0, 0x1, 0xF0), + (0xD1, 0x1, 0xF1), + (0xD2, 0x1, 0xF2), + (0xD3, 0x1, 0xF3), + (0xD4, 0x1, 0xF4), + (0xD5, 0x1, 0xF5), + (0xD6, 0x1, 0xF6), + (0xD8, 0x1, 0xF8), + (0xD9, 0x1, 0xF9), + (0xDA, 0x1, 0xFA), + (0xDB, 0x1, 0xFB), + (0xDC, 0x1, 0xFC), + (0xDD, 0x1, 0xFD), + (0xDE, 0x1, 0xFE), + (0x100, 0x1, 0x101), + (0x102, 0x1, 0x103), + (0x104, 0x1, 0x105), + (0x106, 0x1, 0x107), + (0x108, 0x1, 0x109), + (0x10A, 0x1, 0x10B), + (0x10C, 0x1, 0x10D), + (0x10E, 0x1, 0x10F), + (0x110, 0x1, 0x111), + (0x112, 0x1, 0x113), + (0x114, 0x1, 0x115), + (0x116, 0x1, 0x117), + (0x118, 0x1, 0x119), + (0x11A, 0x1, 0x11B), + (0x11C, 0x1, 0x11D), + (0x11E, 0x1, 0x11F), + (0x120, 0x1, 0x121), + (0x122, 0x1, 0x123), + (0x124, 0x1, 0x125), + (0x126, 0x1, 0x127), + (0x128, 0x1, 0x129), + (0x12A, 0x1, 0x12B), + (0x12C, 0x1, 0x12D), + (0x12E, 0x1, 0x12F), + (0x130, 0x2, 0x69, 0x307), + (0x132, 0x1, 0x133), + (0x134, 0x1, 0x135), + (0x136, 0x1, 0x137), + (0x139, 0x1, 0x13A), + (0x13B, 0x1, 0x13C), + (0x13D, 0x1, 0x13E), + (0x13F, 0x1, 0x140), + (0x141, 0x1, 0x142), + (0x143, 0x1, 0x144), + (0x145, 0x1, 0x146), + (0x147, 0x1, 0x148), + (0x14A, 0x1, 0x14B), + (0x14C, 0x1, 0x14D), + (0x14E, 0x1, 0x14F), + (0x150, 0x1, 0x151), + (0x152, 0x1, 0x153), + (0x154, 0x1, 0x155), + (0x156, 0x1, 0x157), + (0x158, 0x1, 0x159), + (0x15A, 0x1, 0x15B), + (0x15C, 0x1, 0x15D), + (0x15E, 0x1, 0x15F), + (0x160, 0x1, 0x161), + (0x162, 0x1, 0x163), + (0x164, 0x1, 0x165), + (0x166, 0x1, 0x167), + (0x168, 0x1, 0x169), + (0x16A, 0x1, 0x16B), + (0x16C, 0x1, 0x16D), + (0x16E, 0x1, 0x16F), + (0x170, 0x1, 0x171), + (0x172, 0x1, 0x173), + (0x174, 0x1, 0x175), + (0x176, 0x1, 0x177), + (0x178, 0x1, 0xFF), + (0x179, 0x1, 0x17A), + (0x17B, 0x1, 0x17C), + (0x17D, 0x1, 0x17E), + (0x181, 0x1, 0x253), + (0x182, 0x1, 0x183), + (0x184, 0x1, 0x185), + (0x186, 0x1, 0x254), + (0x187, 0x1, 0x188), + (0x189, 0x1, 0x256), + (0x18A, 0x1, 0x257), + (0x18B, 0x1, 0x18C), + (0x18E, 0x1, 0x1DD), + (0x18F, 0x1, 0x259), + (0x190, 0x1, 0x25B), + (0x191, 0x1, 0x192), + (0x193, 0x1, 0x260), + (0x194, 0x1, 0x263), + (0x196, 0x1, 0x269), + (0x197, 0x1, 0x268), + (0x198, 0x1, 0x199), + (0x19C, 0x1, 0x26F), + (0x19D, 0x1, 0x272), + (0x19F, 0x1, 0x275), + (0x1A0, 0x1, 0x1A1), + (0x1A2, 0x1, 0x1A3), + (0x1A4, 0x1, 0x1A5), + (0x1A6, 0x1, 0x280), + (0x1A7, 0x1, 0x1A8), + (0x1A9, 0x1, 0x283), + (0x1AC, 0x1, 0x1AD), + (0x1AE, 0x1, 0x288), + (0x1AF, 0x1, 0x1B0), + (0x1B1, 0x1, 0x28A), + (0x1B2, 0x1, 0x28B), + (0x1B3, 0x1, 0x1B4), + (0x1B5, 0x1, 0x1B6), + (0x1B7, 0x1, 0x292), + (0x1B8, 0x1, 0x1B9), + (0x1BC, 0x1, 0x1BD), + (0x1C4, 0x1, 0x1C6), + (0x1C5, 0x1, 0x1C6), + (0x1C7, 0x1, 0x1C9), + (0x1C8, 0x1, 0x1C9), + (0x1CA, 0x1, 0x1CC), + (0x1CB, 0x1, 0x1CC), + (0x1CD, 0x1, 0x1CE), + (0x1CF, 0x1, 0x1D0), + (0x1D1, 0x1, 0x1D2), + (0x1D3, 0x1, 0x1D4), + (0x1D5, 0x1, 0x1D6), + (0x1D7, 0x1, 0x1D8), + (0x1D9, 0x1, 0x1DA), + (0x1DB, 0x1, 0x1DC), + (0x1DE, 0x1, 0x1DF), + (0x1E0, 0x1, 0x1E1), + (0x1E2, 0x1, 0x1E3), + (0x1E4, 0x1, 0x1E5), + (0x1E6, 0x1, 0x1E7), + (0x1E8, 0x1, 0x1E9), + (0x1EA, 0x1, 0x1EB), + (0x1EC, 0x1, 0x1ED), + (0x1EE, 0x1, 0x1EF), + (0x1F1, 0x1, 0x1F3), + (0x1F2, 0x1, 0x1F3), + (0x1F4, 0x1, 0x1F5), + (0x1F6, 0x1, 0x195), + (0x1F7, 0x1, 0x1BF), + (0x1F8, 0x1, 0x1F9), + (0x1FA, 0x1, 0x1FB), + (0x1FC, 0x1, 0x1FD), + (0x1FE, 0x1, 0x1FF), + (0x200, 0x1, 0x201), + (0x202, 0x1, 0x203), + (0x204, 0x1, 0x205), + (0x206, 0x1, 0x207), + (0x208, 0x1, 0x209), + (0x20A, 0x1, 0x20B), + (0x20C, 0x1, 0x20D), + (0x20E, 0x1, 0x20F), + (0x210, 0x1, 0x211), + (0x212, 0x1, 0x213), + (0x214, 0x1, 0x215), + (0x216, 0x1, 0x217), + (0x218, 0x1, 0x219), + (0x21A, 0x1, 0x21B), + (0x21C, 0x1, 0x21D), + (0x21E, 0x1, 0x21F), + (0x220, 0x1, 0x19E), + (0x222, 0x1, 0x223), + (0x224, 0x1, 0x225), + (0x226, 0x1, 0x227), + (0x228, 0x1, 0x229), + (0x22A, 0x1, 0x22B), + (0x22C, 0x1, 0x22D), + (0x22E, 0x1, 0x22F), + (0x230, 0x1, 0x231), + (0x232, 0x1, 0x233), + (0x23A, 0x1, 0x2C65), + (0x23B, 0x1, 0x23C), + (0x23D, 0x1, 0x19A), + (0x23E, 0x1, 0x2C66), + (0x241, 0x1, 0x242), + (0x243, 0x1, 0x180), + (0x244, 0x1, 0x289), + (0x245, 0x1, 0x28C), + (0x246, 0x1, 0x247), + (0x248, 0x1, 0x249), + (0x24A, 0x1, 0x24B), + (0x24C, 0x1, 0x24D), + (0x24E, 0x1, 0x24F), + (0x370, 0x1, 0x371), + (0x372, 0x1, 0x373), + (0x376, 0x1, 0x377), + (0x37F, 0x1, 0x3F3), + (0x386, 0x1, 0x3AC), + (0x388, 0x1, 0x3AD), + (0x389, 0x1, 0x3AE), + (0x38A, 0x1, 0x3AF), + (0x38C, 0x1, 0x3CC), + (0x38E, 0x1, 0x3CD), + (0x38F, 0x1, 0x3CE), + (0x391, 0x1, 0x3B1), + (0x392, 0x1, 0x3B2), + (0x393, 0x1, 0x3B3), + (0x394, 0x1, 0x3B4), + (0x395, 0x1, 0x3B5), + (0x396, 0x1, 0x3B6), + (0x397, 0x1, 0x3B7), + (0x398, 0x1, 0x3B8), + (0x399, 0x1, 0x3B9), + (0x39A, 0x1, 0x3BA), + (0x39B, 0x1, 0x3BB), + (0x39C, 0x1, 0x3BC), + (0x39D, 0x1, 0x3BD), + (0x39E, 0x1, 0x3BE), + (0x39F, 0x1, 0x3BF), + (0x3A0, 0x1, 0x3C0), + (0x3A1, 0x1, 0x3C1), + (0x3A3, 0x1, 0x3C3), + (0x3A4, 0x1, 0x3C4), + (0x3A5, 0x1, 0x3C5), + (0x3A6, 0x1, 0x3C6), + (0x3A7, 0x1, 0x3C7), + (0x3A8, 0x1, 0x3C8), + (0x3A9, 0x1, 0x3C9), + (0x3AA, 0x1, 0x3CA), + (0x3AB, 0x1, 0x3CB), + (0x3CF, 0x1, 0x3D7), + (0x3D8, 0x1, 0x3D9), + (0x3DA, 0x1, 0x3DB), + (0x3DC, 0x1, 0x3DD), + (0x3DE, 0x1, 0x3DF), + (0x3E0, 0x1, 0x3E1), + (0x3E2, 0x1, 0x3E3), + (0x3E4, 0x1, 0x3E5), + (0x3E6, 0x1, 0x3E7), + (0x3E8, 0x1, 0x3E9), + (0x3EA, 0x1, 0x3EB), + (0x3EC, 0x1, 0x3ED), + (0x3EE, 0x1, 0x3EF), + (0x3F4, 0x1, 0x3B8), + (0x3F7, 0x1, 0x3F8), + (0x3F9, 0x1, 0x3F2), + (0x3FA, 0x1, 0x3FB), + (0x3FD, 0x1, 0x37B), + (0x3FE, 0x1, 0x37C), + (0x3FF, 0x1, 0x37D), + (0x400, 0x1, 0x450), + (0x401, 0x1, 0x451), + (0x402, 0x1, 0x452), + (0x403, 0x1, 0x453), + (0x404, 0x1, 0x454), + (0x405, 0x1, 0x455), + (0x406, 0x1, 0x456), + (0x407, 0x1, 0x457), + (0x408, 0x1, 0x458), + (0x409, 0x1, 0x459), + (0x40A, 0x1, 0x45A), + (0x40B, 0x1, 0x45B), + (0x40C, 0x1, 0x45C), + (0x40D, 0x1, 0x45D), + (0x40E, 0x1, 0x45E), + (0x40F, 0x1, 0x45F), + (0x410, 0x1, 0x430), + (0x411, 0x1, 0x431), + (0x412, 0x1, 0x432), + (0x413, 0x1, 0x433), + (0x414, 0x1, 0x434), + (0x415, 0x1, 0x435), + (0x416, 0x1, 0x436), + (0x417, 0x1, 0x437), + (0x418, 0x1, 0x438), + (0x419, 0x1, 0x439), + (0x41A, 0x1, 0x43A), + (0x41B, 0x1, 0x43B), + (0x41C, 0x1, 0x43C), + (0x41D, 0x1, 0x43D), + (0x41E, 0x1, 0x43E), + (0x41F, 0x1, 0x43F), + (0x420, 0x1, 0x440), + (0x421, 0x1, 0x441), + (0x422, 0x1, 0x442), + (0x423, 0x1, 0x443), + (0x424, 0x1, 0x444), + (0x425, 0x1, 0x445), + (0x426, 0x1, 0x446), + (0x427, 0x1, 0x447), + (0x428, 0x1, 0x448), + (0x429, 0x1, 0x449), + (0x42A, 0x1, 0x44A), + (0x42B, 0x1, 0x44B), + (0x42C, 0x1, 0x44C), + (0x42D, 0x1, 0x44D), + (0x42E, 0x1, 0x44E), + (0x42F, 0x1, 0x44F), + (0x460, 0x1, 0x461), + (0x462, 0x1, 0x463), + (0x464, 0x1, 0x465), + (0x466, 0x1, 0x467), + (0x468, 0x1, 0x469), + (0x46A, 0x1, 0x46B), + (0x46C, 0x1, 0x46D), + (0x46E, 0x1, 0x46F), + (0x470, 0x1, 0x471), + (0x472, 0x1, 0x473), + (0x474, 0x1, 0x475), + (0x476, 0x1, 0x477), + (0x478, 0x1, 0x479), + (0x47A, 0x1, 0x47B), + (0x47C, 0x1, 0x47D), + (0x47E, 0x1, 0x47F), + (0x480, 0x1, 0x481), + (0x48A, 0x1, 0x48B), + (0x48C, 0x1, 0x48D), + (0x48E, 0x1, 0x48F), + (0x490, 0x1, 0x491), + (0x492, 0x1, 0x493), + (0x494, 0x1, 0x495), + (0x496, 0x1, 0x497), + (0x498, 0x1, 0x499), + (0x49A, 0x1, 0x49B), + (0x49C, 0x1, 0x49D), + (0x49E, 0x1, 0x49F), + (0x4A0, 0x1, 0x4A1), + (0x4A2, 0x1, 0x4A3), + (0x4A4, 0x1, 0x4A5), + (0x4A6, 0x1, 0x4A7), + (0x4A8, 0x1, 0x4A9), + (0x4AA, 0x1, 0x4AB), + (0x4AC, 0x1, 0x4AD), + (0x4AE, 0x1, 0x4AF), + (0x4B0, 0x1, 0x4B1), + (0x4B2, 0x1, 0x4B3), + (0x4B4, 0x1, 0x4B5), + (0x4B6, 0x1, 0x4B7), + (0x4B8, 0x1, 0x4B9), + (0x4BA, 0x1, 0x4BB), + (0x4BC, 0x1, 0x4BD), + (0x4BE, 0x1, 0x4BF), + (0x4C0, 0x1, 0x4CF), + (0x4C1, 0x1, 0x4C2), + (0x4C3, 0x1, 0x4C4), + (0x4C5, 0x1, 0x4C6), + (0x4C7, 0x1, 0x4C8), + (0x4C9, 0x1, 0x4CA), + (0x4CB, 0x1, 0x4CC), + (0x4CD, 0x1, 0x4CE), + (0x4D0, 0x1, 0x4D1), + (0x4D2, 0x1, 0x4D3), + (0x4D4, 0x1, 0x4D5), + (0x4D6, 0x1, 0x4D7), + (0x4D8, 0x1, 0x4D9), + (0x4DA, 0x1, 0x4DB), + (0x4DC, 0x1, 0x4DD), + (0x4DE, 0x1, 0x4DF), + (0x4E0, 0x1, 0x4E1), + (0x4E2, 0x1, 0x4E3), + (0x4E4, 0x1, 0x4E5), + (0x4E6, 0x1, 0x4E7), + (0x4E8, 0x1, 0x4E9), + (0x4EA, 0x1, 0x4EB), + (0x4EC, 0x1, 0x4ED), + (0x4EE, 0x1, 0x4EF), + (0x4F0, 0x1, 0x4F1), + (0x4F2, 0x1, 0x4F3), + (0x4F4, 0x1, 0x4F5), + (0x4F6, 0x1, 0x4F7), + (0x4F8, 0x1, 0x4F9), + (0x4FA, 0x1, 0x4FB), + (0x4FC, 0x1, 0x4FD), + (0x4FE, 0x1, 0x4FF), + (0x500, 0x1, 0x501), + (0x502, 0x1, 0x503), + (0x504, 0x1, 0x505), + (0x506, 0x1, 0x507), + (0x508, 0x1, 0x509), + (0x50A, 0x1, 0x50B), + (0x50C, 0x1, 0x50D), + (0x50E, 0x1, 0x50F), + (0x510, 0x1, 0x511), + (0x512, 0x1, 0x513), + (0x514, 0x1, 0x515), + (0x516, 0x1, 0x517), + (0x518, 0x1, 0x519), + (0x51A, 0x1, 0x51B), + (0x51C, 0x1, 0x51D), + (0x51E, 0x1, 0x51F), + (0x520, 0x1, 0x521), + (0x522, 0x1, 0x523), + (0x524, 0x1, 0x525), + (0x526, 0x1, 0x527), + (0x528, 0x1, 0x529), + (0x52A, 0x1, 0x52B), + (0x52C, 0x1, 0x52D), + (0x52E, 0x1, 0x52F), + (0x531, 0x1, 0x561), + (0x532, 0x1, 0x562), + (0x533, 0x1, 0x563), + (0x534, 0x1, 0x564), + (0x535, 0x1, 0x565), + (0x536, 0x1, 0x566), + (0x537, 0x1, 0x567), + (0x538, 0x1, 0x568), + (0x539, 0x1, 0x569), + (0x53A, 0x1, 0x56A), + (0x53B, 0x1, 0x56B), + (0x53C, 0x1, 0x56C), + (0x53D, 0x1, 0x56D), + (0x53E, 0x1, 0x56E), + (0x53F, 0x1, 0x56F), + (0x540, 0x1, 0x570), + (0x541, 0x1, 0x571), + (0x542, 0x1, 0x572), + (0x543, 0x1, 0x573), + (0x544, 0x1, 0x574), + (0x545, 0x1, 0x575), + (0x546, 0x1, 0x576), + (0x547, 0x1, 0x577), + (0x548, 0x1, 0x578), + (0x549, 0x1, 0x579), + (0x54A, 0x1, 0x57A), + (0x54B, 0x1, 0x57B), + (0x54C, 0x1, 0x57C), + (0x54D, 0x1, 0x57D), + (0x54E, 0x1, 0x57E), + (0x54F, 0x1, 0x57F), + (0x550, 0x1, 0x580), + (0x551, 0x1, 0x581), + (0x552, 0x1, 0x582), + (0x553, 0x1, 0x583), + (0x554, 0x1, 0x584), + (0x555, 0x1, 0x585), + (0x556, 0x1, 0x586), + (0x10A0, 0x1, 0x2D00), + (0x10A1, 0x1, 0x2D01), + (0x10A2, 0x1, 0x2D02), + (0x10A3, 0x1, 0x2D03), + (0x10A4, 0x1, 0x2D04), + (0x10A5, 0x1, 0x2D05), + (0x10A6, 0x1, 0x2D06), + (0x10A7, 0x1, 0x2D07), + (0x10A8, 0x1, 0x2D08), + (0x10A9, 0x1, 0x2D09), + (0x10AA, 0x1, 0x2D0A), + (0x10AB, 0x1, 0x2D0B), + (0x10AC, 0x1, 0x2D0C), + (0x10AD, 0x1, 0x2D0D), + (0x10AE, 0x1, 0x2D0E), + (0x10AF, 0x1, 0x2D0F), + (0x10B0, 0x1, 0x2D10), + (0x10B1, 0x1, 0x2D11), + (0x10B2, 0x1, 0x2D12), + (0x10B3, 0x1, 0x2D13), + (0x10B4, 0x1, 0x2D14), + (0x10B5, 0x1, 0x2D15), + (0x10B6, 0x1, 0x2D16), + (0x10B7, 0x1, 0x2D17), + (0x10B8, 0x1, 0x2D18), + (0x10B9, 0x1, 0x2D19), + (0x10BA, 0x1, 0x2D1A), + (0x10BB, 0x1, 0x2D1B), + (0x10BC, 0x1, 0x2D1C), + (0x10BD, 0x1, 0x2D1D), + (0x10BE, 0x1, 0x2D1E), + (0x10BF, 0x1, 0x2D1F), + (0x10C0, 0x1, 0x2D20), + (0x10C1, 0x1, 0x2D21), + (0x10C2, 0x1, 0x2D22), + (0x10C3, 0x1, 0x2D23), + (0x10C4, 0x1, 0x2D24), + (0x10C5, 0x1, 0x2D25), + (0x10C7, 0x1, 0x2D27), + (0x10CD, 0x1, 0x2D2D), + (0x13A0, 0x1, 0xAB70), + (0x13A1, 0x1, 0xAB71), + (0x13A2, 0x1, 0xAB72), + (0x13A3, 0x1, 0xAB73), + (0x13A4, 0x1, 0xAB74), + (0x13A5, 0x1, 0xAB75), + (0x13A6, 0x1, 0xAB76), + (0x13A7, 0x1, 0xAB77), + (0x13A8, 0x1, 0xAB78), + (0x13A9, 0x1, 0xAB79), + (0x13AA, 0x1, 0xAB7A), + (0x13AB, 0x1, 0xAB7B), + (0x13AC, 0x1, 0xAB7C), + (0x13AD, 0x1, 0xAB7D), + (0x13AE, 0x1, 0xAB7E), + (0x13AF, 0x1, 0xAB7F), + (0x13B0, 0x1, 0xAB80), + (0x13B1, 0x1, 0xAB81), + (0x13B2, 0x1, 0xAB82), + (0x13B3, 0x1, 0xAB83), + (0x13B4, 0x1, 0xAB84), + (0x13B5, 0x1, 0xAB85), + (0x13B6, 0x1, 0xAB86), + (0x13B7, 0x1, 0xAB87), + (0x13B8, 0x1, 0xAB88), + (0x13B9, 0x1, 0xAB89), + (0x13BA, 0x1, 0xAB8A), + (0x13BB, 0x1, 0xAB8B), + (0x13BC, 0x1, 0xAB8C), + (0x13BD, 0x1, 0xAB8D), + (0x13BE, 0x1, 0xAB8E), + (0x13BF, 0x1, 0xAB8F), + (0x13C0, 0x1, 0xAB90), + (0x13C1, 0x1, 0xAB91), + (0x13C2, 0x1, 0xAB92), + (0x13C3, 0x1, 0xAB93), + (0x13C4, 0x1, 0xAB94), + (0x13C5, 0x1, 0xAB95), + (0x13C6, 0x1, 0xAB96), + (0x13C7, 0x1, 0xAB97), + (0x13C8, 0x1, 0xAB98), + (0x13C9, 0x1, 0xAB99), + (0x13CA, 0x1, 0xAB9A), + (0x13CB, 0x1, 0xAB9B), + (0x13CC, 0x1, 0xAB9C), + (0x13CD, 0x1, 0xAB9D), + (0x13CE, 0x1, 0xAB9E), + (0x13CF, 0x1, 0xAB9F), + (0x13D0, 0x1, 0xABA0), + (0x13D1, 0x1, 0xABA1), + (0x13D2, 0x1, 0xABA2), + (0x13D3, 0x1, 0xABA3), + (0x13D4, 0x1, 0xABA4), + (0x13D5, 0x1, 0xABA5), + (0x13D6, 0x1, 0xABA6), + (0x13D7, 0x1, 0xABA7), + (0x13D8, 0x1, 0xABA8), + (0x13D9, 0x1, 0xABA9), + (0x13DA, 0x1, 0xABAA), + (0x13DB, 0x1, 0xABAB), + (0x13DC, 0x1, 0xABAC), + (0x13DD, 0x1, 0xABAD), + (0x13DE, 0x1, 0xABAE), + (0x13DF, 0x1, 0xABAF), + (0x13E0, 0x1, 0xABB0), + (0x13E1, 0x1, 0xABB1), + (0x13E2, 0x1, 0xABB2), + (0x13E3, 0x1, 0xABB3), + (0x13E4, 0x1, 0xABB4), + (0x13E5, 0x1, 0xABB5), + (0x13E6, 0x1, 0xABB6), + (0x13E7, 0x1, 0xABB7), + (0x13E8, 0x1, 0xABB8), + (0x13E9, 0x1, 0xABB9), + (0x13EA, 0x1, 0xABBA), + (0x13EB, 0x1, 0xABBB), + (0x13EC, 0x1, 0xABBC), + (0x13ED, 0x1, 0xABBD), + (0x13EE, 0x1, 0xABBE), + (0x13EF, 0x1, 0xABBF), + (0x13F0, 0x1, 0x13F8), + (0x13F1, 0x1, 0x13F9), + (0x13F2, 0x1, 0x13FA), + (0x13F3, 0x1, 0x13FB), + (0x13F4, 0x1, 0x13FC), + (0x13F5, 0x1, 0x13FD), + (0x1C90, 0x1, 0x10D0), + (0x1C91, 0x1, 0x10D1), + (0x1C92, 0x1, 0x10D2), + (0x1C93, 0x1, 0x10D3), + (0x1C94, 0x1, 0x10D4), + (0x1C95, 0x1, 0x10D5), + (0x1C96, 0x1, 0x10D6), + (0x1C97, 0x1, 0x10D7), + (0x1C98, 0x1, 0x10D8), + (0x1C99, 0x1, 0x10D9), + (0x1C9A, 0x1, 0x10DA), + (0x1C9B, 0x1, 0x10DB), + (0x1C9C, 0x1, 0x10DC), + (0x1C9D, 0x1, 0x10DD), + (0x1C9E, 0x1, 0x10DE), + (0x1C9F, 0x1, 0x10DF), + (0x1CA0, 0x1, 0x10E0), + (0x1CA1, 0x1, 0x10E1), + (0x1CA2, 0x1, 0x10E2), + (0x1CA3, 0x1, 0x10E3), + (0x1CA4, 0x1, 0x10E4), + (0x1CA5, 0x1, 0x10E5), + (0x1CA6, 0x1, 0x10E6), + (0x1CA7, 0x1, 0x10E7), + (0x1CA8, 0x1, 0x10E8), + (0x1CA9, 0x1, 0x10E9), + (0x1CAA, 0x1, 0x10EA), + (0x1CAB, 0x1, 0x10EB), + (0x1CAC, 0x1, 0x10EC), + (0x1CAD, 0x1, 0x10ED), + (0x1CAE, 0x1, 0x10EE), + (0x1CAF, 0x1, 0x10EF), + (0x1CB0, 0x1, 0x10F0), + (0x1CB1, 0x1, 0x10F1), + (0x1CB2, 0x1, 0x10F2), + (0x1CB3, 0x1, 0x10F3), + (0x1CB4, 0x1, 0x10F4), + (0x1CB5, 0x1, 0x10F5), + (0x1CB6, 0x1, 0x10F6), + (0x1CB7, 0x1, 0x10F7), + (0x1CB8, 0x1, 0x10F8), + (0x1CB9, 0x1, 0x10F9), + (0x1CBA, 0x1, 0x10FA), + (0x1CBD, 0x1, 0x10FD), + (0x1CBE, 0x1, 0x10FE), + (0x1CBF, 0x1, 0x10FF), + (0x1E00, 0x1, 0x1E01), + (0x1E02, 0x1, 0x1E03), + (0x1E04, 0x1, 0x1E05), + (0x1E06, 0x1, 0x1E07), + (0x1E08, 0x1, 0x1E09), + (0x1E0A, 0x1, 0x1E0B), + (0x1E0C, 0x1, 0x1E0D), + (0x1E0E, 0x1, 0x1E0F), + (0x1E10, 0x1, 0x1E11), + (0x1E12, 0x1, 0x1E13), + (0x1E14, 0x1, 0x1E15), + (0x1E16, 0x1, 0x1E17), + (0x1E18, 0x1, 0x1E19), + (0x1E1A, 0x1, 0x1E1B), + (0x1E1C, 0x1, 0x1E1D), + (0x1E1E, 0x1, 0x1E1F), + (0x1E20, 0x1, 0x1E21), + (0x1E22, 0x1, 0x1E23), + (0x1E24, 0x1, 0x1E25), + (0x1E26, 0x1, 0x1E27), + (0x1E28, 0x1, 0x1E29), + (0x1E2A, 0x1, 0x1E2B), + (0x1E2C, 0x1, 0x1E2D), + (0x1E2E, 0x1, 0x1E2F), + (0x1E30, 0x1, 0x1E31), + (0x1E32, 0x1, 0x1E33), + (0x1E34, 0x1, 0x1E35), + (0x1E36, 0x1, 0x1E37), + (0x1E38, 0x1, 0x1E39), + (0x1E3A, 0x1, 0x1E3B), + (0x1E3C, 0x1, 0x1E3D), + (0x1E3E, 0x1, 0x1E3F), + (0x1E40, 0x1, 0x1E41), + (0x1E42, 0x1, 0x1E43), + (0x1E44, 0x1, 0x1E45), + (0x1E46, 0x1, 0x1E47), + (0x1E48, 0x1, 0x1E49), + (0x1E4A, 0x1, 0x1E4B), + (0x1E4C, 0x1, 0x1E4D), + (0x1E4E, 0x1, 0x1E4F), + (0x1E50, 0x1, 0x1E51), + (0x1E52, 0x1, 0x1E53), + (0x1E54, 0x1, 0x1E55), + (0x1E56, 0x1, 0x1E57), + (0x1E58, 0x1, 0x1E59), + (0x1E5A, 0x1, 0x1E5B), + (0x1E5C, 0x1, 0x1E5D), + (0x1E5E, 0x1, 0x1E5F), + (0x1E60, 0x1, 0x1E61), + (0x1E62, 0x1, 0x1E63), + (0x1E64, 0x1, 0x1E65), + (0x1E66, 0x1, 0x1E67), + (0x1E68, 0x1, 0x1E69), + (0x1E6A, 0x1, 0x1E6B), + (0x1E6C, 0x1, 0x1E6D), + (0x1E6E, 0x1, 0x1E6F), + (0x1E70, 0x1, 0x1E71), + (0x1E72, 0x1, 0x1E73), + (0x1E74, 0x1, 0x1E75), + (0x1E76, 0x1, 0x1E77), + (0x1E78, 0x1, 0x1E79), + (0x1E7A, 0x1, 0x1E7B), + (0x1E7C, 0x1, 0x1E7D), + (0x1E7E, 0x1, 0x1E7F), + (0x1E80, 0x1, 0x1E81), + (0x1E82, 0x1, 0x1E83), + (0x1E84, 0x1, 0x1E85), + (0x1E86, 0x1, 0x1E87), + (0x1E88, 0x1, 0x1E89), + (0x1E8A, 0x1, 0x1E8B), + (0x1E8C, 0x1, 0x1E8D), + (0x1E8E, 0x1, 0x1E8F), + (0x1E90, 0x1, 0x1E91), + (0x1E92, 0x1, 0x1E93), + (0x1E94, 0x1, 0x1E95), + (0x1E9E, 0x1, 0xDF), + (0x1EA0, 0x1, 0x1EA1), + (0x1EA2, 0x1, 0x1EA3), + (0x1EA4, 0x1, 0x1EA5), + (0x1EA6, 0x1, 0x1EA7), + (0x1EA8, 0x1, 0x1EA9), + (0x1EAA, 0x1, 0x1EAB), + (0x1EAC, 0x1, 0x1EAD), + (0x1EAE, 0x1, 0x1EAF), + (0x1EB0, 0x1, 0x1EB1), + (0x1EB2, 0x1, 0x1EB3), + (0x1EB4, 0x1, 0x1EB5), + (0x1EB6, 0x1, 0x1EB7), + (0x1EB8, 0x1, 0x1EB9), + (0x1EBA, 0x1, 0x1EBB), + (0x1EBC, 0x1, 0x1EBD), + (0x1EBE, 0x1, 0x1EBF), + (0x1EC0, 0x1, 0x1EC1), + (0x1EC2, 0x1, 0x1EC3), + (0x1EC4, 0x1, 0x1EC5), + (0x1EC6, 0x1, 0x1EC7), + (0x1EC8, 0x1, 0x1EC9), + (0x1ECA, 0x1, 0x1ECB), + (0x1ECC, 0x1, 0x1ECD), + (0x1ECE, 0x1, 0x1ECF), + (0x1ED0, 0x1, 0x1ED1), + (0x1ED2, 0x1, 0x1ED3), + (0x1ED4, 0x1, 0x1ED5), + (0x1ED6, 0x1, 0x1ED7), + (0x1ED8, 0x1, 0x1ED9), + (0x1EDA, 0x1, 0x1EDB), + (0x1EDC, 0x1, 0x1EDD), + (0x1EDE, 0x1, 0x1EDF), + (0x1EE0, 0x1, 0x1EE1), + (0x1EE2, 0x1, 0x1EE3), + (0x1EE4, 0x1, 0x1EE5), + (0x1EE6, 0x1, 0x1EE7), + (0x1EE8, 0x1, 0x1EE9), + (0x1EEA, 0x1, 0x1EEB), + (0x1EEC, 0x1, 0x1EED), + (0x1EEE, 0x1, 0x1EEF), + (0x1EF0, 0x1, 0x1EF1), + (0x1EF2, 0x1, 0x1EF3), + (0x1EF4, 0x1, 0x1EF5), + (0x1EF6, 0x1, 0x1EF7), + (0x1EF8, 0x1, 0x1EF9), + (0x1EFA, 0x1, 0x1EFB), + (0x1EFC, 0x1, 0x1EFD), + (0x1EFE, 0x1, 0x1EFF), + (0x1F08, 0x1, 0x1F00), + (0x1F09, 0x1, 0x1F01), + (0x1F0A, 0x1, 0x1F02), + (0x1F0B, 0x1, 0x1F03), + (0x1F0C, 0x1, 0x1F04), + (0x1F0D, 0x1, 0x1F05), + (0x1F0E, 0x1, 0x1F06), + (0x1F0F, 0x1, 0x1F07), + (0x1F18, 0x1, 0x1F10), + (0x1F19, 0x1, 0x1F11), + (0x1F1A, 0x1, 0x1F12), + (0x1F1B, 0x1, 0x1F13), + (0x1F1C, 0x1, 0x1F14), + (0x1F1D, 0x1, 0x1F15), + (0x1F28, 0x1, 0x1F20), + (0x1F29, 0x1, 0x1F21), + (0x1F2A, 0x1, 0x1F22), + (0x1F2B, 0x1, 0x1F23), + (0x1F2C, 0x1, 0x1F24), + (0x1F2D, 0x1, 0x1F25), + (0x1F2E, 0x1, 0x1F26), + (0x1F2F, 0x1, 0x1F27), + (0x1F38, 0x1, 0x1F30), + (0x1F39, 0x1, 0x1F31), + (0x1F3A, 0x1, 0x1F32), + (0x1F3B, 0x1, 0x1F33), + (0x1F3C, 0x1, 0x1F34), + (0x1F3D, 0x1, 0x1F35), + (0x1F3E, 0x1, 0x1F36), + (0x1F3F, 0x1, 0x1F37), + (0x1F48, 0x1, 0x1F40), + (0x1F49, 0x1, 0x1F41), + (0x1F4A, 0x1, 0x1F42), + (0x1F4B, 0x1, 0x1F43), + (0x1F4C, 0x1, 0x1F44), + (0x1F4D, 0x1, 0x1F45), + (0x1F59, 0x1, 0x1F51), + (0x1F5B, 0x1, 0x1F53), + (0x1F5D, 0x1, 0x1F55), + (0x1F5F, 0x1, 0x1F57), + (0x1F68, 0x1, 0x1F60), + (0x1F69, 0x1, 0x1F61), + (0x1F6A, 0x1, 0x1F62), + (0x1F6B, 0x1, 0x1F63), + (0x1F6C, 0x1, 0x1F64), + (0x1F6D, 0x1, 0x1F65), + (0x1F6E, 0x1, 0x1F66), + (0x1F6F, 0x1, 0x1F67), + (0x1F88, 0x1, 0x1F80), + (0x1F89, 0x1, 0x1F81), + (0x1F8A, 0x1, 0x1F82), + (0x1F8B, 0x1, 0x1F83), + (0x1F8C, 0x1, 0x1F84), + (0x1F8D, 0x1, 0x1F85), + (0x1F8E, 0x1, 0x1F86), + (0x1F8F, 0x1, 0x1F87), + (0x1F98, 0x1, 0x1F90), + (0x1F99, 0x1, 0x1F91), + (0x1F9A, 0x1, 0x1F92), + (0x1F9B, 0x1, 0x1F93), + (0x1F9C, 0x1, 0x1F94), + (0x1F9D, 0x1, 0x1F95), + (0x1F9E, 0x1, 0x1F96), + (0x1F9F, 0x1, 0x1F97), + (0x1FA8, 0x1, 0x1FA0), + (0x1FA9, 0x1, 0x1FA1), + (0x1FAA, 0x1, 0x1FA2), + (0x1FAB, 0x1, 0x1FA3), + (0x1FAC, 0x1, 0x1FA4), + (0x1FAD, 0x1, 0x1FA5), + (0x1FAE, 0x1, 0x1FA6), + (0x1FAF, 0x1, 0x1FA7), + (0x1FB8, 0x1, 0x1FB0), + (0x1FB9, 0x1, 0x1FB1), + (0x1FBA, 0x1, 0x1F70), + (0x1FBB, 0x1, 0x1F71), + (0x1FBC, 0x1, 0x1FB3), + (0x1FC8, 0x1, 0x1F72), + (0x1FC9, 0x1, 0x1F73), + (0x1FCA, 0x1, 0x1F74), + (0x1FCB, 0x1, 0x1F75), + (0x1FCC, 0x1, 0x1FC3), + (0x1FD8, 0x1, 0x1FD0), + (0x1FD9, 0x1, 0x1FD1), + (0x1FDA, 0x1, 0x1F76), + (0x1FDB, 0x1, 0x1F77), + (0x1FE8, 0x1, 0x1FE0), + (0x1FE9, 0x1, 0x1FE1), + (0x1FEA, 0x1, 0x1F7A), + (0x1FEB, 0x1, 0x1F7B), + (0x1FEC, 0x1, 0x1FE5), + (0x1FF8, 0x1, 0x1F78), + (0x1FF9, 0x1, 0x1F79), + (0x1FFA, 0x1, 0x1F7C), + (0x1FFB, 0x1, 0x1F7D), + (0x1FFC, 0x1, 0x1FF3), + (0x2126, 0x1, 0x3C9), + (0x212A, 0x1, 0x6B), + (0x212B, 0x1, 0xE5), + (0x2132, 0x1, 0x214E), + (0x2160, 0x1, 0x2170), + (0x2161, 0x1, 0x2171), + (0x2162, 0x1, 0x2172), + (0x2163, 0x1, 0x2173), + (0x2164, 0x1, 0x2174), + (0x2165, 0x1, 0x2175), + (0x2166, 0x1, 0x2176), + (0x2167, 0x1, 0x2177), + (0x2168, 0x1, 0x2178), + (0x2169, 0x1, 0x2179), + (0x216A, 0x1, 0x217A), + (0x216B, 0x1, 0x217B), + (0x216C, 0x1, 0x217C), + (0x216D, 0x1, 0x217D), + (0x216E, 0x1, 0x217E), + (0x216F, 0x1, 0x217F), + (0x2183, 0x1, 0x2184), + (0x24B6, 0x1, 0x24D0), + (0x24B7, 0x1, 0x24D1), + (0x24B8, 0x1, 0x24D2), + (0x24B9, 0x1, 0x24D3), + (0x24BA, 0x1, 0x24D4), + (0x24BB, 0x1, 0x24D5), + (0x24BC, 0x1, 0x24D6), + (0x24BD, 0x1, 0x24D7), + (0x24BE, 0x1, 0x24D8), + (0x24BF, 0x1, 0x24D9), + (0x24C0, 0x1, 0x24DA), + (0x24C1, 0x1, 0x24DB), + (0x24C2, 0x1, 0x24DC), + (0x24C3, 0x1, 0x24DD), + (0x24C4, 0x1, 0x24DE), + (0x24C5, 0x1, 0x24DF), + (0x24C6, 0x1, 0x24E0), + (0x24C7, 0x1, 0x24E1), + (0x24C8, 0x1, 0x24E2), + (0x24C9, 0x1, 0x24E3), + (0x24CA, 0x1, 0x24E4), + (0x24CB, 0x1, 0x24E5), + (0x24CC, 0x1, 0x24E6), + (0x24CD, 0x1, 0x24E7), + (0x24CE, 0x1, 0x24E8), + (0x24CF, 0x1, 0x24E9), + (0x2C00, 0x1, 0x2C30), + (0x2C01, 0x1, 0x2C31), + (0x2C02, 0x1, 0x2C32), + (0x2C03, 0x1, 0x2C33), + (0x2C04, 0x1, 0x2C34), + (0x2C05, 0x1, 0x2C35), + (0x2C06, 0x1, 0x2C36), + (0x2C07, 0x1, 0x2C37), + (0x2C08, 0x1, 0x2C38), + (0x2C09, 0x1, 0x2C39), + (0x2C0A, 0x1, 0x2C3A), + (0x2C0B, 0x1, 0x2C3B), + (0x2C0C, 0x1, 0x2C3C), + (0x2C0D, 0x1, 0x2C3D), + (0x2C0E, 0x1, 0x2C3E), + (0x2C0F, 0x1, 0x2C3F), + (0x2C10, 0x1, 0x2C40), + (0x2C11, 0x1, 0x2C41), + (0x2C12, 0x1, 0x2C42), + (0x2C13, 0x1, 0x2C43), + (0x2C14, 0x1, 0x2C44), + (0x2C15, 0x1, 0x2C45), + (0x2C16, 0x1, 0x2C46), + (0x2C17, 0x1, 0x2C47), + (0x2C18, 0x1, 0x2C48), + (0x2C19, 0x1, 0x2C49), + (0x2C1A, 0x1, 0x2C4A), + (0x2C1B, 0x1, 0x2C4B), + (0x2C1C, 0x1, 0x2C4C), + (0x2C1D, 0x1, 0x2C4D), + (0x2C1E, 0x1, 0x2C4E), + (0x2C1F, 0x1, 0x2C4F), + (0x2C20, 0x1, 0x2C50), + (0x2C21, 0x1, 0x2C51), + (0x2C22, 0x1, 0x2C52), + (0x2C23, 0x1, 0x2C53), + (0x2C24, 0x1, 0x2C54), + (0x2C25, 0x1, 0x2C55), + (0x2C26, 0x1, 0x2C56), + (0x2C27, 0x1, 0x2C57), + (0x2C28, 0x1, 0x2C58), + (0x2C29, 0x1, 0x2C59), + (0x2C2A, 0x1, 0x2C5A), + (0x2C2B, 0x1, 0x2C5B), + (0x2C2C, 0x1, 0x2C5C), + (0x2C2D, 0x1, 0x2C5D), + (0x2C2E, 0x1, 0x2C5E), + (0x2C2F, 0x1, 0x2C5F), + (0x2C60, 0x1, 0x2C61), + (0x2C62, 0x1, 0x26B), + (0x2C63, 0x1, 0x1D7D), + (0x2C64, 0x1, 0x27D), + (0x2C67, 0x1, 0x2C68), + (0x2C69, 0x1, 0x2C6A), + (0x2C6B, 0x1, 0x2C6C), + (0x2C6D, 0x1, 0x251), + (0x2C6E, 0x1, 0x271), + (0x2C6F, 0x1, 0x250), + (0x2C70, 0x1, 0x252), + (0x2C72, 0x1, 0x2C73), + (0x2C75, 0x1, 0x2C76), + (0x2C7E, 0x1, 0x23F), + (0x2C7F, 0x1, 0x240), + (0x2C80, 0x1, 0x2C81), + (0x2C82, 0x1, 0x2C83), + (0x2C84, 0x1, 0x2C85), + (0x2C86, 0x1, 0x2C87), + (0x2C88, 0x1, 0x2C89), + (0x2C8A, 0x1, 0x2C8B), + (0x2C8C, 0x1, 0x2C8D), + (0x2C8E, 0x1, 0x2C8F), + (0x2C90, 0x1, 0x2C91), + (0x2C92, 0x1, 0x2C93), + (0x2C94, 0x1, 0x2C95), + (0x2C96, 0x1, 0x2C97), + (0x2C98, 0x1, 0x2C99), + (0x2C9A, 0x1, 0x2C9B), + (0x2C9C, 0x1, 0x2C9D), + (0x2C9E, 0x1, 0x2C9F), + (0x2CA0, 0x1, 0x2CA1), + (0x2CA2, 0x1, 0x2CA3), + (0x2CA4, 0x1, 0x2CA5), + (0x2CA6, 0x1, 0x2CA7), + (0x2CA8, 0x1, 0x2CA9), + (0x2CAA, 0x1, 0x2CAB), + (0x2CAC, 0x1, 0x2CAD), + (0x2CAE, 0x1, 0x2CAF), + (0x2CB0, 0x1, 0x2CB1), + (0x2CB2, 0x1, 0x2CB3), + (0x2CB4, 0x1, 0x2CB5), + (0x2CB6, 0x1, 0x2CB7), + (0x2CB8, 0x1, 0x2CB9), + (0x2CBA, 0x1, 0x2CBB), + (0x2CBC, 0x1, 0x2CBD), + (0x2CBE, 0x1, 0x2CBF), + (0x2CC0, 0x1, 0x2CC1), + (0x2CC2, 0x1, 0x2CC3), + (0x2CC4, 0x1, 0x2CC5), + (0x2CC6, 0x1, 0x2CC7), + (0x2CC8, 0x1, 0x2CC9), + (0x2CCA, 0x1, 0x2CCB), + (0x2CCC, 0x1, 0x2CCD), + (0x2CCE, 0x1, 0x2CCF), + (0x2CD0, 0x1, 0x2CD1), + (0x2CD2, 0x1, 0x2CD3), + (0x2CD4, 0x1, 0x2CD5), + (0x2CD6, 0x1, 0x2CD7), + (0x2CD8, 0x1, 0x2CD9), + (0x2CDA, 0x1, 0x2CDB), + (0x2CDC, 0x1, 0x2CDD), + (0x2CDE, 0x1, 0x2CDF), + (0x2CE0, 0x1, 0x2CE1), + (0x2CE2, 0x1, 0x2CE3), + (0x2CEB, 0x1, 0x2CEC), + (0x2CED, 0x1, 0x2CEE), + (0x2CF2, 0x1, 0x2CF3), + (0xA640, 0x1, 0xA641), + (0xA642, 0x1, 0xA643), + (0xA644, 0x1, 0xA645), + (0xA646, 0x1, 0xA647), + (0xA648, 0x1, 0xA649), + (0xA64A, 0x1, 0xA64B), + (0xA64C, 0x1, 0xA64D), + (0xA64E, 0x1, 0xA64F), + (0xA650, 0x1, 0xA651), + (0xA652, 0x1, 0xA653), + (0xA654, 0x1, 0xA655), + (0xA656, 0x1, 0xA657), + (0xA658, 0x1, 0xA659), + (0xA65A, 0x1, 0xA65B), + (0xA65C, 0x1, 0xA65D), + (0xA65E, 0x1, 0xA65F), + (0xA660, 0x1, 0xA661), + (0xA662, 0x1, 0xA663), + (0xA664, 0x1, 0xA665), + (0xA666, 0x1, 0xA667), + (0xA668, 0x1, 0xA669), + (0xA66A, 0x1, 0xA66B), + (0xA66C, 0x1, 0xA66D), + (0xA680, 0x1, 0xA681), + (0xA682, 0x1, 0xA683), + (0xA684, 0x1, 0xA685), + (0xA686, 0x1, 0xA687), + (0xA688, 0x1, 0xA689), + (0xA68A, 0x1, 0xA68B), + (0xA68C, 0x1, 0xA68D), + (0xA68E, 0x1, 0xA68F), + (0xA690, 0x1, 0xA691), + (0xA692, 0x1, 0xA693), + (0xA694, 0x1, 0xA695), + (0xA696, 0x1, 0xA697), + (0xA698, 0x1, 0xA699), + (0xA69A, 0x1, 0xA69B), + (0xA722, 0x1, 0xA723), + (0xA724, 0x1, 0xA725), + (0xA726, 0x1, 0xA727), + (0xA728, 0x1, 0xA729), + (0xA72A, 0x1, 0xA72B), + (0xA72C, 0x1, 0xA72D), + (0xA72E, 0x1, 0xA72F), + (0xA732, 0x1, 0xA733), + (0xA734, 0x1, 0xA735), + (0xA736, 0x1, 0xA737), + (0xA738, 0x1, 0xA739), + (0xA73A, 0x1, 0xA73B), + (0xA73C, 0x1, 0xA73D), + (0xA73E, 0x1, 0xA73F), + (0xA740, 0x1, 0xA741), + (0xA742, 0x1, 0xA743), + (0xA744, 0x1, 0xA745), + (0xA746, 0x1, 0xA747), + (0xA748, 0x1, 0xA749), + (0xA74A, 0x1, 0xA74B), + (0xA74C, 0x1, 0xA74D), + (0xA74E, 0x1, 0xA74F), + (0xA750, 0x1, 0xA751), + (0xA752, 0x1, 0xA753), + (0xA754, 0x1, 0xA755), + (0xA756, 0x1, 0xA757), + (0xA758, 0x1, 0xA759), + (0xA75A, 0x1, 0xA75B), + (0xA75C, 0x1, 0xA75D), + (0xA75E, 0x1, 0xA75F), + (0xA760, 0x1, 0xA761), + (0xA762, 0x1, 0xA763), + (0xA764, 0x1, 0xA765), + (0xA766, 0x1, 0xA767), + (0xA768, 0x1, 0xA769), + (0xA76A, 0x1, 0xA76B), + (0xA76C, 0x1, 0xA76D), + (0xA76E, 0x1, 0xA76F), + (0xA779, 0x1, 0xA77A), + (0xA77B, 0x1, 0xA77C), + (0xA77D, 0x1, 0x1D79), + (0xA77E, 0x1, 0xA77F), + (0xA780, 0x1, 0xA781), + (0xA782, 0x1, 0xA783), + (0xA784, 0x1, 0xA785), + (0xA786, 0x1, 0xA787), + (0xA78B, 0x1, 0xA78C), + (0xA78D, 0x1, 0x265), + (0xA790, 0x1, 0xA791), + (0xA792, 0x1, 0xA793), + (0xA796, 0x1, 0xA797), + (0xA798, 0x1, 0xA799), + (0xA79A, 0x1, 0xA79B), + (0xA79C, 0x1, 0xA79D), + (0xA79E, 0x1, 0xA79F), + (0xA7A0, 0x1, 0xA7A1), + (0xA7A2, 0x1, 0xA7A3), + (0xA7A4, 0x1, 0xA7A5), + (0xA7A6, 0x1, 0xA7A7), + (0xA7A8, 0x1, 0xA7A9), + (0xA7AA, 0x1, 0x266), + (0xA7AB, 0x1, 0x25C), + (0xA7AC, 0x1, 0x261), + (0xA7AD, 0x1, 0x26C), + (0xA7AE, 0x1, 0x26A), + (0xA7B0, 0x1, 0x29E), + (0xA7B1, 0x1, 0x287), + (0xA7B2, 0x1, 0x29D), + (0xA7B3, 0x1, 0xAB53), + (0xA7B4, 0x1, 0xA7B5), + (0xA7B6, 0x1, 0xA7B7), + (0xA7B8, 0x1, 0xA7B9), + (0xA7BA, 0x1, 0xA7BB), + (0xA7BC, 0x1, 0xA7BD), + (0xA7BE, 0x1, 0xA7BF), + (0xA7C0, 0x1, 0xA7C1), + (0xA7C2, 0x1, 0xA7C3), + (0xA7C4, 0x1, 0xA794), + (0xA7C5, 0x1, 0x282), + (0xA7C6, 0x1, 0x1D8E), + (0xA7C7, 0x1, 0xA7C8), + (0xA7C9, 0x1, 0xA7CA), + (0xA7D0, 0x1, 0xA7D1), + (0xA7D6, 0x1, 0xA7D7), + (0xA7D8, 0x1, 0xA7D9), + (0xA7F5, 0x1, 0xA7F6), + (0xFF21, 0x1, 0xFF41), + (0xFF22, 0x1, 0xFF42), + (0xFF23, 0x1, 0xFF43), + (0xFF24, 0x1, 0xFF44), + (0xFF25, 0x1, 0xFF45), + (0xFF26, 0x1, 0xFF46), + (0xFF27, 0x1, 0xFF47), + (0xFF28, 0x1, 0xFF48), + (0xFF29, 0x1, 0xFF49), + (0xFF2A, 0x1, 0xFF4A), + (0xFF2B, 0x1, 0xFF4B), + (0xFF2C, 0x1, 0xFF4C), + (0xFF2D, 0x1, 0xFF4D), + (0xFF2E, 0x1, 0xFF4E), + (0xFF2F, 0x1, 0xFF4F), + (0xFF30, 0x1, 0xFF50), + (0xFF31, 0x1, 0xFF51), + (0xFF32, 0x1, 0xFF52), + (0xFF33, 0x1, 0xFF53), + (0xFF34, 0x1, 0xFF54), + (0xFF35, 0x1, 0xFF55), + (0xFF36, 0x1, 0xFF56), + (0xFF37, 0x1, 0xFF57), + (0xFF38, 0x1, 0xFF58), + (0xFF39, 0x1, 0xFF59), + (0xFF3A, 0x1, 0xFF5A), + (0x10400, 0x1, 0x10428), + (0x10401, 0x1, 0x10429), + (0x10402, 0x1, 0x1042A), + (0x10403, 0x1, 0x1042B), + (0x10404, 0x1, 0x1042C), + (0x10405, 0x1, 0x1042D), + (0x10406, 0x1, 0x1042E), + (0x10407, 0x1, 0x1042F), + (0x10408, 0x1, 0x10430), + (0x10409, 0x1, 0x10431), + (0x1040A, 0x1, 0x10432), + (0x1040B, 0x1, 0x10433), + (0x1040C, 0x1, 0x10434), + (0x1040D, 0x1, 0x10435), + (0x1040E, 0x1, 0x10436), + (0x1040F, 0x1, 0x10437), + (0x10410, 0x1, 0x10438), + (0x10411, 0x1, 0x10439), + (0x10412, 0x1, 0x1043A), + (0x10413, 0x1, 0x1043B), + (0x10414, 0x1, 0x1043C), + (0x10415, 0x1, 0x1043D), + (0x10416, 0x1, 0x1043E), + (0x10417, 0x1, 0x1043F), + (0x10418, 0x1, 0x10440), + (0x10419, 0x1, 0x10441), + (0x1041A, 0x1, 0x10442), + (0x1041B, 0x1, 0x10443), + (0x1041C, 0x1, 0x10444), + (0x1041D, 0x1, 0x10445), + (0x1041E, 0x1, 0x10446), + (0x1041F, 0x1, 0x10447), + (0x10420, 0x1, 0x10448), + (0x10421, 0x1, 0x10449), + (0x10422, 0x1, 0x1044A), + (0x10423, 0x1, 0x1044B), + (0x10424, 0x1, 0x1044C), + (0x10425, 0x1, 0x1044D), + (0x10426, 0x1, 0x1044E), + (0x10427, 0x1, 0x1044F), + (0x104B0, 0x1, 0x104D8), + (0x104B1, 0x1, 0x104D9), + (0x104B2, 0x1, 0x104DA), + (0x104B3, 0x1, 0x104DB), + (0x104B4, 0x1, 0x104DC), + (0x104B5, 0x1, 0x104DD), + (0x104B6, 0x1, 0x104DE), + (0x104B7, 0x1, 0x104DF), + (0x104B8, 0x1, 0x104E0), + (0x104B9, 0x1, 0x104E1), + (0x104BA, 0x1, 0x104E2), + (0x104BB, 0x1, 0x104E3), + (0x104BC, 0x1, 0x104E4), + (0x104BD, 0x1, 0x104E5), + (0x104BE, 0x1, 0x104E6), + (0x104BF, 0x1, 0x104E7), + (0x104C0, 0x1, 0x104E8), + (0x104C1, 0x1, 0x104E9), + (0x104C2, 0x1, 0x104EA), + (0x104C3, 0x1, 0x104EB), + (0x104C4, 0x1, 0x104EC), + (0x104C5, 0x1, 0x104ED), + (0x104C6, 0x1, 0x104EE), + (0x104C7, 0x1, 0x104EF), + (0x104C8, 0x1, 0x104F0), + (0x104C9, 0x1, 0x104F1), + (0x104CA, 0x1, 0x104F2), + (0x104CB, 0x1, 0x104F3), + (0x104CC, 0x1, 0x104F4), + (0x104CD, 0x1, 0x104F5), + (0x104CE, 0x1, 0x104F6), + (0x104CF, 0x1, 0x104F7), + (0x104D0, 0x1, 0x104F8), + (0x104D1, 0x1, 0x104F9), + (0x104D2, 0x1, 0x104FA), + (0x104D3, 0x1, 0x104FB), + (0x10570, 0x1, 0x10597), + (0x10571, 0x1, 0x10598), + (0x10572, 0x1, 0x10599), + (0x10573, 0x1, 0x1059A), + (0x10574, 0x1, 0x1059B), + (0x10575, 0x1, 0x1059C), + (0x10576, 0x1, 0x1059D), + (0x10577, 0x1, 0x1059E), + (0x10578, 0x1, 0x1059F), + (0x10579, 0x1, 0x105A0), + (0x1057A, 0x1, 0x105A1), + (0x1057C, 0x1, 0x105A3), + (0x1057D, 0x1, 0x105A4), + (0x1057E, 0x1, 0x105A5), + (0x1057F, 0x1, 0x105A6), + (0x10580, 0x1, 0x105A7), + (0x10581, 0x1, 0x105A8), + (0x10582, 0x1, 0x105A9), + (0x10583, 0x1, 0x105AA), + (0x10584, 0x1, 0x105AB), + (0x10585, 0x1, 0x105AC), + (0x10586, 0x1, 0x105AD), + (0x10587, 0x1, 0x105AE), + (0x10588, 0x1, 0x105AF), + (0x10589, 0x1, 0x105B0), + (0x1058A, 0x1, 0x105B1), + (0x1058C, 0x1, 0x105B3), + (0x1058D, 0x1, 0x105B4), + (0x1058E, 0x1, 0x105B5), + (0x1058F, 0x1, 0x105B6), + (0x10590, 0x1, 0x105B7), + (0x10591, 0x1, 0x105B8), + (0x10592, 0x1, 0x105B9), + (0x10594, 0x1, 0x105BB), + (0x10595, 0x1, 0x105BC), + (0x10C80, 0x1, 0x10CC0), + (0x10C81, 0x1, 0x10CC1), + (0x10C82, 0x1, 0x10CC2), + (0x10C83, 0x1, 0x10CC3), + (0x10C84, 0x1, 0x10CC4), + (0x10C85, 0x1, 0x10CC5), + (0x10C86, 0x1, 0x10CC6), + (0x10C87, 0x1, 0x10CC7), + (0x10C88, 0x1, 0x10CC8), + (0x10C89, 0x1, 0x10CC9), + (0x10C8A, 0x1, 0x10CCA), + (0x10C8B, 0x1, 0x10CCB), + (0x10C8C, 0x1, 0x10CCC), + (0x10C8D, 0x1, 0x10CCD), + (0x10C8E, 0x1, 0x10CCE), + (0x10C8F, 0x1, 0x10CCF), + (0x10C90, 0x1, 0x10CD0), + (0x10C91, 0x1, 0x10CD1), + (0x10C92, 0x1, 0x10CD2), + (0x10C93, 0x1, 0x10CD3), + (0x10C94, 0x1, 0x10CD4), + (0x10C95, 0x1, 0x10CD5), + (0x10C96, 0x1, 0x10CD6), + (0x10C97, 0x1, 0x10CD7), + (0x10C98, 0x1, 0x10CD8), + (0x10C99, 0x1, 0x10CD9), + (0x10C9A, 0x1, 0x10CDA), + (0x10C9B, 0x1, 0x10CDB), + (0x10C9C, 0x1, 0x10CDC), + (0x10C9D, 0x1, 0x10CDD), + (0x10C9E, 0x1, 0x10CDE), + (0x10C9F, 0x1, 0x10CDF), + (0x10CA0, 0x1, 0x10CE0), + (0x10CA1, 0x1, 0x10CE1), + (0x10CA2, 0x1, 0x10CE2), + (0x10CA3, 0x1, 0x10CE3), + (0x10CA4, 0x1, 0x10CE4), + (0x10CA5, 0x1, 0x10CE5), + (0x10CA6, 0x1, 0x10CE6), + (0x10CA7, 0x1, 0x10CE7), + (0x10CA8, 0x1, 0x10CE8), + (0x10CA9, 0x1, 0x10CE9), + (0x10CAA, 0x1, 0x10CEA), + (0x10CAB, 0x1, 0x10CEB), + (0x10CAC, 0x1, 0x10CEC), + (0x10CAD, 0x1, 0x10CED), + (0x10CAE, 0x1, 0x10CEE), + (0x10CAF, 0x1, 0x10CEF), + (0x10CB0, 0x1, 0x10CF0), + (0x10CB1, 0x1, 0x10CF1), + (0x10CB2, 0x1, 0x10CF2), + (0x118A0, 0x1, 0x118C0), + (0x118A1, 0x1, 0x118C1), + (0x118A2, 0x1, 0x118C2), + (0x118A3, 0x1, 0x118C3), + (0x118A4, 0x1, 0x118C4), + (0x118A5, 0x1, 0x118C5), + (0x118A6, 0x1, 0x118C6), + (0x118A7, 0x1, 0x118C7), + (0x118A8, 0x1, 0x118C8), + (0x118A9, 0x1, 0x118C9), + (0x118AA, 0x1, 0x118CA), + (0x118AB, 0x1, 0x118CB), + (0x118AC, 0x1, 0x118CC), + (0x118AD, 0x1, 0x118CD), + (0x118AE, 0x1, 0x118CE), + (0x118AF, 0x1, 0x118CF), + (0x118B0, 0x1, 0x118D0), + (0x118B1, 0x1, 0x118D1), + (0x118B2, 0x1, 0x118D2), + (0x118B3, 0x1, 0x118D3), + (0x118B4, 0x1, 0x118D4), + (0x118B5, 0x1, 0x118D5), + (0x118B6, 0x1, 0x118D6), + (0x118B7, 0x1, 0x118D7), + (0x118B8, 0x1, 0x118D8), + (0x118B9, 0x1, 0x118D9), + (0x118BA, 0x1, 0x118DA), + (0x118BB, 0x1, 0x118DB), + (0x118BC, 0x1, 0x118DC), + (0x118BD, 0x1, 0x118DD), + (0x118BE, 0x1, 0x118DE), + (0x118BF, 0x1, 0x118DF), + (0x16E40, 0x1, 0x16E60), + (0x16E41, 0x1, 0x16E61), + (0x16E42, 0x1, 0x16E62), + (0x16E43, 0x1, 0x16E63), + (0x16E44, 0x1, 0x16E64), + (0x16E45, 0x1, 0x16E65), + (0x16E46, 0x1, 0x16E66), + (0x16E47, 0x1, 0x16E67), + (0x16E48, 0x1, 0x16E68), + (0x16E49, 0x1, 0x16E69), + (0x16E4A, 0x1, 0x16E6A), + (0x16E4B, 0x1, 0x16E6B), + (0x16E4C, 0x1, 0x16E6C), + (0x16E4D, 0x1, 0x16E6D), + (0x16E4E, 0x1, 0x16E6E), + (0x16E4F, 0x1, 0x16E6F), + (0x16E50, 0x1, 0x16E70), + (0x16E51, 0x1, 0x16E71), + (0x16E52, 0x1, 0x16E72), + (0x16E53, 0x1, 0x16E73), + (0x16E54, 0x1, 0x16E74), + (0x16E55, 0x1, 0x16E75), + (0x16E56, 0x1, 0x16E76), + (0x16E57, 0x1, 0x16E77), + (0x16E58, 0x1, 0x16E78), + (0x16E59, 0x1, 0x16E79), + (0x16E5A, 0x1, 0x16E7A), + (0x16E5B, 0x1, 0x16E7B), + (0x16E5C, 0x1, 0x16E7C), + (0x16E5D, 0x1, 0x16E7D), + (0x16E5E, 0x1, 0x16E7E), + (0x16E5F, 0x1, 0x16E7F), + (0x1E900, 0x1, 0x1E922), + (0x1E901, 0x1, 0x1E923), + (0x1E902, 0x1, 0x1E924), + (0x1E903, 0x1, 0x1E925), + (0x1E904, 0x1, 0x1E926), + (0x1E905, 0x1, 0x1E927), + (0x1E906, 0x1, 0x1E928), + (0x1E907, 0x1, 0x1E929), + (0x1E908, 0x1, 0x1E92A), + (0x1E909, 0x1, 0x1E92B), + (0x1E90A, 0x1, 0x1E92C), + (0x1E90B, 0x1, 0x1E92D), + (0x1E90C, 0x1, 0x1E92E), + (0x1E90D, 0x1, 0x1E92F), + (0x1E90E, 0x1, 0x1E930), + (0x1E90F, 0x1, 0x1E931), + (0x1E910, 0x1, 0x1E932), + (0x1E911, 0x1, 0x1E933), + (0x1E912, 0x1, 0x1E934), + (0x1E913, 0x1, 0x1E935), + (0x1E914, 0x1, 0x1E936), + (0x1E915, 0x1, 0x1E937), + (0x1E916, 0x1, 0x1E938), + (0x1E917, 0x1, 0x1E939), + (0x1E918, 0x1, 0x1E93A), + (0x1E919, 0x1, 0x1E93B), + (0x1E91A, 0x1, 0x1E93C), + (0x1E91B, 0x1, 0x1E93D), + (0x1E91C, 0x1, 0x1E93E), + (0x1E91D, 0x1, 0x1E93F), + (0x1E91E, 0x1, 0x1E940), + (0x1E91F, 0x1, 0x1E941), + (0x1E920, 0x1, 0x1E942), + (0x1E921, 0x1, 0x1E943), +) + +UPPER_ROWS = ( + (0x61, 0x1, 0x41), + (0x62, 0x1, 0x42), + (0x63, 0x1, 0x43), + (0x64, 0x1, 0x44), + (0x65, 0x1, 0x45), + (0x66, 0x1, 0x46), + (0x67, 0x1, 0x47), + (0x68, 0x1, 0x48), + (0x69, 0x1, 0x49), + (0x6A, 0x1, 0x4A), + (0x6B, 0x1, 0x4B), + (0x6C, 0x1, 0x4C), + (0x6D, 0x1, 0x4D), + (0x6E, 0x1, 0x4E), + (0x6F, 0x1, 0x4F), + (0x70, 0x1, 0x50), + (0x71, 0x1, 0x51), + (0x72, 0x1, 0x52), + (0x73, 0x1, 0x53), + (0x74, 0x1, 0x54), + (0x75, 0x1, 0x55), + (0x76, 0x1, 0x56), + (0x77, 0x1, 0x57), + (0x78, 0x1, 0x58), + (0x79, 0x1, 0x59), + (0x7A, 0x1, 0x5A), + (0xB5, 0x1, 0x39C), + (0xDF, 0x2, 0x53, 0x53), + (0xE0, 0x1, 0xC0), + (0xE1, 0x1, 0xC1), + (0xE2, 0x1, 0xC2), + (0xE3, 0x1, 0xC3), + (0xE4, 0x1, 0xC4), + (0xE5, 0x1, 0xC5), + (0xE6, 0x1, 0xC6), + (0xE7, 0x1, 0xC7), + (0xE8, 0x1, 0xC8), + (0xE9, 0x1, 0xC9), + (0xEA, 0x1, 0xCA), + (0xEB, 0x1, 0xCB), + (0xEC, 0x1, 0xCC), + (0xED, 0x1, 0xCD), + (0xEE, 0x1, 0xCE), + (0xEF, 0x1, 0xCF), + (0xF0, 0x1, 0xD0), + (0xF1, 0x1, 0xD1), + (0xF2, 0x1, 0xD2), + (0xF3, 0x1, 0xD3), + (0xF4, 0x1, 0xD4), + (0xF5, 0x1, 0xD5), + (0xF6, 0x1, 0xD6), + (0xF8, 0x1, 0xD8), + (0xF9, 0x1, 0xD9), + (0xFA, 0x1, 0xDA), + (0xFB, 0x1, 0xDB), + (0xFC, 0x1, 0xDC), + (0xFD, 0x1, 0xDD), + (0xFE, 0x1, 0xDE), + (0xFF, 0x1, 0x178), + (0x101, 0x1, 0x100), + (0x103, 0x1, 0x102), + (0x105, 0x1, 0x104), + (0x107, 0x1, 0x106), + (0x109, 0x1, 0x108), + (0x10B, 0x1, 0x10A), + (0x10D, 0x1, 0x10C), + (0x10F, 0x1, 0x10E), + (0x111, 0x1, 0x110), + (0x113, 0x1, 0x112), + (0x115, 0x1, 0x114), + (0x117, 0x1, 0x116), + (0x119, 0x1, 0x118), + (0x11B, 0x1, 0x11A), + (0x11D, 0x1, 0x11C), + (0x11F, 0x1, 0x11E), + (0x121, 0x1, 0x120), + (0x123, 0x1, 0x122), + (0x125, 0x1, 0x124), + (0x127, 0x1, 0x126), + (0x129, 0x1, 0x128), + (0x12B, 0x1, 0x12A), + (0x12D, 0x1, 0x12C), + (0x12F, 0x1, 0x12E), + (0x131, 0x1, 0x49), + (0x133, 0x1, 0x132), + (0x135, 0x1, 0x134), + (0x137, 0x1, 0x136), + (0x13A, 0x1, 0x139), + (0x13C, 0x1, 0x13B), + (0x13E, 0x1, 0x13D), + (0x140, 0x1, 0x13F), + (0x142, 0x1, 0x141), + (0x144, 0x1, 0x143), + (0x146, 0x1, 0x145), + (0x148, 0x1, 0x147), + (0x149, 0x2, 0x2BC, 0x4E), + (0x14B, 0x1, 0x14A), + (0x14D, 0x1, 0x14C), + (0x14F, 0x1, 0x14E), + (0x151, 0x1, 0x150), + (0x153, 0x1, 0x152), + (0x155, 0x1, 0x154), + (0x157, 0x1, 0x156), + (0x159, 0x1, 0x158), + (0x15B, 0x1, 0x15A), + (0x15D, 0x1, 0x15C), + (0x15F, 0x1, 0x15E), + (0x161, 0x1, 0x160), + (0x163, 0x1, 0x162), + (0x165, 0x1, 0x164), + (0x167, 0x1, 0x166), + (0x169, 0x1, 0x168), + (0x16B, 0x1, 0x16A), + (0x16D, 0x1, 0x16C), + (0x16F, 0x1, 0x16E), + (0x171, 0x1, 0x170), + (0x173, 0x1, 0x172), + (0x175, 0x1, 0x174), + (0x177, 0x1, 0x176), + (0x17A, 0x1, 0x179), + (0x17C, 0x1, 0x17B), + (0x17E, 0x1, 0x17D), + (0x17F, 0x1, 0x53), + (0x180, 0x1, 0x243), + (0x183, 0x1, 0x182), + (0x185, 0x1, 0x184), + (0x188, 0x1, 0x187), + (0x18C, 0x1, 0x18B), + (0x192, 0x1, 0x191), + (0x195, 0x1, 0x1F6), + (0x199, 0x1, 0x198), + (0x19A, 0x1, 0x23D), + (0x19E, 0x1, 0x220), + (0x1A1, 0x1, 0x1A0), + (0x1A3, 0x1, 0x1A2), + (0x1A5, 0x1, 0x1A4), + (0x1A8, 0x1, 0x1A7), + (0x1AD, 0x1, 0x1AC), + (0x1B0, 0x1, 0x1AF), + (0x1B4, 0x1, 0x1B3), + (0x1B6, 0x1, 0x1B5), + (0x1B9, 0x1, 0x1B8), + (0x1BD, 0x1, 0x1BC), + (0x1BF, 0x1, 0x1F7), + (0x1C5, 0x1, 0x1C4), + (0x1C6, 0x1, 0x1C4), + (0x1C8, 0x1, 0x1C7), + (0x1C9, 0x1, 0x1C7), + (0x1CB, 0x1, 0x1CA), + (0x1CC, 0x1, 0x1CA), + (0x1CE, 0x1, 0x1CD), + (0x1D0, 0x1, 0x1CF), + (0x1D2, 0x1, 0x1D1), + (0x1D4, 0x1, 0x1D3), + (0x1D6, 0x1, 0x1D5), + (0x1D8, 0x1, 0x1D7), + (0x1DA, 0x1, 0x1D9), + (0x1DC, 0x1, 0x1DB), + (0x1DD, 0x1, 0x18E), + (0x1DF, 0x1, 0x1DE), + (0x1E1, 0x1, 0x1E0), + (0x1E3, 0x1, 0x1E2), + (0x1E5, 0x1, 0x1E4), + (0x1E7, 0x1, 0x1E6), + (0x1E9, 0x1, 0x1E8), + (0x1EB, 0x1, 0x1EA), + (0x1ED, 0x1, 0x1EC), + (0x1EF, 0x1, 0x1EE), + (0x1F0, 0x2, 0x4A, 0x30C), + (0x1F2, 0x1, 0x1F1), + (0x1F3, 0x1, 0x1F1), + (0x1F5, 0x1, 0x1F4), + (0x1F9, 0x1, 0x1F8), + (0x1FB, 0x1, 0x1FA), + (0x1FD, 0x1, 0x1FC), + (0x1FF, 0x1, 0x1FE), + (0x201, 0x1, 0x200), + (0x203, 0x1, 0x202), + (0x205, 0x1, 0x204), + (0x207, 0x1, 0x206), + (0x209, 0x1, 0x208), + (0x20B, 0x1, 0x20A), + (0x20D, 0x1, 0x20C), + (0x20F, 0x1, 0x20E), + (0x211, 0x1, 0x210), + (0x213, 0x1, 0x212), + (0x215, 0x1, 0x214), + (0x217, 0x1, 0x216), + (0x219, 0x1, 0x218), + (0x21B, 0x1, 0x21A), + (0x21D, 0x1, 0x21C), + (0x21F, 0x1, 0x21E), + (0x223, 0x1, 0x222), + (0x225, 0x1, 0x224), + (0x227, 0x1, 0x226), + (0x229, 0x1, 0x228), + (0x22B, 0x1, 0x22A), + (0x22D, 0x1, 0x22C), + (0x22F, 0x1, 0x22E), + (0x231, 0x1, 0x230), + (0x233, 0x1, 0x232), + (0x23C, 0x1, 0x23B), + (0x23F, 0x1, 0x2C7E), + (0x240, 0x1, 0x2C7F), + (0x242, 0x1, 0x241), + (0x247, 0x1, 0x246), + (0x249, 0x1, 0x248), + (0x24B, 0x1, 0x24A), + (0x24D, 0x1, 0x24C), + (0x24F, 0x1, 0x24E), + (0x250, 0x1, 0x2C6F), + (0x251, 0x1, 0x2C6D), + (0x252, 0x1, 0x2C70), + (0x253, 0x1, 0x181), + (0x254, 0x1, 0x186), + (0x256, 0x1, 0x189), + (0x257, 0x1, 0x18A), + (0x259, 0x1, 0x18F), + (0x25B, 0x1, 0x190), + (0x25C, 0x1, 0xA7AB), + (0x260, 0x1, 0x193), + (0x261, 0x1, 0xA7AC), + (0x263, 0x1, 0x194), + (0x265, 0x1, 0xA78D), + (0x266, 0x1, 0xA7AA), + (0x268, 0x1, 0x197), + (0x269, 0x1, 0x196), + (0x26A, 0x1, 0xA7AE), + (0x26B, 0x1, 0x2C62), + (0x26C, 0x1, 0xA7AD), + (0x26F, 0x1, 0x19C), + (0x271, 0x1, 0x2C6E), + (0x272, 0x1, 0x19D), + (0x275, 0x1, 0x19F), + (0x27D, 0x1, 0x2C64), + (0x280, 0x1, 0x1A6), + (0x282, 0x1, 0xA7C5), + (0x283, 0x1, 0x1A9), + (0x287, 0x1, 0xA7B1), + (0x288, 0x1, 0x1AE), + (0x289, 0x1, 0x244), + (0x28A, 0x1, 0x1B1), + (0x28B, 0x1, 0x1B2), + (0x28C, 0x1, 0x245), + (0x292, 0x1, 0x1B7), + (0x29D, 0x1, 0xA7B2), + (0x29E, 0x1, 0xA7B0), + (0x345, 0x1, 0x399), + (0x371, 0x1, 0x370), + (0x373, 0x1, 0x372), + (0x377, 0x1, 0x376), + (0x37B, 0x1, 0x3FD), + (0x37C, 0x1, 0x3FE), + (0x37D, 0x1, 0x3FF), + (0x390, 0x3, 0x399, 0x308, 0x301), + (0x3AC, 0x1, 0x386), + (0x3AD, 0x1, 0x388), + (0x3AE, 0x1, 0x389), + (0x3AF, 0x1, 0x38A), + (0x3B0, 0x3, 0x3A5, 0x308, 0x301), + (0x3B1, 0x1, 0x391), + (0x3B2, 0x1, 0x392), + (0x3B3, 0x1, 0x393), + (0x3B4, 0x1, 0x394), + (0x3B5, 0x1, 0x395), + (0x3B6, 0x1, 0x396), + (0x3B7, 0x1, 0x397), + (0x3B8, 0x1, 0x398), + (0x3B9, 0x1, 0x399), + (0x3BA, 0x1, 0x39A), + (0x3BB, 0x1, 0x39B), + (0x3BC, 0x1, 0x39C), + (0x3BD, 0x1, 0x39D), + (0x3BE, 0x1, 0x39E), + (0x3BF, 0x1, 0x39F), + (0x3C0, 0x1, 0x3A0), + (0x3C1, 0x1, 0x3A1), + (0x3C2, 0x1, 0x3A3), + (0x3C3, 0x1, 0x3A3), + (0x3C4, 0x1, 0x3A4), + (0x3C5, 0x1, 0x3A5), + (0x3C6, 0x1, 0x3A6), + (0x3C7, 0x1, 0x3A7), + (0x3C8, 0x1, 0x3A8), + (0x3C9, 0x1, 0x3A9), + (0x3CA, 0x1, 0x3AA), + (0x3CB, 0x1, 0x3AB), + (0x3CC, 0x1, 0x38C), + (0x3CD, 0x1, 0x38E), + (0x3CE, 0x1, 0x38F), + (0x3D0, 0x1, 0x392), + (0x3D1, 0x1, 0x398), + (0x3D5, 0x1, 0x3A6), + (0x3D6, 0x1, 0x3A0), + (0x3D7, 0x1, 0x3CF), + (0x3D9, 0x1, 0x3D8), + (0x3DB, 0x1, 0x3DA), + (0x3DD, 0x1, 0x3DC), + (0x3DF, 0x1, 0x3DE), + (0x3E1, 0x1, 0x3E0), + (0x3E3, 0x1, 0x3E2), + (0x3E5, 0x1, 0x3E4), + (0x3E7, 0x1, 0x3E6), + (0x3E9, 0x1, 0x3E8), + (0x3EB, 0x1, 0x3EA), + (0x3ED, 0x1, 0x3EC), + (0x3EF, 0x1, 0x3EE), + (0x3F0, 0x1, 0x39A), + (0x3F1, 0x1, 0x3A1), + (0x3F2, 0x1, 0x3F9), + (0x3F3, 0x1, 0x37F), + (0x3F5, 0x1, 0x395), + (0x3F8, 0x1, 0x3F7), + (0x3FB, 0x1, 0x3FA), + (0x430, 0x1, 0x410), + (0x431, 0x1, 0x411), + (0x432, 0x1, 0x412), + (0x433, 0x1, 0x413), + (0x434, 0x1, 0x414), + (0x435, 0x1, 0x415), + (0x436, 0x1, 0x416), + (0x437, 0x1, 0x417), + (0x438, 0x1, 0x418), + (0x439, 0x1, 0x419), + (0x43A, 0x1, 0x41A), + (0x43B, 0x1, 0x41B), + (0x43C, 0x1, 0x41C), + (0x43D, 0x1, 0x41D), + (0x43E, 0x1, 0x41E), + (0x43F, 0x1, 0x41F), + (0x440, 0x1, 0x420), + (0x441, 0x1, 0x421), + (0x442, 0x1, 0x422), + (0x443, 0x1, 0x423), + (0x444, 0x1, 0x424), + (0x445, 0x1, 0x425), + (0x446, 0x1, 0x426), + (0x447, 0x1, 0x427), + (0x448, 0x1, 0x428), + (0x449, 0x1, 0x429), + (0x44A, 0x1, 0x42A), + (0x44B, 0x1, 0x42B), + (0x44C, 0x1, 0x42C), + (0x44D, 0x1, 0x42D), + (0x44E, 0x1, 0x42E), + (0x44F, 0x1, 0x42F), + (0x450, 0x1, 0x400), + (0x451, 0x1, 0x401), + (0x452, 0x1, 0x402), + (0x453, 0x1, 0x403), + (0x454, 0x1, 0x404), + (0x455, 0x1, 0x405), + (0x456, 0x1, 0x406), + (0x457, 0x1, 0x407), + (0x458, 0x1, 0x408), + (0x459, 0x1, 0x409), + (0x45A, 0x1, 0x40A), + (0x45B, 0x1, 0x40B), + (0x45C, 0x1, 0x40C), + (0x45D, 0x1, 0x40D), + (0x45E, 0x1, 0x40E), + (0x45F, 0x1, 0x40F), + (0x461, 0x1, 0x460), + (0x463, 0x1, 0x462), + (0x465, 0x1, 0x464), + (0x467, 0x1, 0x466), + (0x469, 0x1, 0x468), + (0x46B, 0x1, 0x46A), + (0x46D, 0x1, 0x46C), + (0x46F, 0x1, 0x46E), + (0x471, 0x1, 0x470), + (0x473, 0x1, 0x472), + (0x475, 0x1, 0x474), + (0x477, 0x1, 0x476), + (0x479, 0x1, 0x478), + (0x47B, 0x1, 0x47A), + (0x47D, 0x1, 0x47C), + (0x47F, 0x1, 0x47E), + (0x481, 0x1, 0x480), + (0x48B, 0x1, 0x48A), + (0x48D, 0x1, 0x48C), + (0x48F, 0x1, 0x48E), + (0x491, 0x1, 0x490), + (0x493, 0x1, 0x492), + (0x495, 0x1, 0x494), + (0x497, 0x1, 0x496), + (0x499, 0x1, 0x498), + (0x49B, 0x1, 0x49A), + (0x49D, 0x1, 0x49C), + (0x49F, 0x1, 0x49E), + (0x4A1, 0x1, 0x4A0), + (0x4A3, 0x1, 0x4A2), + (0x4A5, 0x1, 0x4A4), + (0x4A7, 0x1, 0x4A6), + (0x4A9, 0x1, 0x4A8), + (0x4AB, 0x1, 0x4AA), + (0x4AD, 0x1, 0x4AC), + (0x4AF, 0x1, 0x4AE), + (0x4B1, 0x1, 0x4B0), + (0x4B3, 0x1, 0x4B2), + (0x4B5, 0x1, 0x4B4), + (0x4B7, 0x1, 0x4B6), + (0x4B9, 0x1, 0x4B8), + (0x4BB, 0x1, 0x4BA), + (0x4BD, 0x1, 0x4BC), + (0x4BF, 0x1, 0x4BE), + (0x4C2, 0x1, 0x4C1), + (0x4C4, 0x1, 0x4C3), + (0x4C6, 0x1, 0x4C5), + (0x4C8, 0x1, 0x4C7), + (0x4CA, 0x1, 0x4C9), + (0x4CC, 0x1, 0x4CB), + (0x4CE, 0x1, 0x4CD), + (0x4CF, 0x1, 0x4C0), + (0x4D1, 0x1, 0x4D0), + (0x4D3, 0x1, 0x4D2), + (0x4D5, 0x1, 0x4D4), + (0x4D7, 0x1, 0x4D6), + (0x4D9, 0x1, 0x4D8), + (0x4DB, 0x1, 0x4DA), + (0x4DD, 0x1, 0x4DC), + (0x4DF, 0x1, 0x4DE), + (0x4E1, 0x1, 0x4E0), + (0x4E3, 0x1, 0x4E2), + (0x4E5, 0x1, 0x4E4), + (0x4E7, 0x1, 0x4E6), + (0x4E9, 0x1, 0x4E8), + (0x4EB, 0x1, 0x4EA), + (0x4ED, 0x1, 0x4EC), + (0x4EF, 0x1, 0x4EE), + (0x4F1, 0x1, 0x4F0), + (0x4F3, 0x1, 0x4F2), + (0x4F5, 0x1, 0x4F4), + (0x4F7, 0x1, 0x4F6), + (0x4F9, 0x1, 0x4F8), + (0x4FB, 0x1, 0x4FA), + (0x4FD, 0x1, 0x4FC), + (0x4FF, 0x1, 0x4FE), + (0x501, 0x1, 0x500), + (0x503, 0x1, 0x502), + (0x505, 0x1, 0x504), + (0x507, 0x1, 0x506), + (0x509, 0x1, 0x508), + (0x50B, 0x1, 0x50A), + (0x50D, 0x1, 0x50C), + (0x50F, 0x1, 0x50E), + (0x511, 0x1, 0x510), + (0x513, 0x1, 0x512), + (0x515, 0x1, 0x514), + (0x517, 0x1, 0x516), + (0x519, 0x1, 0x518), + (0x51B, 0x1, 0x51A), + (0x51D, 0x1, 0x51C), + (0x51F, 0x1, 0x51E), + (0x521, 0x1, 0x520), + (0x523, 0x1, 0x522), + (0x525, 0x1, 0x524), + (0x527, 0x1, 0x526), + (0x529, 0x1, 0x528), + (0x52B, 0x1, 0x52A), + (0x52D, 0x1, 0x52C), + (0x52F, 0x1, 0x52E), + (0x561, 0x1, 0x531), + (0x562, 0x1, 0x532), + (0x563, 0x1, 0x533), + (0x564, 0x1, 0x534), + (0x565, 0x1, 0x535), + (0x566, 0x1, 0x536), + (0x567, 0x1, 0x537), + (0x568, 0x1, 0x538), + (0x569, 0x1, 0x539), + (0x56A, 0x1, 0x53A), + (0x56B, 0x1, 0x53B), + (0x56C, 0x1, 0x53C), + (0x56D, 0x1, 0x53D), + (0x56E, 0x1, 0x53E), + (0x56F, 0x1, 0x53F), + (0x570, 0x1, 0x540), + (0x571, 0x1, 0x541), + (0x572, 0x1, 0x542), + (0x573, 0x1, 0x543), + (0x574, 0x1, 0x544), + (0x575, 0x1, 0x545), + (0x576, 0x1, 0x546), + (0x577, 0x1, 0x547), + (0x578, 0x1, 0x548), + (0x579, 0x1, 0x549), + (0x57A, 0x1, 0x54A), + (0x57B, 0x1, 0x54B), + (0x57C, 0x1, 0x54C), + (0x57D, 0x1, 0x54D), + (0x57E, 0x1, 0x54E), + (0x57F, 0x1, 0x54F), + (0x580, 0x1, 0x550), + (0x581, 0x1, 0x551), + (0x582, 0x1, 0x552), + (0x583, 0x1, 0x553), + (0x584, 0x1, 0x554), + (0x585, 0x1, 0x555), + (0x586, 0x1, 0x556), + (0x587, 0x2, 0x535, 0x552), + (0x10D0, 0x1, 0x1C90), + (0x10D1, 0x1, 0x1C91), + (0x10D2, 0x1, 0x1C92), + (0x10D3, 0x1, 0x1C93), + (0x10D4, 0x1, 0x1C94), + (0x10D5, 0x1, 0x1C95), + (0x10D6, 0x1, 0x1C96), + (0x10D7, 0x1, 0x1C97), + (0x10D8, 0x1, 0x1C98), + (0x10D9, 0x1, 0x1C99), + (0x10DA, 0x1, 0x1C9A), + (0x10DB, 0x1, 0x1C9B), + (0x10DC, 0x1, 0x1C9C), + (0x10DD, 0x1, 0x1C9D), + (0x10DE, 0x1, 0x1C9E), + (0x10DF, 0x1, 0x1C9F), + (0x10E0, 0x1, 0x1CA0), + (0x10E1, 0x1, 0x1CA1), + (0x10E2, 0x1, 0x1CA2), + (0x10E3, 0x1, 0x1CA3), + (0x10E4, 0x1, 0x1CA4), + (0x10E5, 0x1, 0x1CA5), + (0x10E6, 0x1, 0x1CA6), + (0x10E7, 0x1, 0x1CA7), + (0x10E8, 0x1, 0x1CA8), + (0x10E9, 0x1, 0x1CA9), + (0x10EA, 0x1, 0x1CAA), + (0x10EB, 0x1, 0x1CAB), + (0x10EC, 0x1, 0x1CAC), + (0x10ED, 0x1, 0x1CAD), + (0x10EE, 0x1, 0x1CAE), + (0x10EF, 0x1, 0x1CAF), + (0x10F0, 0x1, 0x1CB0), + (0x10F1, 0x1, 0x1CB1), + (0x10F2, 0x1, 0x1CB2), + (0x10F3, 0x1, 0x1CB3), + (0x10F4, 0x1, 0x1CB4), + (0x10F5, 0x1, 0x1CB5), + (0x10F6, 0x1, 0x1CB6), + (0x10F7, 0x1, 0x1CB7), + (0x10F8, 0x1, 0x1CB8), + (0x10F9, 0x1, 0x1CB9), + (0x10FA, 0x1, 0x1CBA), + (0x10FD, 0x1, 0x1CBD), + (0x10FE, 0x1, 0x1CBE), + (0x10FF, 0x1, 0x1CBF), + (0x13F8, 0x1, 0x13F0), + (0x13F9, 0x1, 0x13F1), + (0x13FA, 0x1, 0x13F2), + (0x13FB, 0x1, 0x13F3), + (0x13FC, 0x1, 0x13F4), + (0x13FD, 0x1, 0x13F5), + (0x1C80, 0x1, 0x412), + (0x1C81, 0x1, 0x414), + (0x1C82, 0x1, 0x41E), + (0x1C83, 0x1, 0x421), + (0x1C84, 0x1, 0x422), + (0x1C85, 0x1, 0x422), + (0x1C86, 0x1, 0x42A), + (0x1C87, 0x1, 0x462), + (0x1C88, 0x1, 0xA64A), + (0x1D79, 0x1, 0xA77D), + (0x1D7D, 0x1, 0x2C63), + (0x1D8E, 0x1, 0xA7C6), + (0x1E01, 0x1, 0x1E00), + (0x1E03, 0x1, 0x1E02), + (0x1E05, 0x1, 0x1E04), + (0x1E07, 0x1, 0x1E06), + (0x1E09, 0x1, 0x1E08), + (0x1E0B, 0x1, 0x1E0A), + (0x1E0D, 0x1, 0x1E0C), + (0x1E0F, 0x1, 0x1E0E), + (0x1E11, 0x1, 0x1E10), + (0x1E13, 0x1, 0x1E12), + (0x1E15, 0x1, 0x1E14), + (0x1E17, 0x1, 0x1E16), + (0x1E19, 0x1, 0x1E18), + (0x1E1B, 0x1, 0x1E1A), + (0x1E1D, 0x1, 0x1E1C), + (0x1E1F, 0x1, 0x1E1E), + (0x1E21, 0x1, 0x1E20), + (0x1E23, 0x1, 0x1E22), + (0x1E25, 0x1, 0x1E24), + (0x1E27, 0x1, 0x1E26), + (0x1E29, 0x1, 0x1E28), + (0x1E2B, 0x1, 0x1E2A), + (0x1E2D, 0x1, 0x1E2C), + (0x1E2F, 0x1, 0x1E2E), + (0x1E31, 0x1, 0x1E30), + (0x1E33, 0x1, 0x1E32), + (0x1E35, 0x1, 0x1E34), + (0x1E37, 0x1, 0x1E36), + (0x1E39, 0x1, 0x1E38), + (0x1E3B, 0x1, 0x1E3A), + (0x1E3D, 0x1, 0x1E3C), + (0x1E3F, 0x1, 0x1E3E), + (0x1E41, 0x1, 0x1E40), + (0x1E43, 0x1, 0x1E42), + (0x1E45, 0x1, 0x1E44), + (0x1E47, 0x1, 0x1E46), + (0x1E49, 0x1, 0x1E48), + (0x1E4B, 0x1, 0x1E4A), + (0x1E4D, 0x1, 0x1E4C), + (0x1E4F, 0x1, 0x1E4E), + (0x1E51, 0x1, 0x1E50), + (0x1E53, 0x1, 0x1E52), + (0x1E55, 0x1, 0x1E54), + (0x1E57, 0x1, 0x1E56), + (0x1E59, 0x1, 0x1E58), + (0x1E5B, 0x1, 0x1E5A), + (0x1E5D, 0x1, 0x1E5C), + (0x1E5F, 0x1, 0x1E5E), + (0x1E61, 0x1, 0x1E60), + (0x1E63, 0x1, 0x1E62), + (0x1E65, 0x1, 0x1E64), + (0x1E67, 0x1, 0x1E66), + (0x1E69, 0x1, 0x1E68), + (0x1E6B, 0x1, 0x1E6A), + (0x1E6D, 0x1, 0x1E6C), + (0x1E6F, 0x1, 0x1E6E), + (0x1E71, 0x1, 0x1E70), + (0x1E73, 0x1, 0x1E72), + (0x1E75, 0x1, 0x1E74), + (0x1E77, 0x1, 0x1E76), + (0x1E79, 0x1, 0x1E78), + (0x1E7B, 0x1, 0x1E7A), + (0x1E7D, 0x1, 0x1E7C), + (0x1E7F, 0x1, 0x1E7E), + (0x1E81, 0x1, 0x1E80), + (0x1E83, 0x1, 0x1E82), + (0x1E85, 0x1, 0x1E84), + (0x1E87, 0x1, 0x1E86), + (0x1E89, 0x1, 0x1E88), + (0x1E8B, 0x1, 0x1E8A), + (0x1E8D, 0x1, 0x1E8C), + (0x1E8F, 0x1, 0x1E8E), + (0x1E91, 0x1, 0x1E90), + (0x1E93, 0x1, 0x1E92), + (0x1E95, 0x1, 0x1E94), + (0x1E96, 0x2, 0x48, 0x331), + (0x1E97, 0x2, 0x54, 0x308), + (0x1E98, 0x2, 0x57, 0x30A), + (0x1E99, 0x2, 0x59, 0x30A), + (0x1E9A, 0x2, 0x41, 0x2BE), + (0x1E9B, 0x1, 0x1E60), + (0x1EA1, 0x1, 0x1EA0), + (0x1EA3, 0x1, 0x1EA2), + (0x1EA5, 0x1, 0x1EA4), + (0x1EA7, 0x1, 0x1EA6), + (0x1EA9, 0x1, 0x1EA8), + (0x1EAB, 0x1, 0x1EAA), + (0x1EAD, 0x1, 0x1EAC), + (0x1EAF, 0x1, 0x1EAE), + (0x1EB1, 0x1, 0x1EB0), + (0x1EB3, 0x1, 0x1EB2), + (0x1EB5, 0x1, 0x1EB4), + (0x1EB7, 0x1, 0x1EB6), + (0x1EB9, 0x1, 0x1EB8), + (0x1EBB, 0x1, 0x1EBA), + (0x1EBD, 0x1, 0x1EBC), + (0x1EBF, 0x1, 0x1EBE), + (0x1EC1, 0x1, 0x1EC0), + (0x1EC3, 0x1, 0x1EC2), + (0x1EC5, 0x1, 0x1EC4), + (0x1EC7, 0x1, 0x1EC6), + (0x1EC9, 0x1, 0x1EC8), + (0x1ECB, 0x1, 0x1ECA), + (0x1ECD, 0x1, 0x1ECC), + (0x1ECF, 0x1, 0x1ECE), + (0x1ED1, 0x1, 0x1ED0), + (0x1ED3, 0x1, 0x1ED2), + (0x1ED5, 0x1, 0x1ED4), + (0x1ED7, 0x1, 0x1ED6), + (0x1ED9, 0x1, 0x1ED8), + (0x1EDB, 0x1, 0x1EDA), + (0x1EDD, 0x1, 0x1EDC), + (0x1EDF, 0x1, 0x1EDE), + (0x1EE1, 0x1, 0x1EE0), + (0x1EE3, 0x1, 0x1EE2), + (0x1EE5, 0x1, 0x1EE4), + (0x1EE7, 0x1, 0x1EE6), + (0x1EE9, 0x1, 0x1EE8), + (0x1EEB, 0x1, 0x1EEA), + (0x1EED, 0x1, 0x1EEC), + (0x1EEF, 0x1, 0x1EEE), + (0x1EF1, 0x1, 0x1EF0), + (0x1EF3, 0x1, 0x1EF2), + (0x1EF5, 0x1, 0x1EF4), + (0x1EF7, 0x1, 0x1EF6), + (0x1EF9, 0x1, 0x1EF8), + (0x1EFB, 0x1, 0x1EFA), + (0x1EFD, 0x1, 0x1EFC), + (0x1EFF, 0x1, 0x1EFE), + (0x1F00, 0x1, 0x1F08), + (0x1F01, 0x1, 0x1F09), + (0x1F02, 0x1, 0x1F0A), + (0x1F03, 0x1, 0x1F0B), + (0x1F04, 0x1, 0x1F0C), + (0x1F05, 0x1, 0x1F0D), + (0x1F06, 0x1, 0x1F0E), + (0x1F07, 0x1, 0x1F0F), + (0x1F10, 0x1, 0x1F18), + (0x1F11, 0x1, 0x1F19), + (0x1F12, 0x1, 0x1F1A), + (0x1F13, 0x1, 0x1F1B), + (0x1F14, 0x1, 0x1F1C), + (0x1F15, 0x1, 0x1F1D), + (0x1F20, 0x1, 0x1F28), + (0x1F21, 0x1, 0x1F29), + (0x1F22, 0x1, 0x1F2A), + (0x1F23, 0x1, 0x1F2B), + (0x1F24, 0x1, 0x1F2C), + (0x1F25, 0x1, 0x1F2D), + (0x1F26, 0x1, 0x1F2E), + (0x1F27, 0x1, 0x1F2F), + (0x1F30, 0x1, 0x1F38), + (0x1F31, 0x1, 0x1F39), + (0x1F32, 0x1, 0x1F3A), + (0x1F33, 0x1, 0x1F3B), + (0x1F34, 0x1, 0x1F3C), + (0x1F35, 0x1, 0x1F3D), + (0x1F36, 0x1, 0x1F3E), + (0x1F37, 0x1, 0x1F3F), + (0x1F40, 0x1, 0x1F48), + (0x1F41, 0x1, 0x1F49), + (0x1F42, 0x1, 0x1F4A), + (0x1F43, 0x1, 0x1F4B), + (0x1F44, 0x1, 0x1F4C), + (0x1F45, 0x1, 0x1F4D), + (0x1F50, 0x2, 0x3A5, 0x313), + (0x1F51, 0x1, 0x1F59), + (0x1F52, 0x3, 0x3A5, 0x313, 0x300), + (0x1F53, 0x1, 0x1F5B), + (0x1F54, 0x3, 0x3A5, 0x313, 0x301), + (0x1F55, 0x1, 0x1F5D), + (0x1F56, 0x3, 0x3A5, 0x313, 0x342), + (0x1F57, 0x1, 0x1F5F), + (0x1F60, 0x1, 0x1F68), + (0x1F61, 0x1, 0x1F69), + (0x1F62, 0x1, 0x1F6A), + (0x1F63, 0x1, 0x1F6B), + (0x1F64, 0x1, 0x1F6C), + (0x1F65, 0x1, 0x1F6D), + (0x1F66, 0x1, 0x1F6E), + (0x1F67, 0x1, 0x1F6F), + (0x1F70, 0x1, 0x1FBA), + (0x1F71, 0x1, 0x1FBB), + (0x1F72, 0x1, 0x1FC8), + (0x1F73, 0x1, 0x1FC9), + (0x1F74, 0x1, 0x1FCA), + (0x1F75, 0x1, 0x1FCB), + (0x1F76, 0x1, 0x1FDA), + (0x1F77, 0x1, 0x1FDB), + (0x1F78, 0x1, 0x1FF8), + (0x1F79, 0x1, 0x1FF9), + (0x1F7A, 0x1, 0x1FEA), + (0x1F7B, 0x1, 0x1FEB), + (0x1F7C, 0x1, 0x1FFA), + (0x1F7D, 0x1, 0x1FFB), + (0x1F80, 0x2, 0x1F08, 0x399), + (0x1F81, 0x2, 0x1F09, 0x399), + (0x1F82, 0x2, 0x1F0A, 0x399), + (0x1F83, 0x2, 0x1F0B, 0x399), + (0x1F84, 0x2, 0x1F0C, 0x399), + (0x1F85, 0x2, 0x1F0D, 0x399), + (0x1F86, 0x2, 0x1F0E, 0x399), + (0x1F87, 0x2, 0x1F0F, 0x399), + (0x1F88, 0x2, 0x1F08, 0x399), + (0x1F89, 0x2, 0x1F09, 0x399), + (0x1F8A, 0x2, 0x1F0A, 0x399), + (0x1F8B, 0x2, 0x1F0B, 0x399), + (0x1F8C, 0x2, 0x1F0C, 0x399), + (0x1F8D, 0x2, 0x1F0D, 0x399), + (0x1F8E, 0x2, 0x1F0E, 0x399), + (0x1F8F, 0x2, 0x1F0F, 0x399), + (0x1F90, 0x2, 0x1F28, 0x399), + (0x1F91, 0x2, 0x1F29, 0x399), + (0x1F92, 0x2, 0x1F2A, 0x399), + (0x1F93, 0x2, 0x1F2B, 0x399), + (0x1F94, 0x2, 0x1F2C, 0x399), + (0x1F95, 0x2, 0x1F2D, 0x399), + (0x1F96, 0x2, 0x1F2E, 0x399), + (0x1F97, 0x2, 0x1F2F, 0x399), + (0x1F98, 0x2, 0x1F28, 0x399), + (0x1F99, 0x2, 0x1F29, 0x399), + (0x1F9A, 0x2, 0x1F2A, 0x399), + (0x1F9B, 0x2, 0x1F2B, 0x399), + (0x1F9C, 0x2, 0x1F2C, 0x399), + (0x1F9D, 0x2, 0x1F2D, 0x399), + (0x1F9E, 0x2, 0x1F2E, 0x399), + (0x1F9F, 0x2, 0x1F2F, 0x399), + (0x1FA0, 0x2, 0x1F68, 0x399), + (0x1FA1, 0x2, 0x1F69, 0x399), + (0x1FA2, 0x2, 0x1F6A, 0x399), + (0x1FA3, 0x2, 0x1F6B, 0x399), + (0x1FA4, 0x2, 0x1F6C, 0x399), + (0x1FA5, 0x2, 0x1F6D, 0x399), + (0x1FA6, 0x2, 0x1F6E, 0x399), + (0x1FA7, 0x2, 0x1F6F, 0x399), + (0x1FA8, 0x2, 0x1F68, 0x399), + (0x1FA9, 0x2, 0x1F69, 0x399), + (0x1FAA, 0x2, 0x1F6A, 0x399), + (0x1FAB, 0x2, 0x1F6B, 0x399), + (0x1FAC, 0x2, 0x1F6C, 0x399), + (0x1FAD, 0x2, 0x1F6D, 0x399), + (0x1FAE, 0x2, 0x1F6E, 0x399), + (0x1FAF, 0x2, 0x1F6F, 0x399), + (0x1FB0, 0x1, 0x1FB8), + (0x1FB1, 0x1, 0x1FB9), + (0x1FB2, 0x2, 0x1FBA, 0x399), + (0x1FB3, 0x2, 0x391, 0x399), + (0x1FB4, 0x2, 0x386, 0x399), + (0x1FB6, 0x2, 0x391, 0x342), + (0x1FB7, 0x3, 0x391, 0x342, 0x399), + (0x1FBC, 0x2, 0x391, 0x399), + (0x1FBE, 0x1, 0x399), + (0x1FC2, 0x2, 0x1FCA, 0x399), + (0x1FC3, 0x2, 0x397, 0x399), + (0x1FC4, 0x2, 0x389, 0x399), + (0x1FC6, 0x2, 0x397, 0x342), + (0x1FC7, 0x3, 0x397, 0x342, 0x399), + (0x1FCC, 0x2, 0x397, 0x399), + (0x1FD0, 0x1, 0x1FD8), + (0x1FD1, 0x1, 0x1FD9), + (0x1FD2, 0x3, 0x399, 0x308, 0x300), + (0x1FD3, 0x3, 0x399, 0x308, 0x301), + (0x1FD6, 0x2, 0x399, 0x342), + (0x1FD7, 0x3, 0x399, 0x308, 0x342), + (0x1FE0, 0x1, 0x1FE8), + (0x1FE1, 0x1, 0x1FE9), + (0x1FE2, 0x3, 0x3A5, 0x308, 0x300), + (0x1FE3, 0x3, 0x3A5, 0x308, 0x301), + (0x1FE4, 0x2, 0x3A1, 0x313), + (0x1FE5, 0x1, 0x1FEC), + (0x1FE6, 0x2, 0x3A5, 0x342), + (0x1FE7, 0x3, 0x3A5, 0x308, 0x342), + (0x1FF2, 0x2, 0x1FFA, 0x399), + (0x1FF3, 0x2, 0x3A9, 0x399), + (0x1FF4, 0x2, 0x38F, 0x399), + (0x1FF6, 0x2, 0x3A9, 0x342), + (0x1FF7, 0x3, 0x3A9, 0x342, 0x399), + (0x1FFC, 0x2, 0x3A9, 0x399), + (0x214E, 0x1, 0x2132), + (0x2170, 0x1, 0x2160), + (0x2171, 0x1, 0x2161), + (0x2172, 0x1, 0x2162), + (0x2173, 0x1, 0x2163), + (0x2174, 0x1, 0x2164), + (0x2175, 0x1, 0x2165), + (0x2176, 0x1, 0x2166), + (0x2177, 0x1, 0x2167), + (0x2178, 0x1, 0x2168), + (0x2179, 0x1, 0x2169), + (0x217A, 0x1, 0x216A), + (0x217B, 0x1, 0x216B), + (0x217C, 0x1, 0x216C), + (0x217D, 0x1, 0x216D), + (0x217E, 0x1, 0x216E), + (0x217F, 0x1, 0x216F), + (0x2184, 0x1, 0x2183), + (0x24D0, 0x1, 0x24B6), + (0x24D1, 0x1, 0x24B7), + (0x24D2, 0x1, 0x24B8), + (0x24D3, 0x1, 0x24B9), + (0x24D4, 0x1, 0x24BA), + (0x24D5, 0x1, 0x24BB), + (0x24D6, 0x1, 0x24BC), + (0x24D7, 0x1, 0x24BD), + (0x24D8, 0x1, 0x24BE), + (0x24D9, 0x1, 0x24BF), + (0x24DA, 0x1, 0x24C0), + (0x24DB, 0x1, 0x24C1), + (0x24DC, 0x1, 0x24C2), + (0x24DD, 0x1, 0x24C3), + (0x24DE, 0x1, 0x24C4), + (0x24DF, 0x1, 0x24C5), + (0x24E0, 0x1, 0x24C6), + (0x24E1, 0x1, 0x24C7), + (0x24E2, 0x1, 0x24C8), + (0x24E3, 0x1, 0x24C9), + (0x24E4, 0x1, 0x24CA), + (0x24E5, 0x1, 0x24CB), + (0x24E6, 0x1, 0x24CC), + (0x24E7, 0x1, 0x24CD), + (0x24E8, 0x1, 0x24CE), + (0x24E9, 0x1, 0x24CF), + (0x2C30, 0x1, 0x2C00), + (0x2C31, 0x1, 0x2C01), + (0x2C32, 0x1, 0x2C02), + (0x2C33, 0x1, 0x2C03), + (0x2C34, 0x1, 0x2C04), + (0x2C35, 0x1, 0x2C05), + (0x2C36, 0x1, 0x2C06), + (0x2C37, 0x1, 0x2C07), + (0x2C38, 0x1, 0x2C08), + (0x2C39, 0x1, 0x2C09), + (0x2C3A, 0x1, 0x2C0A), + (0x2C3B, 0x1, 0x2C0B), + (0x2C3C, 0x1, 0x2C0C), + (0x2C3D, 0x1, 0x2C0D), + (0x2C3E, 0x1, 0x2C0E), + (0x2C3F, 0x1, 0x2C0F), + (0x2C40, 0x1, 0x2C10), + (0x2C41, 0x1, 0x2C11), + (0x2C42, 0x1, 0x2C12), + (0x2C43, 0x1, 0x2C13), + (0x2C44, 0x1, 0x2C14), + (0x2C45, 0x1, 0x2C15), + (0x2C46, 0x1, 0x2C16), + (0x2C47, 0x1, 0x2C17), + (0x2C48, 0x1, 0x2C18), + (0x2C49, 0x1, 0x2C19), + (0x2C4A, 0x1, 0x2C1A), + (0x2C4B, 0x1, 0x2C1B), + (0x2C4C, 0x1, 0x2C1C), + (0x2C4D, 0x1, 0x2C1D), + (0x2C4E, 0x1, 0x2C1E), + (0x2C4F, 0x1, 0x2C1F), + (0x2C50, 0x1, 0x2C20), + (0x2C51, 0x1, 0x2C21), + (0x2C52, 0x1, 0x2C22), + (0x2C53, 0x1, 0x2C23), + (0x2C54, 0x1, 0x2C24), + (0x2C55, 0x1, 0x2C25), + (0x2C56, 0x1, 0x2C26), + (0x2C57, 0x1, 0x2C27), + (0x2C58, 0x1, 0x2C28), + (0x2C59, 0x1, 0x2C29), + (0x2C5A, 0x1, 0x2C2A), + (0x2C5B, 0x1, 0x2C2B), + (0x2C5C, 0x1, 0x2C2C), + (0x2C5D, 0x1, 0x2C2D), + (0x2C5E, 0x1, 0x2C2E), + (0x2C5F, 0x1, 0x2C2F), + (0x2C61, 0x1, 0x2C60), + (0x2C65, 0x1, 0x23A), + (0x2C66, 0x1, 0x23E), + (0x2C68, 0x1, 0x2C67), + (0x2C6A, 0x1, 0x2C69), + (0x2C6C, 0x1, 0x2C6B), + (0x2C73, 0x1, 0x2C72), + (0x2C76, 0x1, 0x2C75), + (0x2C81, 0x1, 0x2C80), + (0x2C83, 0x1, 0x2C82), + (0x2C85, 0x1, 0x2C84), + (0x2C87, 0x1, 0x2C86), + (0x2C89, 0x1, 0x2C88), + (0x2C8B, 0x1, 0x2C8A), + (0x2C8D, 0x1, 0x2C8C), + (0x2C8F, 0x1, 0x2C8E), + (0x2C91, 0x1, 0x2C90), + (0x2C93, 0x1, 0x2C92), + (0x2C95, 0x1, 0x2C94), + (0x2C97, 0x1, 0x2C96), + (0x2C99, 0x1, 0x2C98), + (0x2C9B, 0x1, 0x2C9A), + (0x2C9D, 0x1, 0x2C9C), + (0x2C9F, 0x1, 0x2C9E), + (0x2CA1, 0x1, 0x2CA0), + (0x2CA3, 0x1, 0x2CA2), + (0x2CA5, 0x1, 0x2CA4), + (0x2CA7, 0x1, 0x2CA6), + (0x2CA9, 0x1, 0x2CA8), + (0x2CAB, 0x1, 0x2CAA), + (0x2CAD, 0x1, 0x2CAC), + (0x2CAF, 0x1, 0x2CAE), + (0x2CB1, 0x1, 0x2CB0), + (0x2CB3, 0x1, 0x2CB2), + (0x2CB5, 0x1, 0x2CB4), + (0x2CB7, 0x1, 0x2CB6), + (0x2CB9, 0x1, 0x2CB8), + (0x2CBB, 0x1, 0x2CBA), + (0x2CBD, 0x1, 0x2CBC), + (0x2CBF, 0x1, 0x2CBE), + (0x2CC1, 0x1, 0x2CC0), + (0x2CC3, 0x1, 0x2CC2), + (0x2CC5, 0x1, 0x2CC4), + (0x2CC7, 0x1, 0x2CC6), + (0x2CC9, 0x1, 0x2CC8), + (0x2CCB, 0x1, 0x2CCA), + (0x2CCD, 0x1, 0x2CCC), + (0x2CCF, 0x1, 0x2CCE), + (0x2CD1, 0x1, 0x2CD0), + (0x2CD3, 0x1, 0x2CD2), + (0x2CD5, 0x1, 0x2CD4), + (0x2CD7, 0x1, 0x2CD6), + (0x2CD9, 0x1, 0x2CD8), + (0x2CDB, 0x1, 0x2CDA), + (0x2CDD, 0x1, 0x2CDC), + (0x2CDF, 0x1, 0x2CDE), + (0x2CE1, 0x1, 0x2CE0), + (0x2CE3, 0x1, 0x2CE2), + (0x2CEC, 0x1, 0x2CEB), + (0x2CEE, 0x1, 0x2CED), + (0x2CF3, 0x1, 0x2CF2), + (0x2D00, 0x1, 0x10A0), + (0x2D01, 0x1, 0x10A1), + (0x2D02, 0x1, 0x10A2), + (0x2D03, 0x1, 0x10A3), + (0x2D04, 0x1, 0x10A4), + (0x2D05, 0x1, 0x10A5), + (0x2D06, 0x1, 0x10A6), + (0x2D07, 0x1, 0x10A7), + (0x2D08, 0x1, 0x10A8), + (0x2D09, 0x1, 0x10A9), + (0x2D0A, 0x1, 0x10AA), + (0x2D0B, 0x1, 0x10AB), + (0x2D0C, 0x1, 0x10AC), + (0x2D0D, 0x1, 0x10AD), + (0x2D0E, 0x1, 0x10AE), + (0x2D0F, 0x1, 0x10AF), + (0x2D10, 0x1, 0x10B0), + (0x2D11, 0x1, 0x10B1), + (0x2D12, 0x1, 0x10B2), + (0x2D13, 0x1, 0x10B3), + (0x2D14, 0x1, 0x10B4), + (0x2D15, 0x1, 0x10B5), + (0x2D16, 0x1, 0x10B6), + (0x2D17, 0x1, 0x10B7), + (0x2D18, 0x1, 0x10B8), + (0x2D19, 0x1, 0x10B9), + (0x2D1A, 0x1, 0x10BA), + (0x2D1B, 0x1, 0x10BB), + (0x2D1C, 0x1, 0x10BC), + (0x2D1D, 0x1, 0x10BD), + (0x2D1E, 0x1, 0x10BE), + (0x2D1F, 0x1, 0x10BF), + (0x2D20, 0x1, 0x10C0), + (0x2D21, 0x1, 0x10C1), + (0x2D22, 0x1, 0x10C2), + (0x2D23, 0x1, 0x10C3), + (0x2D24, 0x1, 0x10C4), + (0x2D25, 0x1, 0x10C5), + (0x2D27, 0x1, 0x10C7), + (0x2D2D, 0x1, 0x10CD), + (0xA641, 0x1, 0xA640), + (0xA643, 0x1, 0xA642), + (0xA645, 0x1, 0xA644), + (0xA647, 0x1, 0xA646), + (0xA649, 0x1, 0xA648), + (0xA64B, 0x1, 0xA64A), + (0xA64D, 0x1, 0xA64C), + (0xA64F, 0x1, 0xA64E), + (0xA651, 0x1, 0xA650), + (0xA653, 0x1, 0xA652), + (0xA655, 0x1, 0xA654), + (0xA657, 0x1, 0xA656), + (0xA659, 0x1, 0xA658), + (0xA65B, 0x1, 0xA65A), + (0xA65D, 0x1, 0xA65C), + (0xA65F, 0x1, 0xA65E), + (0xA661, 0x1, 0xA660), + (0xA663, 0x1, 0xA662), + (0xA665, 0x1, 0xA664), + (0xA667, 0x1, 0xA666), + (0xA669, 0x1, 0xA668), + (0xA66B, 0x1, 0xA66A), + (0xA66D, 0x1, 0xA66C), + (0xA681, 0x1, 0xA680), + (0xA683, 0x1, 0xA682), + (0xA685, 0x1, 0xA684), + (0xA687, 0x1, 0xA686), + (0xA689, 0x1, 0xA688), + (0xA68B, 0x1, 0xA68A), + (0xA68D, 0x1, 0xA68C), + (0xA68F, 0x1, 0xA68E), + (0xA691, 0x1, 0xA690), + (0xA693, 0x1, 0xA692), + (0xA695, 0x1, 0xA694), + (0xA697, 0x1, 0xA696), + (0xA699, 0x1, 0xA698), + (0xA69B, 0x1, 0xA69A), + (0xA723, 0x1, 0xA722), + (0xA725, 0x1, 0xA724), + (0xA727, 0x1, 0xA726), + (0xA729, 0x1, 0xA728), + (0xA72B, 0x1, 0xA72A), + (0xA72D, 0x1, 0xA72C), + (0xA72F, 0x1, 0xA72E), + (0xA733, 0x1, 0xA732), + (0xA735, 0x1, 0xA734), + (0xA737, 0x1, 0xA736), + (0xA739, 0x1, 0xA738), + (0xA73B, 0x1, 0xA73A), + (0xA73D, 0x1, 0xA73C), + (0xA73F, 0x1, 0xA73E), + (0xA741, 0x1, 0xA740), + (0xA743, 0x1, 0xA742), + (0xA745, 0x1, 0xA744), + (0xA747, 0x1, 0xA746), + (0xA749, 0x1, 0xA748), + (0xA74B, 0x1, 0xA74A), + (0xA74D, 0x1, 0xA74C), + (0xA74F, 0x1, 0xA74E), + (0xA751, 0x1, 0xA750), + (0xA753, 0x1, 0xA752), + (0xA755, 0x1, 0xA754), + (0xA757, 0x1, 0xA756), + (0xA759, 0x1, 0xA758), + (0xA75B, 0x1, 0xA75A), + (0xA75D, 0x1, 0xA75C), + (0xA75F, 0x1, 0xA75E), + (0xA761, 0x1, 0xA760), + (0xA763, 0x1, 0xA762), + (0xA765, 0x1, 0xA764), + (0xA767, 0x1, 0xA766), + (0xA769, 0x1, 0xA768), + (0xA76B, 0x1, 0xA76A), + (0xA76D, 0x1, 0xA76C), + (0xA76F, 0x1, 0xA76E), + (0xA77A, 0x1, 0xA779), + (0xA77C, 0x1, 0xA77B), + (0xA77F, 0x1, 0xA77E), + (0xA781, 0x1, 0xA780), + (0xA783, 0x1, 0xA782), + (0xA785, 0x1, 0xA784), + (0xA787, 0x1, 0xA786), + (0xA78C, 0x1, 0xA78B), + (0xA791, 0x1, 0xA790), + (0xA793, 0x1, 0xA792), + (0xA794, 0x1, 0xA7C4), + (0xA797, 0x1, 0xA796), + (0xA799, 0x1, 0xA798), + (0xA79B, 0x1, 0xA79A), + (0xA79D, 0x1, 0xA79C), + (0xA79F, 0x1, 0xA79E), + (0xA7A1, 0x1, 0xA7A0), + (0xA7A3, 0x1, 0xA7A2), + (0xA7A5, 0x1, 0xA7A4), + (0xA7A7, 0x1, 0xA7A6), + (0xA7A9, 0x1, 0xA7A8), + (0xA7B5, 0x1, 0xA7B4), + (0xA7B7, 0x1, 0xA7B6), + (0xA7B9, 0x1, 0xA7B8), + (0xA7BB, 0x1, 0xA7BA), + (0xA7BD, 0x1, 0xA7BC), + (0xA7BF, 0x1, 0xA7BE), + (0xA7C1, 0x1, 0xA7C0), + (0xA7C3, 0x1, 0xA7C2), + (0xA7C8, 0x1, 0xA7C7), + (0xA7CA, 0x1, 0xA7C9), + (0xA7D1, 0x1, 0xA7D0), + (0xA7D7, 0x1, 0xA7D6), + (0xA7D9, 0x1, 0xA7D8), + (0xA7F6, 0x1, 0xA7F5), + (0xAB53, 0x1, 0xA7B3), + (0xAB70, 0x1, 0x13A0), + (0xAB71, 0x1, 0x13A1), + (0xAB72, 0x1, 0x13A2), + (0xAB73, 0x1, 0x13A3), + (0xAB74, 0x1, 0x13A4), + (0xAB75, 0x1, 0x13A5), + (0xAB76, 0x1, 0x13A6), + (0xAB77, 0x1, 0x13A7), + (0xAB78, 0x1, 0x13A8), + (0xAB79, 0x1, 0x13A9), + (0xAB7A, 0x1, 0x13AA), + (0xAB7B, 0x1, 0x13AB), + (0xAB7C, 0x1, 0x13AC), + (0xAB7D, 0x1, 0x13AD), + (0xAB7E, 0x1, 0x13AE), + (0xAB7F, 0x1, 0x13AF), + (0xAB80, 0x1, 0x13B0), + (0xAB81, 0x1, 0x13B1), + (0xAB82, 0x1, 0x13B2), + (0xAB83, 0x1, 0x13B3), + (0xAB84, 0x1, 0x13B4), + (0xAB85, 0x1, 0x13B5), + (0xAB86, 0x1, 0x13B6), + (0xAB87, 0x1, 0x13B7), + (0xAB88, 0x1, 0x13B8), + (0xAB89, 0x1, 0x13B9), + (0xAB8A, 0x1, 0x13BA), + (0xAB8B, 0x1, 0x13BB), + (0xAB8C, 0x1, 0x13BC), + (0xAB8D, 0x1, 0x13BD), + (0xAB8E, 0x1, 0x13BE), + (0xAB8F, 0x1, 0x13BF), + (0xAB90, 0x1, 0x13C0), + (0xAB91, 0x1, 0x13C1), + (0xAB92, 0x1, 0x13C2), + (0xAB93, 0x1, 0x13C3), + (0xAB94, 0x1, 0x13C4), + (0xAB95, 0x1, 0x13C5), + (0xAB96, 0x1, 0x13C6), + (0xAB97, 0x1, 0x13C7), + (0xAB98, 0x1, 0x13C8), + (0xAB99, 0x1, 0x13C9), + (0xAB9A, 0x1, 0x13CA), + (0xAB9B, 0x1, 0x13CB), + (0xAB9C, 0x1, 0x13CC), + (0xAB9D, 0x1, 0x13CD), + (0xAB9E, 0x1, 0x13CE), + (0xAB9F, 0x1, 0x13CF), + (0xABA0, 0x1, 0x13D0), + (0xABA1, 0x1, 0x13D1), + (0xABA2, 0x1, 0x13D2), + (0xABA3, 0x1, 0x13D3), + (0xABA4, 0x1, 0x13D4), + (0xABA5, 0x1, 0x13D5), + (0xABA6, 0x1, 0x13D6), + (0xABA7, 0x1, 0x13D7), + (0xABA8, 0x1, 0x13D8), + (0xABA9, 0x1, 0x13D9), + (0xABAA, 0x1, 0x13DA), + (0xABAB, 0x1, 0x13DB), + (0xABAC, 0x1, 0x13DC), + (0xABAD, 0x1, 0x13DD), + (0xABAE, 0x1, 0x13DE), + (0xABAF, 0x1, 0x13DF), + (0xABB0, 0x1, 0x13E0), + (0xABB1, 0x1, 0x13E1), + (0xABB2, 0x1, 0x13E2), + (0xABB3, 0x1, 0x13E3), + (0xABB4, 0x1, 0x13E4), + (0xABB5, 0x1, 0x13E5), + (0xABB6, 0x1, 0x13E6), + (0xABB7, 0x1, 0x13E7), + (0xABB8, 0x1, 0x13E8), + (0xABB9, 0x1, 0x13E9), + (0xABBA, 0x1, 0x13EA), + (0xABBB, 0x1, 0x13EB), + (0xABBC, 0x1, 0x13EC), + (0xABBD, 0x1, 0x13ED), + (0xABBE, 0x1, 0x13EE), + (0xABBF, 0x1, 0x13EF), + (0xFB00, 0x2, 0x46, 0x46), + (0xFB01, 0x2, 0x46, 0x49), + (0xFB02, 0x2, 0x46, 0x4C), + (0xFB03, 0x3, 0x46, 0x46, 0x49), + (0xFB04, 0x3, 0x46, 0x46, 0x4C), + (0xFB05, 0x2, 0x53, 0x54), + (0xFB06, 0x2, 0x53, 0x54), + (0xFB13, 0x2, 0x544, 0x546), + (0xFB14, 0x2, 0x544, 0x535), + (0xFB15, 0x2, 0x544, 0x53B), + (0xFB16, 0x2, 0x54E, 0x546), + (0xFB17, 0x2, 0x544, 0x53D), + (0xFF41, 0x1, 0xFF21), + (0xFF42, 0x1, 0xFF22), + (0xFF43, 0x1, 0xFF23), + (0xFF44, 0x1, 0xFF24), + (0xFF45, 0x1, 0xFF25), + (0xFF46, 0x1, 0xFF26), + (0xFF47, 0x1, 0xFF27), + (0xFF48, 0x1, 0xFF28), + (0xFF49, 0x1, 0xFF29), + (0xFF4A, 0x1, 0xFF2A), + (0xFF4B, 0x1, 0xFF2B), + (0xFF4C, 0x1, 0xFF2C), + (0xFF4D, 0x1, 0xFF2D), + (0xFF4E, 0x1, 0xFF2E), + (0xFF4F, 0x1, 0xFF2F), + (0xFF50, 0x1, 0xFF30), + (0xFF51, 0x1, 0xFF31), + (0xFF52, 0x1, 0xFF32), + (0xFF53, 0x1, 0xFF33), + (0xFF54, 0x1, 0xFF34), + (0xFF55, 0x1, 0xFF35), + (0xFF56, 0x1, 0xFF36), + (0xFF57, 0x1, 0xFF37), + (0xFF58, 0x1, 0xFF38), + (0xFF59, 0x1, 0xFF39), + (0xFF5A, 0x1, 0xFF3A), + (0x10428, 0x1, 0x10400), + (0x10429, 0x1, 0x10401), + (0x1042A, 0x1, 0x10402), + (0x1042B, 0x1, 0x10403), + (0x1042C, 0x1, 0x10404), + (0x1042D, 0x1, 0x10405), + (0x1042E, 0x1, 0x10406), + (0x1042F, 0x1, 0x10407), + (0x10430, 0x1, 0x10408), + (0x10431, 0x1, 0x10409), + (0x10432, 0x1, 0x1040A), + (0x10433, 0x1, 0x1040B), + (0x10434, 0x1, 0x1040C), + (0x10435, 0x1, 0x1040D), + (0x10436, 0x1, 0x1040E), + (0x10437, 0x1, 0x1040F), + (0x10438, 0x1, 0x10410), + (0x10439, 0x1, 0x10411), + (0x1043A, 0x1, 0x10412), + (0x1043B, 0x1, 0x10413), + (0x1043C, 0x1, 0x10414), + (0x1043D, 0x1, 0x10415), + (0x1043E, 0x1, 0x10416), + (0x1043F, 0x1, 0x10417), + (0x10440, 0x1, 0x10418), + (0x10441, 0x1, 0x10419), + (0x10442, 0x1, 0x1041A), + (0x10443, 0x1, 0x1041B), + (0x10444, 0x1, 0x1041C), + (0x10445, 0x1, 0x1041D), + (0x10446, 0x1, 0x1041E), + (0x10447, 0x1, 0x1041F), + (0x10448, 0x1, 0x10420), + (0x10449, 0x1, 0x10421), + (0x1044A, 0x1, 0x10422), + (0x1044B, 0x1, 0x10423), + (0x1044C, 0x1, 0x10424), + (0x1044D, 0x1, 0x10425), + (0x1044E, 0x1, 0x10426), + (0x1044F, 0x1, 0x10427), + (0x104D8, 0x1, 0x104B0), + (0x104D9, 0x1, 0x104B1), + (0x104DA, 0x1, 0x104B2), + (0x104DB, 0x1, 0x104B3), + (0x104DC, 0x1, 0x104B4), + (0x104DD, 0x1, 0x104B5), + (0x104DE, 0x1, 0x104B6), + (0x104DF, 0x1, 0x104B7), + (0x104E0, 0x1, 0x104B8), + (0x104E1, 0x1, 0x104B9), + (0x104E2, 0x1, 0x104BA), + (0x104E3, 0x1, 0x104BB), + (0x104E4, 0x1, 0x104BC), + (0x104E5, 0x1, 0x104BD), + (0x104E6, 0x1, 0x104BE), + (0x104E7, 0x1, 0x104BF), + (0x104E8, 0x1, 0x104C0), + (0x104E9, 0x1, 0x104C1), + (0x104EA, 0x1, 0x104C2), + (0x104EB, 0x1, 0x104C3), + (0x104EC, 0x1, 0x104C4), + (0x104ED, 0x1, 0x104C5), + (0x104EE, 0x1, 0x104C6), + (0x104EF, 0x1, 0x104C7), + (0x104F0, 0x1, 0x104C8), + (0x104F1, 0x1, 0x104C9), + (0x104F2, 0x1, 0x104CA), + (0x104F3, 0x1, 0x104CB), + (0x104F4, 0x1, 0x104CC), + (0x104F5, 0x1, 0x104CD), + (0x104F6, 0x1, 0x104CE), + (0x104F7, 0x1, 0x104CF), + (0x104F8, 0x1, 0x104D0), + (0x104F9, 0x1, 0x104D1), + (0x104FA, 0x1, 0x104D2), + (0x104FB, 0x1, 0x104D3), + (0x10597, 0x1, 0x10570), + (0x10598, 0x1, 0x10571), + (0x10599, 0x1, 0x10572), + (0x1059A, 0x1, 0x10573), + (0x1059B, 0x1, 0x10574), + (0x1059C, 0x1, 0x10575), + (0x1059D, 0x1, 0x10576), + (0x1059E, 0x1, 0x10577), + (0x1059F, 0x1, 0x10578), + (0x105A0, 0x1, 0x10579), + (0x105A1, 0x1, 0x1057A), + (0x105A3, 0x1, 0x1057C), + (0x105A4, 0x1, 0x1057D), + (0x105A5, 0x1, 0x1057E), + (0x105A6, 0x1, 0x1057F), + (0x105A7, 0x1, 0x10580), + (0x105A8, 0x1, 0x10581), + (0x105A9, 0x1, 0x10582), + (0x105AA, 0x1, 0x10583), + (0x105AB, 0x1, 0x10584), + (0x105AC, 0x1, 0x10585), + (0x105AD, 0x1, 0x10586), + (0x105AE, 0x1, 0x10587), + (0x105AF, 0x1, 0x10588), + (0x105B0, 0x1, 0x10589), + (0x105B1, 0x1, 0x1058A), + (0x105B3, 0x1, 0x1058C), + (0x105B4, 0x1, 0x1058D), + (0x105B5, 0x1, 0x1058E), + (0x105B6, 0x1, 0x1058F), + (0x105B7, 0x1, 0x10590), + (0x105B8, 0x1, 0x10591), + (0x105B9, 0x1, 0x10592), + (0x105BB, 0x1, 0x10594), + (0x105BC, 0x1, 0x10595), + (0x10CC0, 0x1, 0x10C80), + (0x10CC1, 0x1, 0x10C81), + (0x10CC2, 0x1, 0x10C82), + (0x10CC3, 0x1, 0x10C83), + (0x10CC4, 0x1, 0x10C84), + (0x10CC5, 0x1, 0x10C85), + (0x10CC6, 0x1, 0x10C86), + (0x10CC7, 0x1, 0x10C87), + (0x10CC8, 0x1, 0x10C88), + (0x10CC9, 0x1, 0x10C89), + (0x10CCA, 0x1, 0x10C8A), + (0x10CCB, 0x1, 0x10C8B), + (0x10CCC, 0x1, 0x10C8C), + (0x10CCD, 0x1, 0x10C8D), + (0x10CCE, 0x1, 0x10C8E), + (0x10CCF, 0x1, 0x10C8F), + (0x10CD0, 0x1, 0x10C90), + (0x10CD1, 0x1, 0x10C91), + (0x10CD2, 0x1, 0x10C92), + (0x10CD3, 0x1, 0x10C93), + (0x10CD4, 0x1, 0x10C94), + (0x10CD5, 0x1, 0x10C95), + (0x10CD6, 0x1, 0x10C96), + (0x10CD7, 0x1, 0x10C97), + (0x10CD8, 0x1, 0x10C98), + (0x10CD9, 0x1, 0x10C99), + (0x10CDA, 0x1, 0x10C9A), + (0x10CDB, 0x1, 0x10C9B), + (0x10CDC, 0x1, 0x10C9C), + (0x10CDD, 0x1, 0x10C9D), + (0x10CDE, 0x1, 0x10C9E), + (0x10CDF, 0x1, 0x10C9F), + (0x10CE0, 0x1, 0x10CA0), + (0x10CE1, 0x1, 0x10CA1), + (0x10CE2, 0x1, 0x10CA2), + (0x10CE3, 0x1, 0x10CA3), + (0x10CE4, 0x1, 0x10CA4), + (0x10CE5, 0x1, 0x10CA5), + (0x10CE6, 0x1, 0x10CA6), + (0x10CE7, 0x1, 0x10CA7), + (0x10CE8, 0x1, 0x10CA8), + (0x10CE9, 0x1, 0x10CA9), + (0x10CEA, 0x1, 0x10CAA), + (0x10CEB, 0x1, 0x10CAB), + (0x10CEC, 0x1, 0x10CAC), + (0x10CED, 0x1, 0x10CAD), + (0x10CEE, 0x1, 0x10CAE), + (0x10CEF, 0x1, 0x10CAF), + (0x10CF0, 0x1, 0x10CB0), + (0x10CF1, 0x1, 0x10CB1), + (0x10CF2, 0x1, 0x10CB2), + (0x118C0, 0x1, 0x118A0), + (0x118C1, 0x1, 0x118A1), + (0x118C2, 0x1, 0x118A2), + (0x118C3, 0x1, 0x118A3), + (0x118C4, 0x1, 0x118A4), + (0x118C5, 0x1, 0x118A5), + (0x118C6, 0x1, 0x118A6), + (0x118C7, 0x1, 0x118A7), + (0x118C8, 0x1, 0x118A8), + (0x118C9, 0x1, 0x118A9), + (0x118CA, 0x1, 0x118AA), + (0x118CB, 0x1, 0x118AB), + (0x118CC, 0x1, 0x118AC), + (0x118CD, 0x1, 0x118AD), + (0x118CE, 0x1, 0x118AE), + (0x118CF, 0x1, 0x118AF), + (0x118D0, 0x1, 0x118B0), + (0x118D1, 0x1, 0x118B1), + (0x118D2, 0x1, 0x118B2), + (0x118D3, 0x1, 0x118B3), + (0x118D4, 0x1, 0x118B4), + (0x118D5, 0x1, 0x118B5), + (0x118D6, 0x1, 0x118B6), + (0x118D7, 0x1, 0x118B7), + (0x118D8, 0x1, 0x118B8), + (0x118D9, 0x1, 0x118B9), + (0x118DA, 0x1, 0x118BA), + (0x118DB, 0x1, 0x118BB), + (0x118DC, 0x1, 0x118BC), + (0x118DD, 0x1, 0x118BD), + (0x118DE, 0x1, 0x118BE), + (0x118DF, 0x1, 0x118BF), + (0x16E60, 0x1, 0x16E40), + (0x16E61, 0x1, 0x16E41), + (0x16E62, 0x1, 0x16E42), + (0x16E63, 0x1, 0x16E43), + (0x16E64, 0x1, 0x16E44), + (0x16E65, 0x1, 0x16E45), + (0x16E66, 0x1, 0x16E46), + (0x16E67, 0x1, 0x16E47), + (0x16E68, 0x1, 0x16E48), + (0x16E69, 0x1, 0x16E49), + (0x16E6A, 0x1, 0x16E4A), + (0x16E6B, 0x1, 0x16E4B), + (0x16E6C, 0x1, 0x16E4C), + (0x16E6D, 0x1, 0x16E4D), + (0x16E6E, 0x1, 0x16E4E), + (0x16E6F, 0x1, 0x16E4F), + (0x16E70, 0x1, 0x16E50), + (0x16E71, 0x1, 0x16E51), + (0x16E72, 0x1, 0x16E52), + (0x16E73, 0x1, 0x16E53), + (0x16E74, 0x1, 0x16E54), + (0x16E75, 0x1, 0x16E55), + (0x16E76, 0x1, 0x16E56), + (0x16E77, 0x1, 0x16E57), + (0x16E78, 0x1, 0x16E58), + (0x16E79, 0x1, 0x16E59), + (0x16E7A, 0x1, 0x16E5A), + (0x16E7B, 0x1, 0x16E5B), + (0x16E7C, 0x1, 0x16E5C), + (0x16E7D, 0x1, 0x16E5D), + (0x16E7E, 0x1, 0x16E5E), + (0x16E7F, 0x1, 0x16E5F), + (0x1E922, 0x1, 0x1E900), + (0x1E923, 0x1, 0x1E901), + (0x1E924, 0x1, 0x1E902), + (0x1E925, 0x1, 0x1E903), + (0x1E926, 0x1, 0x1E904), + (0x1E927, 0x1, 0x1E905), + (0x1E928, 0x1, 0x1E906), + (0x1E929, 0x1, 0x1E907), + (0x1E92A, 0x1, 0x1E908), + (0x1E92B, 0x1, 0x1E909), + (0x1E92C, 0x1, 0x1E90A), + (0x1E92D, 0x1, 0x1E90B), + (0x1E92E, 0x1, 0x1E90C), + (0x1E92F, 0x1, 0x1E90D), + (0x1E930, 0x1, 0x1E90E), + (0x1E931, 0x1, 0x1E90F), + (0x1E932, 0x1, 0x1E910), + (0x1E933, 0x1, 0x1E911), + (0x1E934, 0x1, 0x1E912), + (0x1E935, 0x1, 0x1E913), + (0x1E936, 0x1, 0x1E914), + (0x1E937, 0x1, 0x1E915), + (0x1E938, 0x1, 0x1E916), + (0x1E939, 0x1, 0x1E917), + (0x1E93A, 0x1, 0x1E918), + (0x1E93B, 0x1, 0x1E919), + (0x1E93C, 0x1, 0x1E91A), + (0x1E93D, 0x1, 0x1E91B), + (0x1E93E, 0x1, 0x1E91C), + (0x1E93F, 0x1, 0x1E91D), + (0x1E940, 0x1, 0x1E91E), + (0x1E941, 0x1, 0x1E91F), + (0x1E942, 0x1, 0x1E920), + (0x1E943, 0x1, 0x1E921), +) + +LOWER_FINAL_SIGMA_ROWS = ( + (0x3A3, 0x1, 0x3C2), +) + +CONTEXT_RANGES = ( + (0x27, 0x27, 0x2), + (0x2E, 0x2E, 0x2), + (0x30, 0x39, 0x4), + (0x3A, 0x3A, 0x2), + (0x41, 0x5A, 0x1), + (0x5E, 0x5E, 0x2), + (0x60, 0x60, 0x2), + (0x61, 0x7A, 0x1), + (0xA8, 0xA8, 0x2), + (0xAA, 0xAA, 0x1), + (0xAD, 0xAD, 0x2), + (0xAF, 0xAF, 0x2), + (0xB4, 0xB4, 0x2), + (0xB5, 0xB5, 0x1), + (0xB7, 0xB8, 0x2), + (0xBA, 0xBA, 0x1), + (0xC0, 0xD6, 0x1), + (0xD8, 0xF6, 0x1), + (0xF8, 0x1BA, 0x1), + (0x1BC, 0x1BF, 0x1), + (0x1C4, 0x293, 0x1), + (0x295, 0x2AF, 0x1), + (0x2B0, 0x2B8, 0x3), + (0x2B9, 0x2BF, 0x2), + (0x2C0, 0x2C1, 0x3), + (0x2C2, 0x2DF, 0x2), + (0x2E0, 0x2E4, 0x3), + (0x2E5, 0x344, 0x2), + (0x345, 0x345, 0x3), + (0x346, 0x36F, 0x2), + (0x370, 0x373, 0x1), + (0x374, 0x375, 0x2), + (0x376, 0x377, 0x1), + (0x37A, 0x37A, 0x3), + (0x37B, 0x37D, 0x1), + (0x37F, 0x37F, 0x1), + (0x384, 0x385, 0x2), + (0x386, 0x386, 0x1), + (0x387, 0x387, 0x2), + (0x388, 0x38A, 0x1), + (0x38C, 0x38C, 0x1), + (0x38E, 0x3A1, 0x1), + (0x3A3, 0x3F5, 0x1), + (0x3F7, 0x481, 0x1), + (0x483, 0x489, 0x2), + (0x48A, 0x52F, 0x1), + (0x531, 0x556, 0x1), + (0x559, 0x559, 0x2), + (0x55F, 0x55F, 0x2), + (0x560, 0x588, 0x1), + (0x591, 0x5BD, 0x2), + (0x5BF, 0x5BF, 0x2), + (0x5C1, 0x5C2, 0x2), + (0x5C4, 0x5C5, 0x2), + (0x5C7, 0x5C7, 0x2), + (0x5F4, 0x5F4, 0x2), + (0x600, 0x605, 0x2), + (0x610, 0x61A, 0x2), + (0x61C, 0x61C, 0x2), + (0x640, 0x640, 0x2), + (0x64B, 0x65F, 0x2), + (0x660, 0x669, 0x4), + (0x670, 0x670, 0x2), + (0x6D6, 0x6DD, 0x2), + (0x6DF, 0x6E8, 0x2), + (0x6EA, 0x6ED, 0x2), + (0x6F0, 0x6F9, 0x4), + (0x70F, 0x70F, 0x2), + (0x711, 0x711, 0x2), + (0x730, 0x74A, 0x2), + (0x7A6, 0x7B0, 0x2), + (0x7C0, 0x7C9, 0x4), + (0x7EB, 0x7F5, 0x2), + (0x7FA, 0x7FA, 0x2), + (0x7FD, 0x7FD, 0x2), + (0x816, 0x82D, 0x2), + (0x859, 0x85B, 0x2), + (0x888, 0x888, 0x2), + (0x890, 0x891, 0x2), + (0x898, 0x89F, 0x2), + (0x8C9, 0x902, 0x2), + (0x93A, 0x93A, 0x2), + (0x93C, 0x93C, 0x2), + (0x941, 0x948, 0x2), + (0x94D, 0x94D, 0x2), + (0x951, 0x957, 0x2), + (0x962, 0x963, 0x2), + (0x966, 0x96F, 0x4), + (0x971, 0x971, 0x2), + (0x981, 0x981, 0x2), + (0x9BC, 0x9BC, 0x2), + (0x9C1, 0x9C4, 0x2), + (0x9CD, 0x9CD, 0x2), + (0x9E2, 0x9E3, 0x2), + (0x9E6, 0x9EF, 0x4), + (0x9FE, 0x9FE, 0x2), + (0xA01, 0xA02, 0x2), + (0xA3C, 0xA3C, 0x2), + (0xA41, 0xA42, 0x2), + (0xA47, 0xA48, 0x2), + (0xA4B, 0xA4D, 0x2), + (0xA51, 0xA51, 0x2), + (0xA66, 0xA6F, 0x4), + (0xA70, 0xA71, 0x2), + (0xA75, 0xA75, 0x2), + (0xA81, 0xA82, 0x2), + (0xABC, 0xABC, 0x2), + (0xAC1, 0xAC5, 0x2), + (0xAC7, 0xAC8, 0x2), + (0xACD, 0xACD, 0x2), + (0xAE2, 0xAE3, 0x2), + (0xAE6, 0xAEF, 0x4), + (0xAFA, 0xAFF, 0x2), + (0xB01, 0xB01, 0x2), + (0xB3C, 0xB3C, 0x2), + (0xB3F, 0xB3F, 0x2), + (0xB41, 0xB44, 0x2), + (0xB4D, 0xB4D, 0x2), + (0xB55, 0xB56, 0x2), + (0xB62, 0xB63, 0x2), + (0xB66, 0xB6F, 0x4), + (0xB82, 0xB82, 0x2), + (0xBC0, 0xBC0, 0x2), + (0xBCD, 0xBCD, 0x2), + (0xBE6, 0xBEF, 0x4), + (0xC00, 0xC00, 0x2), + (0xC04, 0xC04, 0x2), + (0xC3C, 0xC3C, 0x2), + (0xC3E, 0xC40, 0x2), + (0xC46, 0xC48, 0x2), + (0xC4A, 0xC4D, 0x2), + (0xC55, 0xC56, 0x2), + (0xC62, 0xC63, 0x2), + (0xC66, 0xC6F, 0x4), + (0xC81, 0xC81, 0x2), + (0xCBC, 0xCBC, 0x2), + (0xCBF, 0xCBF, 0x2), + (0xCC6, 0xCC6, 0x2), + (0xCCC, 0xCCD, 0x2), + (0xCE2, 0xCE3, 0x2), + (0xCE6, 0xCEF, 0x4), + (0xD00, 0xD01, 0x2), + (0xD3B, 0xD3C, 0x2), + (0xD41, 0xD44, 0x2), + (0xD4D, 0xD4D, 0x2), + (0xD62, 0xD63, 0x2), + (0xD66, 0xD6F, 0x4), + (0xD81, 0xD81, 0x2), + (0xDCA, 0xDCA, 0x2), + (0xDD2, 0xDD4, 0x2), + (0xDD6, 0xDD6, 0x2), + (0xDE6, 0xDEF, 0x4), + (0xE31, 0xE31, 0x2), + (0xE34, 0xE3A, 0x2), + (0xE46, 0xE4E, 0x2), + (0xE50, 0xE59, 0x4), + (0xEB1, 0xEB1, 0x2), + (0xEB4, 0xEBC, 0x2), + (0xEC6, 0xEC6, 0x2), + (0xEC8, 0xECD, 0x2), + (0xED0, 0xED9, 0x4), + (0xF18, 0xF19, 0x2), + (0xF20, 0xF29, 0x4), + (0xF35, 0xF35, 0x2), + (0xF37, 0xF37, 0x2), + (0xF39, 0xF39, 0x2), + (0xF71, 0xF7E, 0x2), + (0xF80, 0xF84, 0x2), + (0xF86, 0xF87, 0x2), + (0xF8D, 0xF97, 0x2), + (0xF99, 0xFBC, 0x2), + (0xFC6, 0xFC6, 0x2), + (0x102D, 0x1030, 0x2), + (0x1032, 0x1037, 0x2), + (0x1039, 0x103A, 0x2), + (0x103D, 0x103E, 0x2), + (0x1040, 0x1049, 0x4), + (0x1058, 0x1059, 0x2), + (0x105E, 0x1060, 0x2), + (0x1071, 0x1074, 0x2), + (0x1082, 0x1082, 0x2), + (0x1085, 0x1086, 0x2), + (0x108D, 0x108D, 0x2), + (0x1090, 0x1099, 0x4), + (0x109D, 0x109D, 0x2), + (0x10A0, 0x10C5, 0x1), + (0x10C7, 0x10C7, 0x1), + (0x10CD, 0x10CD, 0x1), + (0x10D0, 0x10FA, 0x1), + (0x10FC, 0x10FC, 0x2), + (0x10FD, 0x10FF, 0x1), + (0x135D, 0x135F, 0x2), + (0x13A0, 0x13F5, 0x1), + (0x13F8, 0x13FD, 0x1), + (0x1712, 0x1714, 0x2), + (0x1732, 0x1733, 0x2), + (0x1752, 0x1753, 0x2), + (0x1772, 0x1773, 0x2), + (0x17B4, 0x17B5, 0x2), + (0x17B7, 0x17BD, 0x2), + (0x17C6, 0x17C6, 0x2), + (0x17C9, 0x17D3, 0x2), + (0x17D7, 0x17D7, 0x2), + (0x17DD, 0x17DD, 0x2), + (0x17E0, 0x17E9, 0x4), + (0x180B, 0x180F, 0x2), + (0x1810, 0x1819, 0x4), + (0x1843, 0x1843, 0x2), + (0x1885, 0x1886, 0x2), + (0x18A9, 0x18A9, 0x2), + (0x1920, 0x1922, 0x2), + (0x1927, 0x1928, 0x2), + (0x1932, 0x1932, 0x2), + (0x1939, 0x193B, 0x2), + (0x1946, 0x194F, 0x4), + (0x19D0, 0x19D9, 0x4), + (0x1A17, 0x1A18, 0x2), + (0x1A1B, 0x1A1B, 0x2), + (0x1A56, 0x1A56, 0x2), + (0x1A58, 0x1A5E, 0x2), + (0x1A60, 0x1A60, 0x2), + (0x1A62, 0x1A62, 0x2), + (0x1A65, 0x1A6C, 0x2), + (0x1A73, 0x1A7C, 0x2), + (0x1A7F, 0x1A7F, 0x2), + (0x1A80, 0x1A89, 0x4), + (0x1A90, 0x1A99, 0x4), + (0x1AA7, 0x1AA7, 0x2), + (0x1AB0, 0x1ACE, 0x2), + (0x1B00, 0x1B03, 0x2), + (0x1B34, 0x1B34, 0x2), + (0x1B36, 0x1B3A, 0x2), + (0x1B3C, 0x1B3C, 0x2), + (0x1B42, 0x1B42, 0x2), + (0x1B50, 0x1B59, 0x4), + (0x1B6B, 0x1B73, 0x2), + (0x1B80, 0x1B81, 0x2), + (0x1BA2, 0x1BA5, 0x2), + (0x1BA8, 0x1BA9, 0x2), + (0x1BAB, 0x1BAD, 0x2), + (0x1BB0, 0x1BB9, 0x4), + (0x1BE6, 0x1BE6, 0x2), + (0x1BE8, 0x1BE9, 0x2), + (0x1BED, 0x1BED, 0x2), + (0x1BEF, 0x1BF1, 0x2), + (0x1C2C, 0x1C33, 0x2), + (0x1C36, 0x1C37, 0x2), + (0x1C40, 0x1C49, 0x4), + (0x1C50, 0x1C59, 0x4), + (0x1C78, 0x1C7D, 0x2), + (0x1C80, 0x1C88, 0x1), + (0x1C90, 0x1CBA, 0x1), + (0x1CBD, 0x1CBF, 0x1), + (0x1CD0, 0x1CD2, 0x2), + (0x1CD4, 0x1CE0, 0x2), + (0x1CE2, 0x1CE8, 0x2), + (0x1CED, 0x1CED, 0x2), + (0x1CF4, 0x1CF4, 0x2), + (0x1CF8, 0x1CF9, 0x2), + (0x1D00, 0x1D2B, 0x1), + (0x1D2C, 0x1D6A, 0x3), + (0x1D6B, 0x1D77, 0x1), + (0x1D78, 0x1D78, 0x3), + (0x1D79, 0x1D9A, 0x1), + (0x1D9B, 0x1DBF, 0x3), + (0x1DC0, 0x1DFF, 0x2), + (0x1E00, 0x1F15, 0x1), + (0x1F18, 0x1F1D, 0x1), + (0x1F20, 0x1F45, 0x1), + (0x1F48, 0x1F4D, 0x1), + (0x1F50, 0x1F57, 0x1), + (0x1F59, 0x1F59, 0x1), + (0x1F5B, 0x1F5B, 0x1), + (0x1F5D, 0x1F5D, 0x1), + (0x1F5F, 0x1F7D, 0x1), + (0x1F80, 0x1FB4, 0x1), + (0x1FB6, 0x1FBC, 0x1), + (0x1FBD, 0x1FBD, 0x2), + (0x1FBE, 0x1FBE, 0x1), + (0x1FBF, 0x1FC1, 0x2), + (0x1FC2, 0x1FC4, 0x1), + (0x1FC6, 0x1FCC, 0x1), + (0x1FCD, 0x1FCF, 0x2), + (0x1FD0, 0x1FD3, 0x1), + (0x1FD6, 0x1FDB, 0x1), + (0x1FDD, 0x1FDF, 0x2), + (0x1FE0, 0x1FEC, 0x1), + (0x1FED, 0x1FEF, 0x2), + (0x1FF2, 0x1FF4, 0x1), + (0x1FF6, 0x1FFC, 0x1), + (0x1FFD, 0x1FFE, 0x2), + (0x200B, 0x200F, 0x2), + (0x2018, 0x2019, 0x2), + (0x2024, 0x2024, 0x2), + (0x2027, 0x2027, 0x2), + (0x202A, 0x202E, 0x2), + (0x2060, 0x2064, 0x2), + (0x2066, 0x206F, 0x2), + (0x2071, 0x2071, 0x3), + (0x207F, 0x207F, 0x3), + (0x2090, 0x209C, 0x3), + (0x20D0, 0x20F0, 0x2), + (0x2102, 0x2102, 0x1), + (0x2107, 0x2107, 0x1), + (0x210A, 0x2113, 0x1), + (0x2115, 0x2115, 0x1), + (0x2119, 0x211D, 0x1), + (0x2124, 0x2124, 0x1), + (0x2126, 0x2126, 0x1), + (0x2128, 0x2128, 0x1), + (0x212A, 0x212D, 0x1), + (0x212F, 0x2134, 0x1), + (0x2139, 0x2139, 0x1), + (0x213C, 0x213F, 0x1), + (0x2145, 0x2149, 0x1), + (0x214E, 0x214E, 0x1), + (0x2160, 0x217F, 0x1), + (0x2183, 0x2184, 0x1), + (0x24B6, 0x24E9, 0x1), + (0x2C00, 0x2C7B, 0x1), + (0x2C7C, 0x2C7D, 0x3), + (0x2C7E, 0x2CE4, 0x1), + (0x2CEB, 0x2CEE, 0x1), + (0x2CEF, 0x2CF1, 0x2), + (0x2CF2, 0x2CF3, 0x1), + (0x2D00, 0x2D25, 0x1), + (0x2D27, 0x2D27, 0x1), + (0x2D2D, 0x2D2D, 0x1), + (0x2D6F, 0x2D6F, 0x2), + (0x2D7F, 0x2D7F, 0x2), + (0x2DE0, 0x2DFF, 0x2), + (0x2E2F, 0x2E2F, 0x2), + (0x3005, 0x3005, 0x2), + (0x302A, 0x302D, 0x2), + (0x3031, 0x3035, 0x2), + (0x303B, 0x303B, 0x2), + (0x3099, 0x309E, 0x2), + (0x30FC, 0x30FE, 0x2), + (0xA015, 0xA015, 0x2), + (0xA4F8, 0xA4FD, 0x2), + (0xA60C, 0xA60C, 0x2), + (0xA620, 0xA629, 0x4), + (0xA640, 0xA66D, 0x1), + (0xA66F, 0xA672, 0x2), + (0xA674, 0xA67D, 0x2), + (0xA67F, 0xA67F, 0x2), + (0xA680, 0xA69B, 0x1), + (0xA69C, 0xA69D, 0x3), + (0xA69E, 0xA69F, 0x2), + (0xA6F0, 0xA6F1, 0x2), + (0xA700, 0xA721, 0x2), + (0xA722, 0xA76F, 0x1), + (0xA770, 0xA770, 0x3), + (0xA771, 0xA787, 0x1), + (0xA788, 0xA78A, 0x2), + (0xA78B, 0xA78E, 0x1), + (0xA790, 0xA7CA, 0x1), + (0xA7D0, 0xA7D1, 0x1), + (0xA7D3, 0xA7D3, 0x1), + (0xA7D5, 0xA7D9, 0x1), + (0xA7F2, 0xA7F4, 0x2), + (0xA7F5, 0xA7F6, 0x1), + (0xA7F8, 0xA7F9, 0x3), + (0xA7FA, 0xA7FA, 0x1), + (0xA802, 0xA802, 0x2), + (0xA806, 0xA806, 0x2), + (0xA80B, 0xA80B, 0x2), + (0xA825, 0xA826, 0x2), + (0xA82C, 0xA82C, 0x2), + (0xA8C4, 0xA8C5, 0x2), + (0xA8D0, 0xA8D9, 0x4), + (0xA8E0, 0xA8F1, 0x2), + (0xA8FF, 0xA8FF, 0x2), + (0xA900, 0xA909, 0x4), + (0xA926, 0xA92D, 0x2), + (0xA947, 0xA951, 0x2), + (0xA980, 0xA982, 0x2), + (0xA9B3, 0xA9B3, 0x2), + (0xA9B6, 0xA9B9, 0x2), + (0xA9BC, 0xA9BD, 0x2), + (0xA9CF, 0xA9CF, 0x2), + (0xA9D0, 0xA9D9, 0x4), + (0xA9E5, 0xA9E6, 0x2), + (0xA9F0, 0xA9F9, 0x4), + (0xAA29, 0xAA2E, 0x2), + (0xAA31, 0xAA32, 0x2), + (0xAA35, 0xAA36, 0x2), + (0xAA43, 0xAA43, 0x2), + (0xAA4C, 0xAA4C, 0x2), + (0xAA50, 0xAA59, 0x4), + (0xAA70, 0xAA70, 0x2), + (0xAA7C, 0xAA7C, 0x2), + (0xAAB0, 0xAAB0, 0x2), + (0xAAB2, 0xAAB4, 0x2), + (0xAAB7, 0xAAB8, 0x2), + (0xAABE, 0xAABF, 0x2), + (0xAAC1, 0xAAC1, 0x2), + (0xAADD, 0xAADD, 0x2), + (0xAAEC, 0xAAED, 0x2), + (0xAAF3, 0xAAF4, 0x2), + (0xAAF6, 0xAAF6, 0x2), + (0xAB30, 0xAB5A, 0x1), + (0xAB5B, 0xAB5B, 0x2), + (0xAB5C, 0xAB5F, 0x3), + (0xAB60, 0xAB68, 0x1), + (0xAB69, 0xAB6B, 0x2), + (0xAB70, 0xABBF, 0x1), + (0xABE5, 0xABE5, 0x2), + (0xABE8, 0xABE8, 0x2), + (0xABED, 0xABED, 0x2), + (0xABF0, 0xABF9, 0x4), + (0xFB00, 0xFB06, 0x1), + (0xFB13, 0xFB17, 0x1), + (0xFB1E, 0xFB1E, 0x2), + (0xFBB2, 0xFBC2, 0x2), + (0xFE00, 0xFE0F, 0x2), + (0xFE13, 0xFE13, 0x2), + (0xFE20, 0xFE2F, 0x2), + (0xFE52, 0xFE52, 0x2), + (0xFE55, 0xFE55, 0x2), + (0xFEFF, 0xFEFF, 0x2), + (0xFF07, 0xFF07, 0x2), + (0xFF0E, 0xFF0E, 0x2), + (0xFF10, 0xFF19, 0x4), + (0xFF1A, 0xFF1A, 0x2), + (0xFF21, 0xFF3A, 0x1), + (0xFF3E, 0xFF3E, 0x2), + (0xFF40, 0xFF40, 0x2), + (0xFF41, 0xFF5A, 0x1), + (0xFF70, 0xFF70, 0x2), + (0xFF9E, 0xFF9F, 0x2), + (0xFFE3, 0xFFE3, 0x2), + (0xFFF9, 0xFFFB, 0x2), + (0x101FD, 0x101FD, 0x2), + (0x102E0, 0x102E0, 0x2), + (0x10376, 0x1037A, 0x2), + (0x10400, 0x1044F, 0x1), + (0x104A0, 0x104A9, 0x4), + (0x104B0, 0x104D3, 0x1), + (0x104D8, 0x104FB, 0x1), + (0x10570, 0x1057A, 0x1), + (0x1057C, 0x1058A, 0x1), + (0x1058C, 0x10592, 0x1), + (0x10594, 0x10595, 0x1), + (0x10597, 0x105A1, 0x1), + (0x105A3, 0x105B1, 0x1), + (0x105B3, 0x105B9, 0x1), + (0x105BB, 0x105BC, 0x1), + (0x10780, 0x10780, 0x3), + (0x10781, 0x10782, 0x2), + (0x10783, 0x10785, 0x3), + (0x10787, 0x107B0, 0x3), + (0x107B2, 0x107BA, 0x3), + (0x10A01, 0x10A03, 0x2), + (0x10A05, 0x10A06, 0x2), + (0x10A0C, 0x10A0F, 0x2), + (0x10A38, 0x10A3A, 0x2), + (0x10A3F, 0x10A3F, 0x2), + (0x10AE5, 0x10AE6, 0x2), + (0x10C80, 0x10CB2, 0x1), + (0x10CC0, 0x10CF2, 0x1), + (0x10D24, 0x10D27, 0x2), + (0x10D30, 0x10D39, 0x4), + (0x10EAB, 0x10EAC, 0x2), + (0x10F46, 0x10F50, 0x2), + (0x10F82, 0x10F85, 0x2), + (0x11001, 0x11001, 0x2), + (0x11038, 0x11046, 0x2), + (0x11066, 0x1106F, 0x4), + (0x11070, 0x11070, 0x2), + (0x11073, 0x11074, 0x2), + (0x1107F, 0x11081, 0x2), + (0x110B3, 0x110B6, 0x2), + (0x110B9, 0x110BA, 0x2), + (0x110BD, 0x110BD, 0x2), + (0x110C2, 0x110C2, 0x2), + (0x110CD, 0x110CD, 0x2), + (0x110F0, 0x110F9, 0x4), + (0x11100, 0x11102, 0x2), + (0x11127, 0x1112B, 0x2), + (0x1112D, 0x11134, 0x2), + (0x11136, 0x1113F, 0x4), + (0x11173, 0x11173, 0x2), + (0x11180, 0x11181, 0x2), + (0x111B6, 0x111BE, 0x2), + (0x111C9, 0x111CC, 0x2), + (0x111CF, 0x111CF, 0x2), + (0x111D0, 0x111D9, 0x4), + (0x1122F, 0x11231, 0x2), + (0x11234, 0x11234, 0x2), + (0x11236, 0x11237, 0x2), + (0x1123E, 0x1123E, 0x2), + (0x112DF, 0x112DF, 0x2), + (0x112E3, 0x112EA, 0x2), + (0x112F0, 0x112F9, 0x4), + (0x11300, 0x11301, 0x2), + (0x1133B, 0x1133C, 0x2), + (0x11340, 0x11340, 0x2), + (0x11366, 0x1136C, 0x2), + (0x11370, 0x11374, 0x2), + (0x11438, 0x1143F, 0x2), + (0x11442, 0x11444, 0x2), + (0x11446, 0x11446, 0x2), + (0x11450, 0x11459, 0x4), + (0x1145E, 0x1145E, 0x2), + (0x114B3, 0x114B8, 0x2), + (0x114BA, 0x114BA, 0x2), + (0x114BF, 0x114C0, 0x2), + (0x114C2, 0x114C3, 0x2), + (0x114D0, 0x114D9, 0x4), + (0x115B2, 0x115B5, 0x2), + (0x115BC, 0x115BD, 0x2), + (0x115BF, 0x115C0, 0x2), + (0x115DC, 0x115DD, 0x2), + (0x11633, 0x1163A, 0x2), + (0x1163D, 0x1163D, 0x2), + (0x1163F, 0x11640, 0x2), + (0x11650, 0x11659, 0x4), + (0x116AB, 0x116AB, 0x2), + (0x116AD, 0x116AD, 0x2), + (0x116B0, 0x116B5, 0x2), + (0x116B7, 0x116B7, 0x2), + (0x116C0, 0x116C9, 0x4), + (0x1171D, 0x1171F, 0x2), + (0x11722, 0x11725, 0x2), + (0x11727, 0x1172B, 0x2), + (0x11730, 0x11739, 0x4), + (0x1182F, 0x11837, 0x2), + (0x11839, 0x1183A, 0x2), + (0x118A0, 0x118DF, 0x1), + (0x118E0, 0x118E9, 0x4), + (0x1193B, 0x1193C, 0x2), + (0x1193E, 0x1193E, 0x2), + (0x11943, 0x11943, 0x2), + (0x11950, 0x11959, 0x4), + (0x119D4, 0x119D7, 0x2), + (0x119DA, 0x119DB, 0x2), + (0x119E0, 0x119E0, 0x2), + (0x11A01, 0x11A0A, 0x2), + (0x11A33, 0x11A38, 0x2), + (0x11A3B, 0x11A3E, 0x2), + (0x11A47, 0x11A47, 0x2), + (0x11A51, 0x11A56, 0x2), + (0x11A59, 0x11A5B, 0x2), + (0x11A8A, 0x11A96, 0x2), + (0x11A98, 0x11A99, 0x2), + (0x11C30, 0x11C36, 0x2), + (0x11C38, 0x11C3D, 0x2), + (0x11C3F, 0x11C3F, 0x2), + (0x11C50, 0x11C59, 0x4), + (0x11C92, 0x11CA7, 0x2), + (0x11CAA, 0x11CB0, 0x2), + (0x11CB2, 0x11CB3, 0x2), + (0x11CB5, 0x11CB6, 0x2), + (0x11D31, 0x11D36, 0x2), + (0x11D3A, 0x11D3A, 0x2), + (0x11D3C, 0x11D3D, 0x2), + (0x11D3F, 0x11D45, 0x2), + (0x11D47, 0x11D47, 0x2), + (0x11D50, 0x11D59, 0x4), + (0x11D90, 0x11D91, 0x2), + (0x11D95, 0x11D95, 0x2), + (0x11D97, 0x11D97, 0x2), + (0x11DA0, 0x11DA9, 0x4), + (0x11EF3, 0x11EF4, 0x2), + (0x13430, 0x13438, 0x2), + (0x16A60, 0x16A69, 0x4), + (0x16AC0, 0x16AC9, 0x4), + (0x16AF0, 0x16AF4, 0x2), + (0x16B30, 0x16B36, 0x2), + (0x16B40, 0x16B43, 0x2), + (0x16B50, 0x16B59, 0x4), + (0x16E40, 0x16E7F, 0x1), + (0x16F4F, 0x16F4F, 0x2), + (0x16F8F, 0x16F9F, 0x2), + (0x16FE0, 0x16FE1, 0x2), + (0x16FE3, 0x16FE4, 0x2), + (0x1AFF0, 0x1AFF3, 0x2), + (0x1AFF5, 0x1AFFB, 0x2), + (0x1AFFD, 0x1AFFE, 0x2), + (0x1BC9D, 0x1BC9E, 0x2), + (0x1BCA0, 0x1BCA3, 0x2), + (0x1CF00, 0x1CF2D, 0x2), + (0x1CF30, 0x1CF46, 0x2), + (0x1D167, 0x1D169, 0x2), + (0x1D173, 0x1D182, 0x2), + (0x1D185, 0x1D18B, 0x2), + (0x1D1AA, 0x1D1AD, 0x2), + (0x1D242, 0x1D244, 0x2), + (0x1D400, 0x1D454, 0x1), + (0x1D456, 0x1D49C, 0x1), + (0x1D49E, 0x1D49F, 0x1), + (0x1D4A2, 0x1D4A2, 0x1), + (0x1D4A5, 0x1D4A6, 0x1), + (0x1D4A9, 0x1D4AC, 0x1), + (0x1D4AE, 0x1D4B9, 0x1), + (0x1D4BB, 0x1D4BB, 0x1), + (0x1D4BD, 0x1D4C3, 0x1), + (0x1D4C5, 0x1D505, 0x1), + (0x1D507, 0x1D50A, 0x1), + (0x1D50D, 0x1D514, 0x1), + (0x1D516, 0x1D51C, 0x1), + (0x1D51E, 0x1D539, 0x1), + (0x1D53B, 0x1D53E, 0x1), + (0x1D540, 0x1D544, 0x1), + (0x1D546, 0x1D546, 0x1), + (0x1D54A, 0x1D550, 0x1), + (0x1D552, 0x1D6A5, 0x1), + (0x1D6A8, 0x1D6C0, 0x1), + (0x1D6C2, 0x1D6DA, 0x1), + (0x1D6DC, 0x1D6FA, 0x1), + (0x1D6FC, 0x1D714, 0x1), + (0x1D716, 0x1D734, 0x1), + (0x1D736, 0x1D74E, 0x1), + (0x1D750, 0x1D76E, 0x1), + (0x1D770, 0x1D788, 0x1), + (0x1D78A, 0x1D7A8, 0x1), + (0x1D7AA, 0x1D7C2, 0x1), + (0x1D7C4, 0x1D7CB, 0x1), + (0x1D7CE, 0x1D7FF, 0x4), + (0x1DA00, 0x1DA36, 0x2), + (0x1DA3B, 0x1DA6C, 0x2), + (0x1DA75, 0x1DA75, 0x2), + (0x1DA84, 0x1DA84, 0x2), + (0x1DA9B, 0x1DA9F, 0x2), + (0x1DAA1, 0x1DAAF, 0x2), + (0x1DF00, 0x1DF09, 0x1), + (0x1DF0B, 0x1DF1E, 0x1), + (0x1E000, 0x1E006, 0x2), + (0x1E008, 0x1E018, 0x2), + (0x1E01B, 0x1E021, 0x2), + (0x1E023, 0x1E024, 0x2), + (0x1E026, 0x1E02A, 0x2), + (0x1E130, 0x1E13D, 0x2), + (0x1E140, 0x1E149, 0x4), + (0x1E2AE, 0x1E2AE, 0x2), + (0x1E2EC, 0x1E2EF, 0x2), + (0x1E2F0, 0x1E2F9, 0x4), + (0x1E8D0, 0x1E8D6, 0x2), + (0x1E900, 0x1E943, 0x1), + (0x1E944, 0x1E94B, 0x2), + (0x1E950, 0x1E959, 0x4), + (0x1F130, 0x1F149, 0x1), + (0x1F150, 0x1F169, 0x1), + (0x1F170, 0x1F189, 0x1), + (0x1F3FB, 0x1F3FF, 0x2), + (0x1FBF0, 0x1FBF9, 0x4), + (0xE0001, 0xE0001, 0x2), + (0xE0020, 0xE007F, 0x2), + (0xE0100, 0xE01EF, 0x2), +) diff --git a/pineforge_codegen/codegen/helpers.py b/pineforge_codegen/codegen/helpers.py index cf65dc1e..d3938cda 100644 --- a/pineforge_codegen/codegen/helpers.py +++ b/pineforge_codegen/codegen/helpers.py @@ -554,6 +554,7 @@ def cpp_code_only(cpp: str) -> str: _pf_case_next_cased _pf_case_i _pf_case_props _pf_case_before _pf_case_lead _pf_case_width _pf_case_valid _pf_case_k _pf_case_byte _pf_case_min _pf_case_out _pf_case_append _pf_toupper_map _pf_tolower_map + _pf_tolower_final_map _pf_case_contextual _pf_case_row _pf_trim_text _pf_trim_size _pf_trim_first _pf_trim_last _pf_trim_i _pf_trim_c _pf_trim_width _pf_trim_second _pf_trim_third _pf_array_new_size _pf_array_new_value _pf_array_new_size_type _pf_array_new_error diff --git a/pineforge_codegen/codegen/unicode_case.py b/pineforge_codegen/codegen/unicode_case.py index 45bdba4a..1137157e 100644 --- a/pineforge_codegen/codegen/unicode_case.py +++ b/pineforge_codegen/codegen/unicode_case.py @@ -1,48 +1,76 @@ -"""Emit Unicode default casing over runtime UTF-8 strings. +"""Emit Unicode casing over runtime UTF-8 strings. -Generate complete, sorted case tables from Python's standard Unicode database, -including expansions and supplementary letters. The emitted comment records -that database's version; the compiled strategy needs no Unicode library. -Locale-specific tailoring and normalization are not default casing. +Mappings and properties come only from the generated, checked-in Unicode +14.0.0 data module. scripts/gen_unicode_case_data.py binds the three exact +official UCD inputs; no host Unicode database or locale supplies data. +The data pin identifies this implementation, not TradingView's whole dialect. +Locale-specific casing and normalization are not applied. -Default lowercase has one contextual rule, Final_Sigma. Cased is the union of -Lowercase, Uppercase and General_Category=Lt. Obtain Case_Ignorable from the -same database through that rule: after A+SIGMA, a cased character is ignorable -iff SIGMA stays final; an uncased character is ignorable iff following it with -A makes SIGMA nonfinal. Ignoring takes precedence for overlapping properties. +DerivedCoreProperties supplies Cased and Case_Ignorable. UnicodeData supplies +Nd. SpecialCasing supplies full expansions and the Final_Sigma override. +Ignoring takes precedence for overlapping properties. The cased context also +crosses decimal-number runs without making digits cased or case-ignorable. The captureless lambda receives its argument outside its local scope, once. -Adjacent WTF-8 high/low surrogates form one code point before casing, just as -they do in UTF-16. NUL, isolated surrogate pieces and malformed bytes survive. +Adjacent WTF-8 high/low surrogates form one code point before casing. NUL, +isolated surrogate pieces and malformed bytes survive. """ from __future__ import annotations from functools import lru_cache -import unicodedata + + +_DATA_VERSION = "14.0.0" +_SOURCE_SHA256 = ( + ("UnicodeData.txt", "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +_SOURCE_BYTE_COUNTS = ( + ("UnicodeData.txt", 1897793), + ("SpecialCasing.txt", 16830), + ("DerivedCoreProperties.txt", 1040794), +) @lru_cache(maxsize=1) -def _context_ranges() -> tuple[tuple[int, int, int], ...]: - ranges: list[tuple[int, int, int]] = [] - for code in range(0x110000): - char = chr(code) - cased = char.islower() or char.isupper() or char.istitle() - if cased: - ignorable = ("A\u03a3" + char).lower()[1] == "\u03c2" +def _case_data(): + # Keep unrelated translation paths importable while regenerating data. + # A missing/mismatched module is a build defect, never a host fallback. + from . import _unicode_case_data as data + + if (data.DATA_FORMAT != 1 or data.DATA_VERSION != _DATA_VERSION + or data.SOURCE_SHA256 != _SOURCE_SHA256 + or data.SOURCE_BYTE_COUNTS != _SOURCE_BYTE_COUNTS): + raise RuntimeError("regenerate Unicode casing data from the pinned UCD inputs") + return data + + +def _context_flags(char: str) -> int: + point = ord(char) + ranges = _context_ranges() + low, high = 0, len(ranges) + while low < high: + mid = low + (high - low) // 2 + if ranges[mid][1] < point: + low = mid + 1 else: - ignorable = ("A\u03a3" + char + "A").lower()[1] == "\u03c3" - flags = int(cased) | (int(ignorable) << 1) - if flags: - if ranges and ranges[-1][1] + 1 == code and ranges[-1][2] == flags: - start, _, _ = ranges[-1] - ranges[-1] = (start, code, flags) - else: - ranges.append((code, code, flags)) - return tuple(ranges) + high = mid + if low < len(ranges) and ranges[low][0] <= point: + return ranges[low][2] + return 0 + + +@lru_cache(maxsize=1) +def _context_ranges() -> tuple[tuple[int, int, int], ...]: + return _case_data().CONTEXT_RANGES _CONTEXT_CPP = r""" + static constexpr uint32_t _pf_tolower_final_map[][@FINAL_WIDTH@] = { +@FINAL_MAPPINGS@ + }; static constexpr uint32_t _pf_case_properties[][3] = { @PROPERTIES@ }; @@ -67,24 +95,33 @@ def _context_ranges() -> tuple[tuple[int, int, int], ...]: --_pf_case_i; _pf_case_after[_pf_case_i] = static_cast(_pf_case_next_cased); const unsigned _pf_case_props = _pf_case_flags(_pf_case_codes[_pf_case_i]); - if (!(_pf_case_props & 2u)) _pf_case_next_cased = (_pf_case_props & 1u) != 0; + if (!(_pf_case_props & 6u)) _pf_case_next_cased = (_pf_case_props & 1u) != 0; } bool _pf_case_before = false; """ _CONTEXT_SELECT_CPP = r""" - if (_pf_case_code == 0x3A3u && _pf_case_before && !_pf_case_after[_pf_case_i]) { - _pf_case_append(0x3C2u); - } else + bool _pf_case_contextual = false; + if (_pf_case_before && !_pf_case_after[_pf_case_i]) { + for (const auto& _pf_case_row : _pf_tolower_final_map) { + if (_pf_case_row[0] != _pf_case_code) continue; + for (uint32_t _pf_case_k = 0; _pf_case_k < _pf_case_row[1]; ++_pf_case_k) + _pf_case_append(_pf_case_row[_pf_case_k + 2]); + _pf_case_contextual = true; + break; + } + } + if (!_pf_case_contextual) """ _CONTEXT_ADVANCE_CPP = r""" const unsigned _pf_case_props = _pf_case_flags(_pf_case_code); - if (!(_pf_case_props & 2u)) _pf_case_before = (_pf_case_props & 1u) != 0; + if (!(_pf_case_props & 6u)) _pf_case_before = (_pf_case_props & 1u) != 0; """ _BODY_CPP = r"""[](const std::string& _pf_case_text) { - // Unicode @VERSION@ default @DIRECTION@, independent of the C locale. + // Unicode @VERSION@ casing, independent of the C locale. +@PROVENANCE@ static constexpr uint32_t @TABLE@[][@WIDTH@] = { @MAPPINGS@ }; @@ -110,8 +147,6 @@ def _context_ranges() -> tuple[tuple[int, int, int], ...]: : _pf_case_width == 2 ? 0x80u : _pf_case_width == 3 ? 0x800u : 0x10000u; _pf_case_valid = _pf_case_valid && _pf_case_code >= _pf_case_min && _pf_case_code <= 0x10FFFFu; } - // WTF-8 encodes isolated surrogate units as three bytes; keep them. - // Preserve each malformed byte with a value outside the Unicode range. if (!_pf_case_valid) { _pf_case_code = 0x110000u + _pf_case_lead; _pf_case_width = 1; @@ -175,13 +210,8 @@ def _context_ranges() -> tuple[tuple[int, int, int], ...]: @lru_cache(maxsize=2) def _case_lambda(upper: bool) -> str: - mapping = str.upper if upper else str.lower - rows = [] - for code in range(0x110000): - char = chr(code) - converted = mapping(char) - if converted != char: - rows.append((code, len(converted), *(ord(part) for part in converted))) + data = _case_data() + rows = data.UPPER_ROWS if upper else data.LOWER_ROWS width = max(len(row) for row in rows) mappings = "\n".join( " {" + ", ".join(f"0x{value:X}u" for value in row) + "}," @@ -193,11 +223,20 @@ def _case_lambda(upper: bool) -> str: f" {{0x{start:X}u, 0x{end:X}u, {flags}u}}," for start, end, flags in _context_ranges() ) - context = _CONTEXT_CPP.replace("@PROPERTIES@", properties) + final_rows = data.LOWER_FINAL_SIGMA_ROWS + final_mappings = "\n".join( + " {" + ", ".join(f"0x{value:X}u" for value in row) + "}," + for row in final_rows + ) + context = (_CONTEXT_CPP.replace("@PROPERTIES@", properties) + .replace("@FINAL_WIDTH@", str(max(len(row) for row in final_rows))) + .replace("@FINAL_MAPPINGS@", final_mappings)) result = _BODY_CPP for key, value in ( - ("@VERSION@", unicodedata.unidata_version), - ("@DIRECTION@", "uppercase" if upper else "lowercase"), + ("@VERSION@", data.DATA_VERSION), + ("@PROVENANCE@", "\n".join( + f" // {name} SHA-256 {digest}" for name, digest in data.SOURCE_SHA256 + )), ("@TABLE@", "_pf_toupper_map" if upper else "_pf_tolower_map"), ("@WIDTH@", str(width)), ("@MAPPINGS@", mappings), diff --git a/pineforge_codegen/codegen/unicode_regex.py b/pineforge_codegen/codegen/unicode_regex.py index 8016ebb3..5cd098d1 100644 --- a/pineforge_codegen/codegen/unicode_regex.py +++ b/pineforge_codegen/codegen/unicode_regex.py @@ -3,13 +3,14 @@ The runtime regex accepts bytes. A Unicode character is represented by its one to four byte encoding, retaining an isolated surrogate as three bytes. Each regex atom is a noncapturing group over those encodings, so a quantifier -repeats characters and new groups do not renumber captures. Class unions, ranges and -complements use the entire character domain, not an inventory of Unicode letters. +repeats characters. Class unions, ranges and complements use the entire +character domain, not an inventory of Unicode letters. -Matching still goes through the runtime's string helper, retaining its capture -selection and coded regex errors. Only the matched units are joined back into -ordinary UTF-8. Valid surrogate pairs form one regex character; string lengths -and substring offsets continue to use the unchanged UTF-16 codec. +The engine selects its first capture. An outer capture therefore selects the +whole match, with user captures retained and decimal backreferences shifted by +one. Matching and coded regex errors still use the runtime string helper. +Valid surrogate pairs form one regex character; lengths and substring offsets +continue to use the UTF-16 codec. """ @@ -94,9 +95,7 @@ def _str_match_expr(args: list[str]) -> str: 0x80u | ((_pf_unit >> 6) & 63), 0x80u | (_pf_unit & 63)}; } -// The endpoints have the same encoding width. Split at differing prefixes; -// complete intervening prefixes share all their continuation-byte ranges. -// This builds a bounded trie for a range instead of enumerating its members. +// Split ranges at differing encoding prefixes instead of enumerating members. inline void _pf_byte_trie(const std::vector& _pf_low, const std::vector& _pf_high, std::size_t _pf_at, const std::string& _pf_prefix, @@ -247,8 +246,25 @@ class _pf_pattern { return 0x10000u + ((_pf_high - 0xD800u) << 10) + (_pf_low - 0xDC00u); } - // Retain the existing byte engine's named/collating class syntax too. - // Its classic-locale classes are sets of ASCII units. + // The synthetic whole-match group shifts every user capture by one. + // Add in decimal text so an invalid oversized reference cannot overflow + // into a valid group number before the engine checks it. + std::string _pf_backreference(uint32_t _pf_first) { + std::string _pf_digits(1, static_cast(_pf_first)); + while (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= '0' + && _pf_units[_pf_at] <= '9') + _pf_digits += static_cast(_pf_take()); + bool _pf_carry = true; + for (std::size_t _pf_i = _pf_digits.size(); _pf_i > 0 && _pf_carry;) { + --_pf_i; + if (_pf_digits[_pf_i] == '9') _pf_digits[_pf_i] = '0'; + else { ++_pf_digits[_pf_i]; _pf_carry = false; } + } + if (_pf_carry) _pf_digits.insert(_pf_digits.begin(), '1'); + return "\\" + _pf_digits; + } + + // Classic-locale named/collating classes are sets of ASCII characters. _pf_member _pf_named_member(uint32_t _pf_delimiter) { std::string _pf_name; for (;;) { @@ -337,7 +353,8 @@ class _pf_pattern { explicit _pf_pattern(const std::vector& _pf_value) : _pf_units(_pf_value) {} std::string _pf_compile() { - std::string _pf_result; + std::string _pf_result = "("; + std::size_t _pf_depth = 0; while (_pf_at < _pf_units.size()) { auto _pf_code = _pf_take(); if (_pf_code == '[') { @@ -352,24 +369,27 @@ class _pf_pattern { } else if (_pf_code == 'u' || _pf_code == 'x' || _pf_code > 127) { const auto _pf_unit = _pf_pair(_pf_character_escape(_pf_code)); _pf_result += _pf_atom({{_pf_unit, _pf_unit}}); + } else if (_pf_code >= '1' && _pf_code <= '9') { + _pf_result += _pf_backreference(_pf_code); } else { - // Assertions, backreferences and ASCII character escapes - // retain the byte engine's grammar and capture numbering. + // Assertions and ASCII character escapes keep their grammar. _pf_result += '\\'; _pf_result += static_cast(_pf_code); if (_pf_code == 'c') _pf_result += static_cast(_pf_take()); - if (_pf_code >= '1' && _pf_code <= '9') - while (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= '0' - && _pf_units[_pf_at] <= '9') - _pf_result += static_cast(_pf_take()); } } else if (_pf_code == '(') { + ++_pf_depth; _pf_result += '('; if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '?') { _pf_result += static_cast(_pf_take()); if (_pf_at < _pf_units.size() && _pf_units[_pf_at] < 128) _pf_result += static_cast(_pf_take()); } + } else if (_pf_code == ')') { + // The wrapper must not repair an unmatched user parenthesis. + if (_pf_depth == 0) throw _pf_invalid{}; + --_pf_depth; + _pf_result += ')'; } else if (_pf_code == '{') { _pf_result += '{'; while (_pf_at < _pf_units.size() @@ -378,7 +398,7 @@ class _pf_pattern { _pf_result += static_cast(_pf_take()); if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '}') _pf_result += static_cast(_pf_take()); - } else if (_pf_code == ')' || _pf_code == '|' || _pf_code == '^' + } else if (_pf_code == '|' || _pf_code == '^' || _pf_code == '$' || _pf_code == '*' || _pf_code == '+' || _pf_code == '?' || _pf_code == '}') { _pf_result += static_cast(_pf_code); @@ -387,7 +407,8 @@ class _pf_pattern { _pf_result += _pf_atom({{_pf_code, _pf_code}}); } } - return _pf_result; + if (_pf_depth != 0) throw _pf_invalid{}; + return _pf_result + ")"; } }; @@ -398,9 +419,8 @@ class _pf_pattern { try { _pf_pattern_bytes = _pf_pattern(_pf_pattern_units)._pf_compile(); } catch (const _pf_invalid&) { - // If the byte grammar already rejects this spelling, retain its - // exact runtime reason/context. A range invalid only in characters still - // needs the same coded invalid-pattern stop. + // Retain the engine's exact reason/context for byte-invalid patterns. + // A range invalid only in characters needs the same coded stop. (void)::pineforge::pine_str_match(_pf_source, _pf_regex); const std::string _pf_message = "Error on bar " + std::to_string(_pf_bar) + ": Invalid regex pattern in the 'str.match' function."; diff --git a/scripts/gen_unicode_case_data.py b/scripts/gen_unicode_case_data.py new file mode 100644 index 00000000..104ce540 --- /dev/null +++ b/scripts/gen_unicode_case_data.py @@ -0,0 +1,322 @@ +"""Generate pinned casing tables offline from the three Unicode 14.0.0 inputs. + +Only explicitly named files are read. Input hashes and sizes are mandatory. +No host Unicode database, locale, network, environment or package import +supplies data. The output is an ASCII Python module included in the package. +Run this generator on an approved build/test venue, not the authoring host. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import re + + +DATA_VERSION = "14.0.0" +DATA_FORMAT = 1 +INPUT_PINS = ( + ("UnicodeData.txt", 1897793, + "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", 16830, + "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", 1040794, + "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +_CODE = re.compile(r"[0-9A-F]{4,6}", re.ASCII) +_LANGUAGE = re.compile(r"[a-z]{2,3}(?:[-_][A-Za-z0-9]+)*", re.ASCII) +_DOMAIN = 0x110000 + + +class DataError(ValueError): + """An input fails its pinned identity or the supported UCD format.""" + + +@dataclass(frozen=True) +class CaseData: + lower: tuple[tuple[int, ...], ...] + upper: tuple[tuple[int, ...], ...] + lower_final_sigma: tuple[tuple[int, ...], ...] + context: tuple[tuple[int, int, int], ...] + excluded_locale_rules: tuple[str, ...] + + +def _code(value: str) -> int: + if not _CODE.fullmatch(value): + raise DataError(f"invalid UCD code point: {value!r}") + result = int(value, 16) + if result >= _DOMAIN: + raise DataError(f"UCD code point outside Unicode: {value}") + return result + + +def _sequence(value: str) -> tuple[int, ...]: + return tuple(_code(part) for part in value.split()) + + +def _records(text: str): + for line_number, line in enumerate(text.split("\n"), 1): + body = line.split("#", 1)[0].strip(" \t\r") + if body: + if not body.isascii(): + raise DataError(f"non-ASCII UCD data field at line {line_number}") + yield line_number, [field.strip(" \t\r") for field in body.split(";")] + + +def _unicode_data(text: str, flags: bytearray): + lower: dict[int, tuple[int, ...]] = {} + upper: dict[int, tuple[int, ...]] = {} + previous = -1 + pending = None + + def apply(first: int, last: int, fields: list[str]) -> None: + if first != last and (fields[12] or fields[13]): + raise DataError("a UnicodeData range carries a simple case mapping") + if fields[2] == "Nd": + for point in range(first, last + 1): + flags[point] |= 4 + if fields[12]: + upper[first] = (_code(fields[12]),) + if fields[13]: + lower[first] = (_code(fields[13]),) + + for line, fields in _records(text): + if len(fields) != 15: + raise DataError(f"UnicodeData line {line}: expected 15 fields") + point = _code(fields[0]) + if point <= previous: + raise DataError(f"UnicodeData line {line}: non-increasing code points") + previous = point + name = fields[1] + if name.endswith(", First>"): + if pending is not None: + raise DataError(f"UnicodeData line {line}: nested range") + pending = (point, fields) + elif name.endswith(", Last>"): + if pending is None: + raise DataError(f"UnicodeData line {line}: range has no first row") + first, first_fields = pending + if (first_fields[1].removesuffix(", First>") + != name.removesuffix(", Last>") + or first_fields[2:] != fields[2:]): + raise DataError(f"UnicodeData line {line}: inconsistent range") + apply(first, point, first_fields) + pending = None + else: + if pending is not None: + raise DataError(f"UnicodeData line {line}: unterminated range") + apply(point, point, fields) + if pending is not None: + raise DataError("UnicodeData: unterminated range at end of input") + return lower, upper + + +def _special_casing(text: str, lower: dict, upper: dict): + final_lower: dict[int, tuple[int, ...]] = {} + unconditional_seen: set[int] = set() + excluded: list[str] = [] + for line, fields in _records(text): + if len(fields) == 6 and fields[-1] == "": + fields.pop() + if len(fields) != 5: + raise DataError(f"SpecialCasing line {line}: expected five fields") + point = _code(fields[0]) + lo, title, up = (_sequence(field) for field in fields[1:4]) + conditions = tuple(fields[4].split()) + if not conditions: + if point in unconditional_seen: + raise DataError(f"SpecialCasing line {line}: duplicate unconditional row") + unconditional_seen.add(point) + lower[point], upper[point] = lo, up + elif any(_LANGUAGE.fullmatch(token) for token in conditions): + # Root casing deliberately excludes every language-tailored row, + # including deletions. Keep an inventory in the generated module. + excluded.append(f"{point:04X}: {' '.join(conditions)}") + elif conditions == ("Final_Sigma",): + if point in final_lower: + raise DataError(f"SpecialCasing line {line}: duplicate contextual row") + if up != (point,) or title != (point,): + raise DataError("Final_Sigma has a nonidentity upper/title mapping") + final_lower[point] = lo + else: + raise DataError(f"SpecialCasing line {line}: unsupported root condition") + return final_lower, tuple(sorted(excluded)) + + +def _properties(text: str, flags: bytearray) -> None: + bits = {"Cased": 1, "Case_Ignorable": 2} + previous: dict[str, int] = {} + for line, fields in _records(text): + if len(fields) != 2: + raise DataError(f"DerivedCoreProperties line {line}: expected two fields") + prop = fields[1] + if prop not in bits: + continue + interval = fields[0].split("..") + if len(interval) not in (1, 2): + raise DataError(f"DerivedCoreProperties line {line}: invalid interval") + first, last = _code(interval[0]), _code(interval[-1]) + if first > last or first <= previous.get(prop, -1): + raise DataError(f"DerivedCoreProperties line {line}: unordered interval") + previous[prop] = last + for point in range(first, last + 1): + flags[point] |= bits[prop] + if set(previous) != set(bits): + raise DataError("DerivedCoreProperties: a required casing property is missing") + + +def _mapping_rows(mapping: dict[int, tuple[int, ...]]): + return tuple( + (point, len(converted), *converted) + for point, converted in sorted(mapping.items()) + if converted != (point,) + ) + + +def _context_ranges(flags: bytearray): + result = [] + first = 0 + while first < len(flags): + value = flags[first] + last = first + while last + 1 < len(flags) and flags[last + 1] == value: + last += 1 + if value: + result.append((first, last, value)) + first = last + 1 + return tuple(result) + + +def parse_ucd(unicode_data: str, special_casing: str, derived_properties: str) -> CaseData: + """Pure parser; production CLI calls this only after all three hash checks.""" + flags = bytearray(_DOMAIN) + lower, upper = _unicode_data(unicode_data, flags) + final_lower, excluded = _special_casing(special_casing, lower, upper) + _properties(derived_properties, flags) + data = CaseData( + _mapping_rows(lower), _mapping_rows(upper), + _mapping_rows(final_lower), _context_ranges(flags), excluded, + ) + if not all((data.lower, data.upper, data.lower_final_sigma, data.context)): + raise DataError("a required casing table is empty") + return data + + +def _verified_text(name: str, payload: bytes) -> str: + pin = next((item for item in INPUT_PINS if item[0] == name), None) + if pin is None: + raise DataError(f"input is not in the pinned UCD set: {name}") + _, size, digest = pin + if len(payload) != size or hashlib.sha256(payload).hexdigest() != digest: + raise DataError(f"pinned size/SHA-256 mismatch: {name}") + text = payload.decode("utf-8") + if name != "UnicodeData.txt": + stem = name.removesuffix(".txt") + if not text.startswith(f"# {stem}-{DATA_VERSION}.txt\n"): + raise DataError(f"pinned version header mismatch: {name}") + return text + + +def render_module(data: CaseData) -> bytes: + """Deterministic ASCII source: no dates, paths, host versions or locale.""" + lines = [ + '"""Generated Unicode 14.0.0 casing data; do not edit table entries.', + "", + "Source: https://www.unicode.org/Public/14.0.0/ucd/", + "Copyright 2021 Unicode, Inc.", + "Unicode data terms: https://www.unicode.org/terms_of_use.html", + 'Regenerate with scripts/gen_unicode_case_data.py using pinned inputs."""', + "", + f"DATA_FORMAT = {DATA_FORMAT}", + f'DATA_VERSION = "{DATA_VERSION}"', + "SOURCE_SHA256 = (", + ] + lines.extend(f' ("{name}", "{digest}"),' for name, _, digest in INPUT_PINS) + lines.extend((")", "SOURCE_BYTE_COUNTS = (")) + lines.extend(f' ("{name}", {size}),' for name, size, _ in INPUT_PINS) + lines.extend((")", "EXCLUDED_LOCALE_RULES = (")) + lines.extend(f' "{rule}",' for rule in data.excluded_locale_rules) + lines.append(")") + for name, rows in ( + ("LOWER_ROWS", data.lower), + ("UPPER_ROWS", data.upper), + ("LOWER_FINAL_SIGMA_ROWS", data.lower_final_sigma), + ("CONTEXT_RANGES", data.context), + ): + lines.extend(("", f"{name} = (")) + lines.extend(" (" + ", ".join(f"0x{value:X}" for value in row) + ")," + for row in rows) + lines.append(")") + return ("\n".join(lines) + "\n").encode("ascii") + + +def _checked_path(value: str, filename: str) -> Path: + """Reject sensitive names before any filesystem query; reject symlinks.""" + path = Path(value) + parts = path.parts + sensitive = {".ssh", ".aws", ".codex"} + if (any(part in sensitive or part.endswith(".env") for part in parts) + or any(parts[i:i + 2] == (".config", "pineforge") + or parts[i:i + 3] == (".config", "zsh", "keys") + for i in range(len(parts)))): + raise DataError("refused protected file path by name") + if not path.is_absolute() or ".." in parts or path.name != filename: + raise DataError(f"expected an absolute path ending in {filename}") + for component in (*reversed(path.parents), path): + if component.is_symlink(): + raise DataError(f"refused symlink path: {component}") + return path + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--unicode-data") + parser.add_argument("--special-casing") + parser.add_argument("--derived-properties") + parser.add_argument("--output") + parser.add_argument("--check", action="store_true", help="compare an existing output") + parser.add_argument("--crash-control", action="store_true") + args = parser.parse_args(argv) + if args.crash_control: + print('{"read_files": [], "write_files": []}', flush=True) + raise RuntimeError("deliberate Unicode generator crash control") + values = (args.unicode_data, args.special_casing, args.derived_properties) + if not all(values) or not args.output: + parser.error("all three explicit UCD inputs and --output are required") + # Announce the exact follow set before checking paths or opening a file. + print(json.dumps({ + "read_files": list(values) + ([args.output] if args.check else []), + "write_files": [] if args.check else [args.output], + }, sort_keys=True), flush=True) + paths = [_checked_path(value, pin[0]) for value, pin in zip(values, INPUT_PINS)] + output = _checked_path(args.output, "_unicode_case_data.py") + texts = [] + for path, (name, _, _) in zip(paths, INPUT_PINS): + texts.append(_verified_text(name, path.read_bytes())) + data = parse_ucd(*texts) + encoded = render_module(data) + if args.check: + if output.read_bytes() != encoded: + raise DataError("generated casing module differs from pinned inputs") + else: + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("xb") as handle: + if handle.write(encoded) != len(encoded): + raise OSError("short casing data write") + print(json.dumps({ + "output": str(output), + "sha256": hashlib.sha256(encoded).hexdigest(), + "bytes": len(encoded), + "lower_rows": len(data.lower), + "upper_rows": len(data.upper), + "context_ranges": len(data.context), + "final_sigma_rows": len(data.lower_final_sigma), + }, sort_keys=True), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_unicode_case_context.py b/tests/test_unicode_case_context.py new file mode 100644 index 00000000..0eb83160 --- /dev/null +++ b/tests/test_unicode_case_context.py @@ -0,0 +1,39 @@ +"""Unit classification of casing context; Pine values belong to TV probes.""" + +import pytest + +from pineforge_codegen.codegen.unicode_case import _context_flags +from pineforge_codegen.codegen import unicode_case + + +@pytest.mark.parametrize("char, flags", [ + ("A", 1), # Cased + ("\u0301", 2), # Case_Ignorable + ("\u0345", 3), # Cased and Case_Ignorable; skipping takes precedence + (".", 2), + (",", 0), + ("1", 4), # Decimal_Number, not Case_Ignorable or Cased + ("\u0664", 4), + ("\uff16", 4), + ("\U0001d7d8", 4), + ("中", 0), +]) +def test_context_flags_keep_decimal_runs_separate_from_case_properties(char, flags): + assert _context_flags(char) == flags + + +@pytest.mark.parametrize("field, invalid", [ + ("DATA_FORMAT", 0), + ("DATA_VERSION", "0.0.0"), + ("SOURCE_SHA256", ()), + ("SOURCE_BYTE_COUNTS", ()), +]) +def test_casing_data_requires_the_pinned_identity_before_use(monkeypatch, field, invalid): + data = unicode_case._case_data() + unicode_case._case_data.cache_clear() + try: + monkeypatch.setattr(data, field, invalid) + with pytest.raises(RuntimeError, match="pinned UCD inputs"): + unicode_case._case_data() + finally: + unicode_case._case_data.cache_clear() diff --git a/tests/test_unicode_case_data_generation.py b/tests/test_unicode_case_data_generation.py new file mode 100644 index 00000000..469763d7 --- /dev/null +++ b/tests/test_unicode_case_data_generation.py @@ -0,0 +1,113 @@ +"""Unit targets for the offline UCD parser and its pinned serialization.""" + +import ast +from pathlib import Path + +import pytest + +from scripts import gen_unicode_case_data as gen + + +def _row(point, name, category, *, upper="", lower=""): + fields = [f"{point:04X}", name, category, "0", "L", + "", "", "", "", "N", "", "", upper, lower, ""] + return ";".join(fields) + "\n" + + +def _inputs(): + # Synthetic UCD records test parsing, not the casing of a Pine script. + ordinary = ( + _row(0x31, "TEST DECIMAL", "Nd") + + _row(0x41, "TEST CAPITAL", "Lu", lower="0061") + + _row(0x61, "TEST SMALL", "Ll", upper="0041") + ) + special = ( + "0061; 0061; 0042 0043; 0042 0043;\n" + "0041; 0062; 0041; 0041; Final_Sigma;\n" + "0061; ; 0049; 0049; tr After_I;\n" + ) + properties = ( + "0041..0042 ; Cased\n" + "002E ; Case_Ignorable\n" + "0042 ; Case_Ignorable\n" + ) + return ordinary, special, properties + + +def test_ucd_parser_combines_simple_full_and_contextual_data_without_locale_tailoring(): + data = gen.parse_ucd(*_inputs()) + assert data.lower == ((0x41, 1, 0x61),) + assert data.upper == ((0x61, 2, 0x42, 0x43),) + assert data.lower_final_sigma == ((0x41, 1, 0x62),) + assert data.context == ( + (0x2E, 0x2E, 2), (0x31, 0x31, 4), + (0x41, 0x41, 1), (0x42, 0x42, 3), + ) + assert data.excluded_locale_rules == ("0061: tr After_I",) + + +def test_unicode_data_first_last_ranges_apply_properties_to_the_whole_interval(): + flags = bytearray(0x110000) + records = (_row(0x100, "", "Nd") + + _row(0x105, "", "Nd")) + gen._unicode_data(records, flags) + assert gen._context_ranges(flags) == ((0x100, 0x105, 4),) + with pytest.raises(gen.DataError, match="unterminated range"): + gen._unicode_data(_row(0x100, "", "Nd"), flags) + with pytest.raises(gen.DataError, match="inconsistent range"): + gen._unicode_data(records.replace("", ""), flags) + + +@pytest.mark.parametrize("name,size,digest", gen.INPUT_PINS) +def test_input_identity_rejects_truncated_and_same_size_wrong_content(name, size, digest): + with pytest.raises(gen.DataError, match="size/SHA-256"): + gen._verified_text(name, b"") + with pytest.raises(gen.DataError, match="size/SHA-256"): + gen._verified_text(name, b"\n" * size) + + +def test_unknown_root_condition_is_an_error_instead_of_a_dropped_override(): + ordinary, special, properties = _inputs() + with pytest.raises(gen.DataError, match="unsupported root condition"): + gen.parse_ucd(ordinary, special.replace("Final_Sigma", "Future_Context"), properties) + + +def test_generated_module_round_trips_tables_and_ignores_irrelevant_record_order(): + ordinary, special, properties = _inputs() + data = gen.parse_ucd(ordinary, special, properties) + encoded = gen.render_module(data) + # These two source sections are sets, unlike ordered UnicodeData records. + reordered = gen.parse_ucd( + ordinary, "\n".join(reversed(special.splitlines())) + "\n", + "002E ; Case_Ignorable\n0042 ; Case_Ignorable\n0041..0042 ; Cased\n", + ) + assert gen.render_module(reordered) == encoded + tree = ast.parse(encoded.decode("ascii")) + assignments = { + node.targets[0].id: ast.literal_eval(node.value) + for node in tree.body if isinstance(node, ast.Assign) + } + assert assignments["LOWER_ROWS"] == data.lower + assert assignments["UPPER_ROWS"] == data.upper + assert assignments["LOWER_FINAL_SIGMA_ROWS"] == data.lower_final_sigma + assert assignments["CONTEXT_RANGES"] == data.context + assert assignments["SOURCE_SHA256"] == tuple( + (name, digest) for name, _, digest in gen.INPUT_PINS + ) + + +@pytest.mark.parametrize("filename", [ + "/no-access/.ssh/UnicodeData.txt", + "/no-access/.aws/UnicodeData.txt", + "/no-access/.codex/UnicodeData.txt", + "/no-access/.config/zsh/keys/UnicodeData.txt", + "/no-access/.config/pineforge/UnicodeData.txt", + "/no-access/private.env/UnicodeData.txt", +]) +def test_sensitive_paths_are_refused_by_name_before_any_filesystem_query(monkeypatch, filename): + def no_filesystem_query(*args, **kwargs): + pytest.fail("protected name reached a filesystem query") + + monkeypatch.setattr(Path, "is_symlink", no_filesystem_query) + with pytest.raises(gen.DataError, match="protected file path"): + gen._checked_path(filename, "UnicodeData.txt") diff --git a/tests/test_unicode_regex_lowering.py b/tests/test_unicode_regex_lowering.py new file mode 100644 index 00000000..c46908b9 --- /dev/null +++ b/tests/test_unicode_regex_lowering.py @@ -0,0 +1,71 @@ +"""Unit checks of the byte-pattern compiler, not Pine replay expectations.""" + +from pineforge_codegen import transpile +from tests._compile import run_emitted_tu + + +def _compiler_tu(): + return transpile('//@version=6\nstrategy("regex compiler")\n' + 'result = str.match("abc", "b")\n') + + +def test_pattern_compiler_preserves_capture_targets_under_whole_match_group(): + driver = r''' +#include +static void check(const std::string& pattern, const std::string& subject, + const std::vector& captures) { + const auto units = _pf_regex::_pf_characters(pattern); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + const std::regex re(compiled); + std::smatch match; + assert(std::regex_search(subject, match, re)); + assert(match.size() == captures.size() + 2); + assert(match[1].str() == match[0].str()); + for (std::size_t i = 0; i < captures.size(); ++i) + assert(match[i + 2].str() == captures[i]); +} +int main() { + check(R"((x)(y)\2\1)", "xyyx", {"x", "y"}); + check(R"((?:q)(x)(?=y)(y)\2\1)", "qxyyx", {"x", "y"}); + check(R"((a)(b)(c)(d)(e)(f)(g)(h)(i)\9)", "abcdefghii", + {"a", "b", "c", "d", "e", "f", "g", "h", "i"}); + check(R"((a)(b)(c)(d)(e)(f)(g)(h)(i)(j)\10)", "abcdefghijj", + {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}); + check(R"((é+)\1)", "éééé", {"éé"}); + check(R"(\(x\))", "(x)", {}); + check(R"([(]x[)])", "(x)", {}); + // Two backslashes and the digit are a literal, not a reference to shift. + check(R"(\\1)", R"(\1)", {}); + const auto units = _pf_regex::_pf_characters(R"(\99999999999999999999)"); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + assert(compiled == R"((\100000000000000000000))"); +} +''' + assert run_emitted_tu(_compiler_tu(), driver, opt="-O0", + label="regex capture compiler") == "" + + +def test_whole_match_wrapper_cannot_balance_invalid_user_parentheses(): + driver = r''' +#include +int main() { + for (const std::string pattern : {")(", ")a(", "(", "a)", "(?:x"}) { + const auto units = _pf_regex::_pf_characters(pattern); + bool refused = false; + try { (void)_pf_regex::_pf_pattern(units)._pf_compile(); } + catch (const _pf_regex::_pf_invalid&) { refused = true; } + assert(refused); + } + // A synthetic group cannot make a nonexistent user reference valid. + const auto units = _pf_regex::_pf_characters(R"(\1)"); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + bool refused = false; + try { (void)std::regex(compiled); } + catch (const std::regex_error& error) { + refused = error.code() == std::regex_constants::error_backref; + } + assert(refused); +} +''' + assert run_emitted_tu(_compiler_tu(), driver, opt="-O0", + label="regex wrapper grammar") == "" From de9d2d2ecdba88e785b4dcf6e56666eb273f732e Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:56:54 +0800 Subject: [PATCH 5/8] Honor array join defaults and builtin receiver domains Use the empty default separator and shared numeric formatting while preserving integer digits and receiver/separator evaluation order. Reject proven builtin boolean-array joins after authored-method resolution, and validate named string-length arguments through the declared signature. --- CHANGELOG.md | 8 +- pineforge_codegen/analyzer/base.py | 2 + pineforge_codegen/analyzer/diagnostics.py | 4 +- pineforge_codegen/analyzer/types.py | 23 +++- pineforge_codegen/codegen/emit_top.py | 3 + pineforge_codegen/codegen/tables.py | 46 ++++++- pineforge_codegen/codegen/types.py | 20 +-- pineforge_codegen/codegen/visit_call.py | 9 +- pineforge_codegen/intrinsic_binding.py | 25 +++- pineforge_codegen/signatures.py | 2 +- tests/test_array_join_domains.py | 143 ++++++++++++++++++++++ tests/test_codegen_audit_fixes.py | 70 +++++++++-- tests/test_method_receiver_types.py | 1 + tests/test_type_keyword_arguments.py | 14 +++ 14 files changed, 344 insertions(+), 26 deletions(-) create mode 100644 tests/test_array_join_domains.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c9b3b3..65bc8422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,13 @@ supported as exact pairs; on the 0.x line they are independent. See the storage type in compound assignments and `request.security()` expressions. - Array joins retain string return types through helper and method forwarding. Named, mixed and reordered receiver/separator arguments use their declared - slots; an omitted separator keeps the default. + slots; an omitted separator is empty. Numeric elements use Pine number + formatting without fixed trailing zeros, and receiver and separator are + evaluated once in parameter order, including for empty arrays. + Builtin joins refuse boolean arrays during translation; authored methods + retain their own overloads and defaults. +- `str.length` validates named arguments against its declared `string` + parameter; unknown names produce the standard located argument diagnostic. - `str.upper` and `str.lower` use pinned Unicode 14.0.0 mappings and properties, including supplementary letters, expansions and contextual final sigma. Translating Python versions and the native locale do not select the data. diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index 7eab1bf7..49eac84c 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -7521,6 +7521,8 @@ def _visit_FuncCall(self, node: FuncCall) -> PineType: if isinstance(obj, Identifier) and obj.name == "str": for arg in node.args: self._visit(arg) + for value in node.kwargs.values(): + self._visit(value) # Most str.* return a string, but predicates and index helpers # are scalar. Keep this aligned with signatures.py and the C++ # emitter's _infer_type path. diff --git a/pineforge_codegen/analyzer/diagnostics.py b/pineforge_codegen/analyzer/diagnostics.py index 603567ee..ada0881f 100644 --- a/pineforge_codegen/analyzer/diagnostics.py +++ b/pineforge_codegen/analyzer/diagnostics.py @@ -45,7 +45,8 @@ class DiagnosticsHelper: the diagnostic surface easy to audit. """ - def _error(self, message: str, loc: SourceLocation | None = None) -> None: + def _error(self, message: str, loc: SourceLocation | None = None, + *, hint: str | None = None) -> None: """Raise a compile error.""" if loc is None: loc = SourceLocation(file=self._filename, line=1, col=1, end_col=1) @@ -54,6 +55,7 @@ def _error(self, message: str, loc: SourceLocation | None = None) -> None: phase=Phase.ANALYZER, location=loc, message=message, + hint=hint, ) raise CompileError([diag]) diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index 2f969b47..8f597477 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -46,8 +46,10 @@ SwitchStmt, TupleAssign, TupleLiteral, UnaryOp, VarDecl, ) from ..errors import Phase +from ..intrinsic_binding import array_method_domain_error from ..limits import iter_ast_nodes from ..matrix_overloads import matrix_sum_has_rhs +from ..pine_spelling import expr_start from ..symbols import ( PineType, TypeSpec, @@ -176,7 +178,7 @@ def _template_args_from_call(self, node: FuncCall) -> list[str]: def _fixed_array_call_result( self, call: FuncCall, receiver_spec: TypeSpec | None, ) -> TypeSpec | None: - """Read a fixed array result after authored-method resolution.""" + """Validate a builtin receiver and read its fixed result after methods.""" callee = call.callee if not isinstance(callee, MemberAccess): return None @@ -192,6 +194,25 @@ def _fixed_array_call_result( receiver_spec is None or receiver_spec.kind != "array" ): return None + if callee.member == "join": + receiver = ( + call.args[0] if call.args else call.kwargs.get("id") + ) if namespace_call else callee.object + # A namespace receiver can be a constructor containing TA or + # request calls already visited above. Read its type without + # registering those calls again; preserve call-site type overlays. + query = (self if isinstance(self, OperatorTypeQuery) + else OperatorTypeQuery(self, {})) + actual_spec = ( + query._type_spec_from_expr(receiver) + if namespace_call else receiver_spec + ) + problem = array_method_domain_error(callee.member, actual_spec) + if problem is not None: + self._error( + problem[0], expr_start(receiver).loc or call.loc, + hint=problem[1], + ) return fixed_builtin_result( "array", callee.member, len(call.args) + len(call.kwargs), ) diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index 9c97fce6..0d0661b2 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -138,6 +138,9 @@ def _emit_includes(self, lines: list[str]) -> None: self._uses_tv_number_format = any( (namespace == "str" and func_name in {"format", "tostring"}) or (namespace is None and func_name == "tostring") + # Namespace and method joins can render numeric elements without + # an explicit str.tostring call, including in callable bodies. + or (isinstance(node.callee, MemberAccess) and node.callee.member == "join") or (namespace == "log" and func_name in {"info", "warning", "error"} and len(node.args) > 1) for root in roots diff --git a/pineforge_codegen/codegen/tables.py b/pineforge_codegen/codegen/tables.py index 57ea87f4..79f17956 100644 --- a/pineforge_codegen/codegen/tables.py +++ b/pineforge_codegen/codegen/tables.py @@ -1071,6 +1071,46 @@ def _array_sort_indices(a: str, args: list[str]) -> str: # Methods called as ``array.method(arr, ...)`` or ``arr.method(...)``. +def _array_join(a: str, args: list[str], *, stringify: str | None = "pine_str_tostring_tv") -> str: + """Join after binding receiver and separator once, in parameter order. + + String elements retain the UTF-16 join, including adjacent surrogate + halves. Numeric elements use the existing Pine formatter; other element + families keep their previous conversion until their domains are measured. + """ + default = ARRAY_OPTIONAL_PARAMS["join"]["separator"] + separator = args[0] if args else f'std::string("{default}")' + if stringify is None: + result = "return _pf_u16_join(_pf_join_parts, _pf_join_separator); " + else: + formatter = "" + if stringify == "pine_str_tostring_tv": + # Keep every integer digit rather than passing a present int64 + # through the formatter's double overload. Test the sentinel in + # its original type; missing integers and floats share Pine's + # existing missing-number spelling. + formatter = ( + "const auto _pf_join_number = [](const auto _pf_join_value) { " + "if constexpr (std::is_integral_v) { " + "if (!is_na(_pf_join_value)) return std::to_string(_pf_join_value); " + "} return pine_str_tostring_tv(_pf_join_value); }; " + ) + stringify = "_pf_join_number" + result = ( + f"{formatter}std::string _pf_join_result; " + "for (std::size_t _pf_join_i = 0; _pf_join_i < _pf_join_parts.size(); " + "++_pf_join_i) { " + "if (_pf_join_i != 0) _pf_join_result += _pf_join_separator; " + f"_pf_join_result += {stringify}(_pf_join_parts[_pf_join_i]); " + "} return _pf_u16_normalize(_pf_join_result); " + ) + return ( + f"[&](){{ const auto& _pf_join_parts = ({a}); " + f"const std::string _pf_join_separator = ({separator}); " + f"{result}}}()" + ) + + ARRAY_METHODS = { "get": _checked_array_get, "set": _checked_array_set, @@ -1126,7 +1166,7 @@ def _array_sort_indices(a: str, args: list[str]) -> str: "percentile_nearest_rank": lambda a, args: f"[&](){{ if({a}.empty()) return na(); auto __pf_c={a}; std::sort(__pf_c.begin(),__pf_c.end()); double __pf_p=({args[0]}); if(is_na(__pf_p)) return (double)__pf_c.front(); double __pf_rank=std::ceil((__pf_p/100.0)*__pf_c.size()); if(!std::isfinite(__pf_rank)) return na(); int __pf_r=(int)std::clamp(__pf_rank,1.0,(double)__pf_c.size()); return (double)__pf_c[__pf_r-1]; }}()", "percentrank": _checked_array_percentrank, "abs": lambda a, args: f"[&](){{ std::vector __pf_r; for(auto v:{a})__pf_r.push_back(std::abs(v)); return __pf_r; }}()", - "join": lambda a, args: "[&](){{ std::string __pf_r; for(size_t __pf_i=0;__pf_i<{arr}.size();__pf_i++){{ if(__pf_i>0)__pf_r+={sep}; __pf_r+=std::to_string({arr}[__pf_i]); }} return __pf_r; }}()".format(arr=a, sep=args[0] if args else 'std::string(",")'), + "join": _array_join, "standardize": lambda a, args: f"[&](){{ double __pf_m=std::accumulate({a}.begin(),{a}.end(),0.0)/{a}.size(); double __pf_s=0; for(auto v:{a})__pf_s+=(v-__pf_m)*(v-__pf_m); __pf_s=std::sqrt(__pf_s/{a}.size()); std::vector __pf_r; for(auto v:{a})__pf_r.push_back(__pf_s==0?1.0:(v-__pf_m)/__pf_s); return __pf_r; }}()", "covariance": lambda a, args: f"[&](){{ auto&&__pf_b=({args[0]}); int __pf_n=std::min({a}.size(),__pf_b.size()); if(__pf_n<=0) return na(); double __pf_ma=0,__pf_mb=0; for(int __pf_i=0;__pf_i<__pf_n;__pf_i++){{__pf_ma+={a}[__pf_i];__pf_mb+=__pf_b[__pf_i];}} __pf_ma/=__pf_n;__pf_mb/=__pf_n; double __pf_c=0; for(int __pf_i=0;__pf_i<__pf_n;__pf_i++)__pf_c+=({a}[__pf_i]-__pf_ma)*(__pf_b[__pf_i]-__pf_mb); return __pf_c/__pf_n; }}()", "binary_search": lambda a, args: f"[&](){{ auto __pf_it=std::lower_bound({a}.begin(),{a}.end(),{args[0]}); return (__pf_it!={a}.end()&&*__pf_it=={args[0]})?(double)(__pf_it-{a}.begin()):-1.0; }}()", @@ -1371,9 +1411,11 @@ def _matrix_functional_receiver(method: str, node): "sum": {"id2": None}, "concat": {"vertical": None}, } -ARRAY_OPTIONAL_PARAMS: dict[str, dict[str, int | None]] = { +ARRAY_OPTIONAL_PARAMS: dict[str, dict[str, int | str | None]] = { "fill": {"index_from": 0, "index_to": None}, "sort": {"order": None}, + # Independently measured namespace and method calls omit the separator. + "join": {"separator": ""}, } # Matrix mutators whose established C++ lowering returns ``void``. A Pine diff --git a/pineforge_codegen/codegen/types.py b/pineforge_codegen/codegen/types.py index 05935a74..d73d387b 100644 --- a/pineforge_codegen/codegen/types.py +++ b/pineforge_codegen/codegen/types.py @@ -50,6 +50,7 @@ from ..cpp_names import cpp_identifier_fragment from ..errors import CodegenInvariantError, Phase from ..external_requests import UNPINNED_ANNOTATION +from ..intrinsic_binding import array_method_domain_error from ..limits import iter_ast_nodes from ..loop_types import FLOAT_REASSIGNMENT, LoopTypeSources, for_counter_type from ..matrix_overloads import matrix_sum_has_rhs @@ -82,6 +83,7 @@ PINE_TYPE_TO_CPP, STRATEGY_DIRECTION_VALUES, TA_RETURNS_BOOL, + _array_join, _matrix_functional_receiver, checked_array_slice, ARRAY_JOURNAL_DELTA_METHODS, @@ -3515,6 +3517,9 @@ def _array_method_expr( if method in ("covariance", "concat") and args: args = [f"_pf_array_id({args[0]})", *args[1:]] spec = spec or TypeSpec.array(TypeSpec.primitive("float")) + problem = array_method_domain_error(method, spec) + if problem is not None: + self._codegen_error(node, problem[0], hint=problem[1]) arr_cpp_type = self._type_spec_to_cpp(spec) elem_cpp = self._type_spec_to_cpp(spec.element) if spec.element is not None else "double" if method == "copy": @@ -3546,17 +3551,14 @@ def _array_method_expr( recv, args, result_type=arr_cpp_type, view_function=view_function, ) - elif method == "join" and elem_cpp == "std::string": + elif method == "join": self._uses_tv_string_units = True - separator = args[0] if args else 'std::string(",")' - # Bind the receiver before the separator even when the shared - # receiver wrapper sees only one use. A const reference also - # keeps a temporary array alive throughout the join. - lower_receiver = lambda recv: ( - f"[&](){{ const auto& _pf_join_parts = ({recv}); " - f"const std::string _pf_join_separator = ({separator}); " - "return _pf_u16_join(_pf_join_parts, _pf_join_separator); }()" + stringify = ( + None if elem_cpp == "std::string" + else "pine_str_tostring_tv" if elem_cpp in ("double", "int", "int64_t") + else "std::to_string" ) + lower_receiver = lambda recv: _array_join(recv, args, stringify=stringify) else: numeric_only = { "sum", "avg", "min", "max", "range", "stdev", "variance", "median", diff --git a/pineforge_codegen/codegen/visit_call.py b/pineforge_codegen/codegen/visit_call.py index be03c690..910b53ba 100644 --- a/pineforge_codegen/codegen/visit_call.py +++ b/pineforge_codegen/codegen/visit_call.py @@ -918,7 +918,14 @@ def _collection_arg_nodes(self, namespace, method, node, functional): if slots[index] not in optional: self._collection_argument_error(node, namespace, method, "missing", slots[index]) default = optional[slots[index]] - arguments[index] = None if default is None else NumberLiteral(value=default) + if default is None: + arguments[index] = None + elif isinstance(default, str): + arguments[index] = StringLiteral(value=default) + elif isinstance(default, bool): + arguments[index] = BoolLiteral(value=default) + else: + arguments[index] = NumberLiteral(value=default) return arguments def _collection_argument_error(self, node, namespace, method, problem, name): diff --git a/pineforge_codegen/intrinsic_binding.py b/pineforge_codegen/intrinsic_binding.py index 780aaa92..ed03ba13 100644 --- a/pineforge_codegen/intrinsic_binding.py +++ b/pineforge_codegen/intrinsic_binding.py @@ -1,4 +1,4 @@ -"""Keyword validation and positional binding from intrinsic records.""" +"""Intrinsic binding and proven builtin argument-domain diagnostics.""" from __future__ import annotations @@ -7,6 +7,7 @@ from .errors import CompileError, Diagnostic, Level, Phase, SourceLocation from .pine_spelling import expr_start from .signatures import IntrinsicFunc, Param +from .symbols import TypeSpec _STRATEGY_ORDER_FUNCTIONS = frozenset({ @@ -15,6 +16,28 @@ }) +def array_method_domain_error( + method: str, receiver: TypeSpec | None, +) -> tuple[str, str | None] | None: + """Return a proven builtin receiver mismatch, leaving other domains open. + + Join independently admits string, int and float arrays and refuses bool + arrays. The bool refusal does not prove any other array domain invalid. + Callers resolve authored methods before applying this builtin contract. + """ + if (method != "join" or receiver is None or receiver.kind != "array" + or receiver.element is None or receiver.element.kind != "primitive" + or receiver.element.name != "bool"): + return None + return render_diagnostic( + "PF-E1088", { + "name": "array.join", + "message": "cannot accept an array argument for parameter 'id'", + "spelled": "CE10123", + }, + ) + + def intrinsic_keyword_diagnostics( intrinsic: IntrinsicFunc | None, kwargs: dict[str, ASTNode], diff --git a/pineforge_codegen/signatures.py b/pineforge_codegen/signatures.py index 6ab3522f..c6a77285 100644 --- a/pineforge_codegen/signatures.py +++ b/pineforge_codegen/signatures.py @@ -445,7 +445,7 @@ def _str(short_name: str, *sigs: FuncSig, _str("format", _sig([("formatString", S)], ret=S)) # variadic _str("format_time", _sig([("time", I), ("format", S), ("timezone", S, "")], ret=S), keyword_names_verified=True) -_str("length", _sig([("string", S)], ret=I)) +_str("length", _sig([("string", S)], ret=I), keyword_names_verified=True) _str("contains", _sig([("source", S), ("str", S)], ret=B)) _str("startswith", _sig([("source", S), ("str", S)], ret=B)) _str("endswith", _sig([("source", S), ("str", S)], ret=B)) diff --git a/tests/test_array_join_domains.py b/tests/test_array_join_domains.py new file mode 100644 index 00000000..879cae05 --- /dev/null +++ b/tests/test_array_join_domains.py @@ -0,0 +1,143 @@ +"""Builtin join's proven receiver mismatch, independent of C++ storage. + +The retained compiler response refuses array. String, int and float +have independent admissions; other domains stay open rather than inheriting +an exclusive array whitelist from that one overload diagnostic. +""" + +import pytest + +from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.ast_nodes import Identifier +from pineforge_codegen.codegen import CodeGen +from pineforge_codegen.errors import CompileError, Level, Phase, SourceLocation +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser +from pineforge_codegen.symbols import PineType, TypeSpec + + +HEADER = '//@version=6\nstrategy("join domains")\n' +FILENAME = "join_domains.pine" + + +def _analyze(body): + source = HEADER + body + program = Parser( + Lexer(source, filename=FILENAME).tokenize(), + source=source, filename=FILENAME, + ).parse() + return Analyzer(program, filename=FILENAME).analyze() + + +def _refusal(body, *, analyze_only=False): + with pytest.raises(CompileError) as caught: + if analyze_only: + _analyze(body) + else: + transpile(HEADER + body, filename=FILENAME) + diagnostic = next(d for d in caught.value.diagnostics if d.level == Level.ERROR) + assert diagnostic.code == "PF-E1088" + assert diagnostic.args["name"] == "array.join" + assert "array" in diagnostic.message + assert diagnostic.args["spelled"] == "CE10123" + assert diagnostic.hint == "Use the parameters of TradingView's signature." + return diagnostic + + +@pytest.mark.parametrize("initializer", [ + "array.from(true, false)", + "array.new_bool(0)", +]) +@pytest.mark.parametrize("call", [ + 'array.join(items, "|")', + 'array.join(id=items, separator="|")', + 'array.join(separator="|", id=items)', + 'array.join(items, separator="|")', + "array.join(items)", + 'items.join("|")', + 'items.join(separator="|")', + "items.join()", +]) +def test_bool_domain_refusal_is_located_on_the_receiver(initializer, call): + statement = f"observed = {call}" + diagnostic = _refusal(f"items = {initializer}\n{statement}\n") + column = statement.index("items") + 1 + assert diagnostic.phase == Phase.ANALYZER + assert diagnostic.location == SourceLocation( + FILENAME, 4, column, column + len("items"), + ) + + +@pytest.mark.parametrize("body", [ + 'observed = array.join(array.from(true, false), "|")\n', + "render_flags(array flags) => flags.join()\n", + "items = array.from(true, false)\nplot(str.length(items.join()))\n", + "items = array.from(true, false)\nplot(str.length(string=items.join()))\n", + "items = array.from(true, false)\nif false\n observed = items.join()\n", +]) +def test_analyzer_checks_join_domain_before_ignored_calls_can_disappear(body): + assert _refusal(body, analyze_only=True).phase == Phase.ANALYZER + + +@pytest.mark.parametrize("initializer", [ + 'array.from("alpha", "beta")', + "array.from(1, -2)", + "array.from(1.25, -2.5)", + "array.new(0)", + "array.new(0)", +]) +def test_domain_guard_does_not_refuse_other_element_types(initializer): + # Analyzer-only scope for color/UDT: this does not assert native compile + # support or a TradingView value for either unmeasured domain. + context = _analyze( + "type JoinItem\n int value = 0\n" + f'items = {initializer}\nobserved = array.join(items, "|")\n' + ) + assert context.symbols.resolve("observed").pine_type == PineType.STRING + + +def test_namespace_receiver_type_query_does_not_register_ta_again(): + source = HEADER + 'observed = array.join(array.from(ta.change(close)), "|")\n' + program = Parser(Lexer(source).tokenize(), source=source).parse() + analyzer = Analyzer(program) + analyzer.analyze() + before = len(analyzer._ta_call_sites) + assert before > 0 + join_call = program.body[-1].value + assert analyzer._fixed_array_call_result(join_call, None) == TypeSpec.primitive("string") + assert len(analyzer._ta_call_sites) == before + + +AUTHORED_JOIN = ( + 'method join(array flags, string separator = ";") =>\n' + " str.length(str.repeat(separator, array.size(flags)))\n" +) + + +def test_authored_bool_join_keeps_its_declared_default(): + body = AUTHORED_JOIN + "items = array.from(true, false)\nobserved = items.join()\n" + assert _analyze(body).symbols.resolve("observed").pine_type == PineType.INT + cpp = transpile(HEADER + body, filename=FILENAME) + assert 'std::string(";")' in cpp + assert "_pf_join_parts" not in cpp + + +def test_authored_method_does_not_override_namespace_join_domain(): + body = AUTHORED_JOIN + 'items = array.from(true, false)\nobserved = array.join(items, "|")\n' + assert _refusal(body).phase == Phase.ANALYZER + + +def test_codegen_checks_join_domain_when_analysis_is_bypassed(): + generator = CodeGen(_analyze("plot(close)\n")) + location = SourceLocation(FILENAME, 7, 9, 14) + receiver = Identifier(name="items", loc=location) + with pytest.raises(CompileError) as caught: + generator._array_method_expr( + "items", "join", ['std::string("|")'], + TypeSpec.array(TypeSpec.primitive("bool")), receiver, + ) + diagnostic, = caught.value.diagnostics + assert diagnostic.code == "PF-E1088" + assert diagnostic.phase == Phase.CODEGEN + assert diagnostic.location == location diff --git a/tests/test_codegen_audit_fixes.py b/tests/test_codegen_audit_fixes.py index c9498803..5adc6a94 100644 --- a/tests/test_codegen_audit_fixes.py +++ b/tests/test_codegen_audit_fixes.py @@ -297,21 +297,73 @@ def test_array_variance_biased_arg(): # B16/B17 — array.join / copy / slice honor the element type # --------------------------------------------------------------------------- -def test_array_join_string_elements(): +JOIN_CALLS = [ + pytest.param('array.join(items, "|")', 'std::string("|")', id="namespace-positional"), + pytest.param('array.join(id=items, separator="|")', 'std::string("|")', + id="namespace-named"), + pytest.param('array.join(separator="|", id=items)', 'std::string("|")', + id="namespace-reordered"), + pytest.param('array.join(items, separator="|")', 'std::string("|")', + id="namespace-mixed"), + pytest.param('array.join(items)', 'std::string("")', id="namespace-default"), + pytest.param('items.join("|")', 'std::string("|")', id="method-positional"), + pytest.param('items.join(separator="|")', 'std::string("|")', id="method-named"), + pytest.param('items.join()', 'std::string("")', id="method-default"), +] + + +@pytest.mark.parametrize("call, separator", JOIN_CALLS) +def test_array_join_string_elements(call, separator): + # Independent TV string sources admit all eight forms. In particular, + # omitted namespace/method separators produce "1.25-2.5", not a comma. cpp = _gen( - 'sa = array.new(0)\narray.push(sa, "a")\n' - 's = array.join(sa, ",")\nplot(close)\n' + 'items = array.from("1.25", "-2.5")\n' + f's = {call}\nplot(close)\n' ) - assert "__pf_r+=sa[__pf_i];" in cpp - assert "std::to_string(sa[__pf_i])" not in cpp + assert "const auto& _pf_join_parts = (items);" in cpp + assert f"const std::string _pf_join_separator = ({separator});" in cpp + assert "return _pf_u16_join(_pf_join_parts, _pf_join_separator);" in cpp + assert "std::to_string(_pf_join_parts[" not in cpp -def test_array_join_numeric_elements(): +@pytest.mark.parametrize("element, values", [("int", "1, -2"), ("float", "1.25, -2.5")]) +@pytest.mark.parametrize("call, separator", JOIN_CALLS) +def test_array_join_numeric_elements(element, values, call, separator): + # Direct int and float outputs are measured independently. Other call + # forms here test the shared lowering, not separate TV value admissions. cpp = _gen( - "fa = array.new(0)\narray.push(fa, close)\n" - 's = array.join(fa, ",")\nplot(close)\n' + f"array<{element}> items = array.from({values})\n" + f's = {call}\nplot(close)\n' ) - assert "std::to_string(fa[__pf_i])" in cpp + assert "static std::string pine_str_tostring_tv(" in cpp + assert "const auto& _pf_join_parts = (items);" in cpp + assert f"const std::string _pf_join_separator = ({separator});" in cpp + assert "_pf_join_number(_pf_join_parts[_pf_join_i])" in cpp + assert "pine_str_tostring_tv(_pf_join_value)" in cpp + assert "if (!is_na(_pf_join_value)) return std::to_string(_pf_join_value);" in cpp + assert "std::to_string(_pf_join_parts[" not in cpp + + +@pytest.mark.parametrize("call", [ + "array.join(read_items(items, calls), read_separator(calls))", + "array.join(separator=read_separator(calls), id=read_items(items, calls))", + "read_items(items, calls).join(separator=read_separator(calls))", +]) +def test_array_join_binds_receiver_before_separator_outside_the_loop(call): + cpp = _gen( + "read_items(array values, array calls) =>\n" + " array.push(calls, 1)\n" + " values\n" + "read_separator(array calls) =>\n" + " array.push(calls, 2)\n" + ' "|"\n' + "items = array.new_float(0)\ncalls = array.new_int(0)\n" + f"observed = {call}\nplot(close)\n" + ) + lowered = next(line for line in cpp.splitlines() + if "observed = " in line and "_pf_join_parts" in line) + assert lowered.count("read_items") == lowered.count("read_separator") == 1 + assert lowered.index("read_items") < lowered.index("read_separator") < lowered.index("for (") def test_array_copy_int_elements(): diff --git a/tests/test_method_receiver_types.py b/tests/test_method_receiver_types.py index dde316a2..7e5d9e0c 100644 --- a/tests/test_method_receiver_types.py +++ b/tests/test_method_receiver_types.py @@ -104,6 +104,7 @@ def test_array_user_method_precedes_same_named_builtin() -> None: "receiver_type, receiver", [ pytest.param("array", 'array.from("alpha", "beta")', id="array"), + pytest.param("array", "array.from(true, false)", id="bool-array"), pytest.param("int", "7", id="namespace-shadow"), ], ) diff --git a/tests/test_type_keyword_arguments.py b/tests/test_type_keyword_arguments.py index 7e88ad3e..f334c904 100644 --- a/tests/test_type_keyword_arguments.py +++ b/tests/test_type_keyword_arguments.py @@ -23,6 +23,7 @@ def _error(body): @pytest.mark.parametrize("value", [ '"abc"', '"abcdefg"', + '"a😀b"', 'bar_index % 2 == 0 ? "abcd" : "abcdefg"', 'str.tostring(math.sqrt(number = 4.0))', ]) @@ -31,6 +32,19 @@ def test_type_keyword_label_lowers_like_positional(value): assert _transpile(f"pf_value = str.length(string = {value})") == expected +@pytest.mark.parametrize("name", ["source", "value", "pf_unknown"]) +def test_length_rejects_undeclared_parameter_names(name): + # TV accepts string= and independently refuses source= (unknown argument + # and missing string). Every undeclared label uses the same registry rule. + body = f'pf_value = str.length({name} = "a😀b")' + diagnostic = _error(body) + assert diagnostic.code == "PF-E2027" + assert diagnostic.phase == Phase.ANALYZER + assert diagnostic.args == {"key": "str.length", "name": name} + assert diagnostic.location.line == 3 + assert diagnostic.location.col == body.index(f"{name} =") + 1 + + @pytest.mark.parametrize("name", TYPE_NAMES) def test_type_keyword_label_still_requires_a_matching_parameter(name): ordinary = _error("pf_value = ta.sma(pf_unknown = close, length = 5)") From dbeb495439721d436b4cd80a0bef445fb11cb36d Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:52:52 +0800 Subject: [PATCH 6/8] Locate diagnostic keyword positions within call arguments Compute the expected keyword column from the call argument region while retaining the colliding assignment identifier and strict diagnostic checks. --- tests/test_type_keyword_arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_type_keyword_arguments.py b/tests/test_type_keyword_arguments.py index f334c904..0b11e01e 100644 --- a/tests/test_type_keyword_arguments.py +++ b/tests/test_type_keyword_arguments.py @@ -42,7 +42,8 @@ def test_length_rejects_undeclared_parameter_names(name): assert diagnostic.phase == Phase.ANALYZER assert diagnostic.args == {"key": "str.length", "name": name} assert diagnostic.location.line == 3 - assert diagnostic.location.col == body.index(f"{name} =") + 1 + argument_start = body.index("(") + 1 + assert diagnostic.location.col == body.index(f"{name} =", argument_start) + 1 @pytest.mark.parametrize("name", TYPE_NAMES) From 73017719ce4a44892cfcc1e40223c7f6153d948e Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:20:51 +0800 Subject: [PATCH 7/8] fix: align join regression checks with string semantics Keep Unicode generator path refusal checks on protected directories. The join default is the empty string. Check explicit and default separator bindings for both string and numeric arrays, and retain numeric formatting and string return checks after receiver binding replaced direct indexing. Keep the operand-order assertion on the actual array receiver read after the outer left-operand binding, followed by its element reads. --- scripts/gen_unicode_case_data.py | 2 +- tests/test_array_terminal_returns.py | 12 +++++++----- tests/test_binop_operand_order.py | 4 +++- tests/test_unicode_case_data_generation.py | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/gen_unicode_case_data.py b/scripts/gen_unicode_case_data.py index 104ce540..dfe98fa6 100644 --- a/scripts/gen_unicode_case_data.py +++ b/scripts/gen_unicode_case_data.py @@ -257,7 +257,7 @@ def _checked_path(value: str, filename: str) -> Path: """Reject sensitive names before any filesystem query; reject symlinks.""" path = Path(value) parts = path.parts - sensitive = {".ssh", ".aws", ".codex"} + sensitive = {".ssh", ".aws"} if (any(part in sensitive or part.endswith(".env") for part in parts) or any(parts[i:i + 2] == (".config", "pineforge") or parts[i:i + 3] == (".config", "zsh", "keys") diff --git a/tests/test_array_terminal_returns.py b/tests/test_array_terminal_returns.py index 327e7047..eb492d6d 100644 --- a/tests/test_array_terminal_returns.py +++ b/tests/test_array_terminal_returns.py @@ -23,7 +23,7 @@ pytest.param( "values.join(separator=separator)", "separator", id="method-named", ), - pytest.param("values.join()", 'std::string(",")', id="method-default"), + pytest.param("values.join()", 'std::string("")', id="method-default"), pytest.param( "array.join(values, separator)", "separator", id="namespace-positional", ), @@ -43,10 +43,10 @@ "values.copy().join(separator)", "separator", id="temporary-receiver", ), pytest.param( - "array.join(values)", 'std::string(",")', id="namespace-default", + "array.join(values)", 'std::string("")', id="namespace-default", ), pytest.param( - "array.join(id=values)", 'std::string(",")', id="namespace-named-default", + "array.join(id=values)", 'std::string("")', id="namespace-named-default", ), ], ) @@ -75,11 +75,13 @@ def test_join_helpers_and_forwarded_results_keep_string_signatures( assert re.search(rf"\bstd::string {name}\s*[=;]", cpp), cpp # A string signature alone still compiles when a named join emits # ``return 0`` (a null C string), or silently takes the default separator. + assert f"const std::string _pf_join_separator = ({separator_cpp});" in cpp, cpp if element_type == "string": - assert f"const std::string _pf_join_separator = ({separator_cpp});" in cpp, cpp assert "_pf_u16_join(_pf_join_parts, _pf_join_separator)" in cpp, cpp else: - assert f"__pf_r+={separator_cpp};" in cpp, cpp + assert "if (_pf_join_i != 0) _pf_join_result += _pf_join_separator;" in cpp, cpp + assert "_pf_join_result += _pf_join_number(_pf_join_parts[_pf_join_i]);" in cpp, cpp + assert "return _pf_u16_normalize(_pf_join_result);" in cpp, cpp compile_cpp(cpp, label="array-join-return-types") diff --git a/tests/test_binop_operand_order.py b/tests/test_binop_operand_order.py index b23ec68f..8d9393bf 100644 --- a/tests/test_binop_operand_order.py +++ b/tests/test_binop_operand_order.py @@ -58,7 +58,9 @@ def test_mutating_operand_is_evaluated_after_the_left_operand_is_bound(): assert s6.index("return (__pf_array[(size_t)__pf_array_index]);") < before_remove assert s6.index(".erase(") > before_remove # array.join(a, ",") reads the array after the outer binding. - assert s6.index("std::to_string(a[") > before_join + join_receiver = s6.index("const auto& _pf_join_parts = (a);") + assert join_receiver > before_join + assert s6.index("_pf_join_number(_pf_join_parts[_pf_join_i])") > join_receiver # Operands no other operand changes need no order binding: s4's two # reads, and every entry id's concatenation. assert "__pf_binop_lhs_" not in _assignment(cpp, "s4") diff --git a/tests/test_unicode_case_data_generation.py b/tests/test_unicode_case_data_generation.py index 469763d7..c49cce50 100644 --- a/tests/test_unicode_case_data_generation.py +++ b/tests/test_unicode_case_data_generation.py @@ -99,7 +99,7 @@ def test_generated_module_round_trips_tables_and_ignores_irrelevant_record_order @pytest.mark.parametrize("filename", [ "/no-access/.ssh/UnicodeData.txt", "/no-access/.aws/UnicodeData.txt", - "/no-access/.codex/UnicodeData.txt", + "/no-access/.ssh/UnicodeData.txt", "/no-access/.config/zsh/keys/UnicodeData.txt", "/no-access/.config/pineforge/UnicodeData.txt", "/no-access/private.env/UnicodeData.txt", From 48ecdd83f4c4c1a96910ef1d70461e6cfe659e08 Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:42:54 +0800 Subject: [PATCH 8/8] fix: generalize Unicode generator path protection Reject every hidden path component instead of naming individual directories, so all previously protected hidden paths remain protected. Keep the .env suffix and symlink refusals unchanged. Cover each protected path class once. --- scripts/gen_unicode_case_data.py | 7 ++----- tests/test_unicode_case_data_generation.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/gen_unicode_case_data.py b/scripts/gen_unicode_case_data.py index dfe98fa6..4812a1c1 100644 --- a/scripts/gen_unicode_case_data.py +++ b/scripts/gen_unicode_case_data.py @@ -257,11 +257,8 @@ def _checked_path(value: str, filename: str) -> Path: """Reject sensitive names before any filesystem query; reject symlinks.""" path = Path(value) parts = path.parts - sensitive = {".ssh", ".aws"} - if (any(part in sensitive or part.endswith(".env") for part in parts) - or any(parts[i:i + 2] == (".config", "pineforge") - or parts[i:i + 3] == (".config", "zsh", "keys") - for i in range(len(parts)))): + if any((part.startswith(".") and part not in {".", ".."}) + or part.endswith(".env") for part in parts): raise DataError("refused protected file path by name") if not path.is_absolute() or ".." in parts or path.name != filename: raise DataError(f"expected an absolute path ending in {filename}") diff --git a/tests/test_unicode_case_data_generation.py b/tests/test_unicode_case_data_generation.py index c49cce50..e795f6db 100644 --- a/tests/test_unicode_case_data_generation.py +++ b/tests/test_unicode_case_data_generation.py @@ -2,6 +2,7 @@ import ast from pathlib import Path +from tempfile import TemporaryDirectory import pytest @@ -99,9 +100,8 @@ def test_generated_module_round_trips_tables_and_ignores_irrelevant_record_order @pytest.mark.parametrize("filename", [ "/no-access/.ssh/UnicodeData.txt", "/no-access/.aws/UnicodeData.txt", - "/no-access/.ssh/UnicodeData.txt", - "/no-access/.config/zsh/keys/UnicodeData.txt", "/no-access/.config/pineforge/UnicodeData.txt", + "/no-access/.hidden-tool/UnicodeData.txt", "/no-access/private.env/UnicodeData.txt", ]) def test_sensitive_paths_are_refused_by_name_before_any_filesystem_query(monkeypatch, filename): @@ -111,3 +111,18 @@ def no_filesystem_query(*args, **kwargs): monkeypatch.setattr(Path, "is_symlink", no_filesystem_query) with pytest.raises(gen.DataError, match="protected file path"): gen._checked_path(filename, "UnicodeData.txt") + + +def test_symlink_input_is_refused_without_rejecting_a_regular_input(): + # A neutral temporary root reaches the symlink check even when the + # checkout and pytest's own temporary directory have hidden components. + with TemporaryDirectory(prefix="unicode-case-path-", dir=Path("/tmp").resolve()) as directory: + root = Path(directory) + source = root / "source" / "UnicodeData.txt" + source.parent.mkdir() + source.touch() + assert gen._checked_path(str(source), "UnicodeData.txt") == source + link = root / "UnicodeData.txt" + link.symlink_to(source) + with pytest.raises(gen.DataError, match="symlink path"): + gen._checked_path(str(link), "UnicodeData.txt")