diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c11b388..ff41de09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,69 @@ supported as exact pairs; on the 0.x line they are independent. See the uses their generated spelling. Declarations, calls and history storage use the same allocated identities. Regenerate C++ and relink affected strategies. +- Requested helpers preserve declared string argument types when inferring + local variables, including string `na` values, so these expressions emit + string storage and string missing values. + +- String `+`, `+=` and string-array joins recompose adjacent UTF-16 surrogate + halves, including across array parts and separators. Unpaired halves are + preserved; string search predicates compare UTF-16 units. Regenerate C++ + and relink affected strategies. +- Persistent string locals initialized from helper parameters retain their + storage type in compound assignments and `request.security()` expressions. +- Array joins retain string return types through helper and method forwarding. + Named, mixed and reordered receiver/separator arguments use their declared + slots; an omitted separator is empty. Numeric elements use Pine number + formatting without fixed trailing zeros, and receiver and separator are + evaluated once in parameter order, including for empty arrays. + Builtin joins refuse boolean arrays during translation; authored methods + retain their own overloads and defaults. +- `str.length` validates named arguments against its declared `string` + parameter; unknown names produce the standard located argument diagnostic. +- `str.upper` and `str.lower` use pinned Unicode 14.0.0 mappings and properties, + including supplementary letters, expansions and contextual final sigma. + Translating Python versions and the native locale do not select the data. + The context crosses decimal runs; locale tailoring and normalization are + not applied. The data pin does not identify TradingView's whole dialect. +- `str.match` lowers its literals, character classes, ranges and repeated + atoms as Unicode characters, including complete supplementary characters, + so matches no longer depend on UTF-8 byte lengths. Word classes remain ASCII; + string lengths and substring offsets still count UTF-16 units. + Matches return the complete match while preserving user captures and + backreferences for pattern evaluation. + Regenerate C++ and relink affected strategies to use this correction. +- String input defaults, options and titles preserve embedded NUL bytes in + getters, metadata and checked receipts. Internal timeframe and footprint + column consumers carry literal byte lengths; their validation is unchanged. + Public C input setters retain their NUL-terminated contract. +- `str.split` applies string context to missing arguments, including literal + `na` in positional, named and helper calls. + +- `str.pos` returns integer `na` when the substring is absent, so `na()` detects + the missing match and `nz()` uses its replacement (#382, item 5). It returned + `-1`; found positions retain their UTF-16 offsets. The TradingView expectation + comes from the reporter's readings. Regenerate C++ and relink affected + strategies. + +- `str.substring` checks an explicit `na` end before boundary arithmetic and + returns the engine's canonical string `na` (stored as the empty string). + An `na` start becomes zero, including a wide integer `na` propagated from + `str.pos(...) + 1`, so the two-argument form returns the whole text. Finite + endpoints are ordered before computing the count; an end before the start + stops the run. UTF-16 slicing, end clamping and invalid-begin stops use the + existing helper. The oracle is the reporter's #382 item 9 readings. +- `str.substring` refuses a provably negative constant `begin_pos` during + analysis, including integer arithmetic, constant aliases and named arguments. + Dynamic negative starts retain their runtime error, and missing positions + retain their existing handling. +- `str.trim` additionally removes VT (U+000B), FF (U+000C), U+00A0, U+1680, + U+2003, U+2028, U+3000 and U+FEFF from the ends. Interior bytes and kept + characters are preserved; other Unicode spaces remain unchanged (#382, item 8). + +- Reads of authored variables named like bar fields use their lexical + declarations and types. A string binding passed to `array.from()` previously + read the numeric bar field, which could fail to compile or produce a wrong + value. Leaving a block that shadows a builtin restores the builtin. - A zone name the engine rejects now makes calendar-form `timestamp()` stop the run at its first evaluation, instead of silently calculating a UTC value. The paired engine reports a typed timezone error identifying the failed bar. diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index 5f99998b..f8f85b7f 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -6474,24 +6474,26 @@ def _visit_MethodDef(self, node) -> PineType: def _top_level_branch_needs_lexical_scope( self, body: list[ASTNode] ) -> bool: - """Whether a top-level branch declares over a direct script binding. + """Whether a top-level branch declares over a script binding or builtin. Functions already own a symbol-table scope and ``for``/``for in`` create one explicitly. Top-level if/while/switch branches historically reused the global Scope, so a local declaration could overwrite the analyzer's outer Symbol even though codegen emits a C++ lexical local. - Add a child scope only for the collision shape handled by this change; - every unrelated program keeps the established analysis path. + Builtins need the same child scope as authored outer bindings, so + leaving a branch restores the builtin and its type. """ if self._collection_scope_stack: return False for stmt in body: if (isinstance(stmt, VarDecl) - and stmt.name in self._direct_program_binding_names): + and (stmt.name in self._direct_program_binding_names + or stmt.name in BUILTIN_VARS)): return True if (isinstance(stmt, TupleAssign) and any( name in self._direct_program_binding_names + or name in BUILTIN_VARS for name in stmt.names if name != "_" )): @@ -7696,6 +7698,8 @@ def _visit_FuncCall(self, node: FuncCall) -> PineType: if isinstance(obj, Identifier) and obj.name == "str": for arg in node.args: self._visit(arg) + for value in node.kwargs.values(): + self._visit(value) # Most str.* return a string, but predicates and index helpers # are scalar. Keep this aligned with signatures.py and the C++ # emitter's _infer_type path. @@ -8015,6 +8019,12 @@ def _visit_FuncCall(self, node: FuncCall) -> PineType: ) return PineType.UNKNOWN return receiver_pine_type + # Preserve fixed scalar array results in the coarse return-type + # visitor as well as TypeSpec inference. Authored methods above + # retain precedence over builtin names. + fixed_array_result = self._fixed_array_call_result(node, receiver_spec) + if fixed_array_result is not None: + return self._element_pine_type(fixed_array_result) # Matrix method dispatch: ``m.get(0, 0)`` on ``matrix`` must # type as INT, not VOID, so ``v = m.get(...)`` propagates the # element PineType. ``_type_spec_from_expr`` already carries the diff --git a/pineforge_codegen/analyzer/diagnostics.py b/pineforge_codegen/analyzer/diagnostics.py index 603567ee..ada0881f 100644 --- a/pineforge_codegen/analyzer/diagnostics.py +++ b/pineforge_codegen/analyzer/diagnostics.py @@ -45,7 +45,8 @@ class DiagnosticsHelper: the diagnostic surface easy to audit. """ - def _error(self, message: str, loc: SourceLocation | None = None) -> None: + def _error(self, message: str, loc: SourceLocation | None = None, + *, hint: str | None = None) -> None: """Raise a compile error.""" if loc is None: loc = SourceLocation(file=self._filename, line=1, col=1, end_col=1) @@ -54,6 +55,7 @@ def _error(self, message: str, loc: SourceLocation | None = None) -> None: phase=Phase.ANALYZER, location=loc, message=message, + hint=hint, ) raise CompileError([diag]) diff --git a/pineforge_codegen/analyzer/operators.py b/pineforge_codegen/analyzer/operators.py index d5497b51..b2cedcca 100644 --- a/pineforge_codegen/analyzer/operators.py +++ b/pineforge_codegen/analyzer/operators.py @@ -6,6 +6,7 @@ # Only argument-independent scalar results belong here. The signature # registry also contains storage-oriented approximations of polymorphic calls. _FIXED_RESULT_CALLS = { + "array": frozenset({"join"}), "str": frozenset({ "tostring", "format", "format_time", "length", "contains", "startswith", "endswith", "pos", "substring", "replace", "replace_all", "lower", @@ -53,6 +54,8 @@ def fixed_builtin_result(namespace: str, name: str, arity: int) -> TypeSpec | No else "int" if name in {"barssince", "highestbars", "lowestbars"} else "float" ) + elif namespace == "array": + result = "string" elif namespace == "str": result = ( "bool" if name in {"contains", "startswith", "endswith"} diff --git a/pineforge_codegen/analyzer/types.py b/pineforge_codegen/analyzer/types.py index 1c19d353..e242bed9 100644 --- a/pineforge_codegen/analyzer/types.py +++ b/pineforge_codegen/analyzer/types.py @@ -46,16 +46,18 @@ SwitchStmt, TupleAssign, TupleLiteral, UnaryOp, VarDecl, ) from ..errors import Phase +from ..intrinsic_binding import array_method_domain_error from ..limits import iter_ast_nodes from ..matrix_overloads import matrix_sum_has_rhs from ..reference_casts import na_reference_cast_kind +from ..pine_spelling import expr_start from ..symbols import ( PineType, TypeSpec, method_receiver_type_name, unify_type_specs, ) -from .operators import join_value_types, reference_value_type +from .operators import fixed_builtin_result, join_value_types, reference_value_type # Drawing-objects-as-data type names (spec §4.1). Defined locally — the # analyzer must not import from ``codegen`` (codegen imports analyzer, so the @@ -196,6 +198,48 @@ def _template_args_from_call(self, node: FuncCall) -> list[str]: raw = ann.get("template_args") or [] return [str(x).replace(" ", "") for x in raw] + def _fixed_array_call_result( + self, call: FuncCall, receiver_spec: TypeSpec | None, + ) -> TypeSpec | None: + """Validate a builtin receiver and read its fixed result after methods.""" + callee = call.callee + if not isinstance(callee, MemberAccess): + return None + namespace_call = ( + isinstance(callee.object, Identifier) + and callee.object.name == "array" + and self._symbols.resolve("array") is None + and self._value_bindings.declaration_bindings.get( + id(callee.object) + ) is None + ) + if not namespace_call and ( + receiver_spec is None or receiver_spec.kind != "array" + ): + return None + if callee.member == "join": + receiver = ( + call.args[0] if call.args else call.kwargs.get("id") + ) if namespace_call else callee.object + # A namespace receiver can be a constructor containing TA or + # request calls already visited above. Read its type without + # registering those calls again; preserve call-site type overlays. + query = (self if isinstance(self, OperatorTypeQuery) + else OperatorTypeQuery(self, {})) + actual_spec = ( + query._type_spec_from_expr(receiver) + if namespace_call else receiver_spec + ) + problem = array_method_domain_error(callee.member, actual_spec) + if problem is not None: + self._error( + problem[0], expr_start(receiver).loc or call.loc, + hint=problem[1], + ) + return fixed_builtin_result( + "array", callee.member, len(call.args) + len(call.kwargs), + ) + def _array_from_element_spec(self, value: ASTNode | None) -> TypeSpec | None: """Exact scalar element type for one ``array.from`` argument. @@ -612,6 +656,11 @@ def direct_user_udt_ctor_name(node: ASTNode) -> str | None: # Its body-derived return type is not available yet, but a # same-named builtin must not lend it a false type. return None + fixed_array_result = self._fixed_array_call_result( + value, typed_receiver_spec, + ) + if fixed_array_result is not None: + return fixed_array_result if ( typed_receiver_spec is not None and typed_receiver_spec.kind == "udt" diff --git a/pineforge_codegen/analyzer/v6_forms.py b/pineforge_codegen/analyzer/v6_forms.py index 42cd774d..2f93341d 100644 --- a/pineforge_codegen/analyzer/v6_forms.py +++ b/pineforge_codegen/analyzer/v6_forms.py @@ -9,7 +9,9 @@ from ..diagnostic_codes import render_diagnostic from ..errors import CompileError, Diagnostic, Level, Phase, SourceLocation from ..limits import TimeBudget, syntax_children +from ..loop_types import IntegerConstants, LoopTypeSources from ..pine_spelling import expr_start +from ..signatures import STR_FUNCTIONS def _refuse(code: str, node: ASTNode, filename: str, @@ -114,3 +116,63 @@ def validate_v6_forms(program: Program, filename: str, value for value in notes.get("param_defaults", ()) if isinstance(value, ASTNode) ) + + +def _substring_begin(node: FuncCall) -> ASTNode | None: + """Read the begin slot of a call that binds to a declared signature.""" + for signature in STR_FUNCTIONS["substring"].signatures: + names = [parameter.name for parameter in signature.params] + if (len(node.args) > len(names) + or set(node.kwargs) - set(names) + or set(names[:len(node.args)]) & set(node.kwargs)): + continue + bound = dict(zip(names, node.args)) + bound.update(node.kwargs) + if any(parameter.default is None and parameter.name not in bound + for parameter in signature.params): + continue + return bound["begin_pos"] + # Malformed calls retain their existing argument-binding handling. + return None + + +def validate_v6_constant_arguments( + program: Program, bindings: dict[int, tuple[int, str] | None], + filename: str, budget: TimeBudget | None = None, +) -> None: + """Check constant domains after lexical resolution, before call lowering.""" + if program.version != 6: + return + constants = None + stack = [program] + while stack: + node = stack.pop() + if budget is not None: + budget.check(node.loc, Phase.ANALYZER) + notes = node.annotations or {} + if notes.get("pine_version") == 5: + continue + if (isinstance(node, FuncCall) and _callee_name(node) == "str.substring" + and bindings.get(id(node.callee.object)) is None): + begin = _substring_begin(node) + if begin is not None: + if constants is None: + constants = IntegerConstants(LoopTypeSources(program), bindings, budget) + value = constants.value(begin) + if value is not None and value < 0: + raise CompileError([Diagnostic( + level=Level.ERROR, + phase=Phase.ANALYZER, + location=(expr_start(begin).loc or begin.loc or node.loc + or SourceLocation(filename, 1, 1, 1)), + message=( + "str.substring() requires 'begin_pos' " + "to be greater than or equal to 0." + ), + )]) + stack.extend(syntax_children(node)) + if isinstance(node, (FuncDef, MethodDef)): + stack.extend( + value for value in notes.get("param_defaults", ()) + if isinstance(value, ASTNode) + ) diff --git a/pineforge_codegen/analyzer/value_bindings.py b/pineforge_codegen/analyzer/value_bindings.py index 389d5106..66700e5e 100644 --- a/pineforge_codegen/analyzer/value_bindings.py +++ b/pineforge_codegen/analyzer/value_bindings.py @@ -230,7 +230,7 @@ def _missing(self, name: str, scope: _Scope, position: int, def resolve(self) -> None: if self.program.version != 6: return # Analyzer owns missing/unsupported-version diagnostics. - from .v6_forms import validate_v6_forms + from .v6_forms import validate_v6_constant_arguments, validate_v6_forms validate_v6_forms(self.program, self._filename, self.budget) # Import after Analyzer has loaded; reuse its namespace inventory. from .base import _BUILTIN_NAMESPACE_NAMES @@ -266,6 +266,9 @@ def resolve(self) -> None: ) node.annotations = {**(node.annotations or {}), _VALUE_USE: origin} self.check_use(node) + validate_v6_constant_arguments( + self.program, self.declaration_bindings, self._filename, self.budget, + ) def _body(self, body: list, scope: _Scope) -> None: for stmt in body: diff --git a/pineforge_codegen/codegen/_unicode_case_data.py b/pineforge_codegen/codegen/_unicode_case_data.py new file mode 100644 index 00000000..5696c7ec --- /dev/null +++ b/pineforge_codegen/codegen/_unicode_case_data.py @@ -0,0 +1,3657 @@ +"""Generated Unicode 14.0.0 casing data; do not edit table entries. + +Source: https://www.unicode.org/Public/14.0.0/ucd/ +Copyright 2021 Unicode, Inc. +Unicode data terms: https://www.unicode.org/terms_of_use.html +Regenerate with scripts/gen_unicode_case_data.py using pinned inputs.""" + +DATA_FORMAT = 1 +DATA_VERSION = "14.0.0" +SOURCE_SHA256 = ( + ("UnicodeData.txt", "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +SOURCE_BYTE_COUNTS = ( + ("UnicodeData.txt", 1897793), + ("SpecialCasing.txt", 16830), + ("DerivedCoreProperties.txt", 1040794), +) +EXCLUDED_LOCALE_RULES = ( + "0049: az Not_Before_Dot", + "0049: lt More_Above", + "0049: tr Not_Before_Dot", + "004A: lt More_Above", + "0069: az", + "0069: tr", + "00CC: lt", + "00CD: lt", + "0128: lt", + "012E: lt More_Above", + "0130: az", + "0130: tr", + "0307: az After_I", + "0307: lt After_Soft_Dotted", + "0307: tr After_I", +) + +LOWER_ROWS = ( + (0x41, 0x1, 0x61), + (0x42, 0x1, 0x62), + (0x43, 0x1, 0x63), + (0x44, 0x1, 0x64), + (0x45, 0x1, 0x65), + (0x46, 0x1, 0x66), + (0x47, 0x1, 0x67), + (0x48, 0x1, 0x68), + (0x49, 0x1, 0x69), + (0x4A, 0x1, 0x6A), + (0x4B, 0x1, 0x6B), + (0x4C, 0x1, 0x6C), + (0x4D, 0x1, 0x6D), + (0x4E, 0x1, 0x6E), + (0x4F, 0x1, 0x6F), + (0x50, 0x1, 0x70), + (0x51, 0x1, 0x71), + (0x52, 0x1, 0x72), + (0x53, 0x1, 0x73), + (0x54, 0x1, 0x74), + (0x55, 0x1, 0x75), + (0x56, 0x1, 0x76), + (0x57, 0x1, 0x77), + (0x58, 0x1, 0x78), + (0x59, 0x1, 0x79), + (0x5A, 0x1, 0x7A), + (0xC0, 0x1, 0xE0), + (0xC1, 0x1, 0xE1), + (0xC2, 0x1, 0xE2), + (0xC3, 0x1, 0xE3), + (0xC4, 0x1, 0xE4), + (0xC5, 0x1, 0xE5), + (0xC6, 0x1, 0xE6), + (0xC7, 0x1, 0xE7), + (0xC8, 0x1, 0xE8), + (0xC9, 0x1, 0xE9), + (0xCA, 0x1, 0xEA), + (0xCB, 0x1, 0xEB), + (0xCC, 0x1, 0xEC), + (0xCD, 0x1, 0xED), + (0xCE, 0x1, 0xEE), + (0xCF, 0x1, 0xEF), + (0xD0, 0x1, 0xF0), + (0xD1, 0x1, 0xF1), + (0xD2, 0x1, 0xF2), + (0xD3, 0x1, 0xF3), + (0xD4, 0x1, 0xF4), + (0xD5, 0x1, 0xF5), + (0xD6, 0x1, 0xF6), + (0xD8, 0x1, 0xF8), + (0xD9, 0x1, 0xF9), + (0xDA, 0x1, 0xFA), + (0xDB, 0x1, 0xFB), + (0xDC, 0x1, 0xFC), + (0xDD, 0x1, 0xFD), + (0xDE, 0x1, 0xFE), + (0x100, 0x1, 0x101), + (0x102, 0x1, 0x103), + (0x104, 0x1, 0x105), + (0x106, 0x1, 0x107), + (0x108, 0x1, 0x109), + (0x10A, 0x1, 0x10B), + (0x10C, 0x1, 0x10D), + (0x10E, 0x1, 0x10F), + (0x110, 0x1, 0x111), + (0x112, 0x1, 0x113), + (0x114, 0x1, 0x115), + (0x116, 0x1, 0x117), + (0x118, 0x1, 0x119), + (0x11A, 0x1, 0x11B), + (0x11C, 0x1, 0x11D), + (0x11E, 0x1, 0x11F), + (0x120, 0x1, 0x121), + (0x122, 0x1, 0x123), + (0x124, 0x1, 0x125), + (0x126, 0x1, 0x127), + (0x128, 0x1, 0x129), + (0x12A, 0x1, 0x12B), + (0x12C, 0x1, 0x12D), + (0x12E, 0x1, 0x12F), + (0x130, 0x2, 0x69, 0x307), + (0x132, 0x1, 0x133), + (0x134, 0x1, 0x135), + (0x136, 0x1, 0x137), + (0x139, 0x1, 0x13A), + (0x13B, 0x1, 0x13C), + (0x13D, 0x1, 0x13E), + (0x13F, 0x1, 0x140), + (0x141, 0x1, 0x142), + (0x143, 0x1, 0x144), + (0x145, 0x1, 0x146), + (0x147, 0x1, 0x148), + (0x14A, 0x1, 0x14B), + (0x14C, 0x1, 0x14D), + (0x14E, 0x1, 0x14F), + (0x150, 0x1, 0x151), + (0x152, 0x1, 0x153), + (0x154, 0x1, 0x155), + (0x156, 0x1, 0x157), + (0x158, 0x1, 0x159), + (0x15A, 0x1, 0x15B), + (0x15C, 0x1, 0x15D), + (0x15E, 0x1, 0x15F), + (0x160, 0x1, 0x161), + (0x162, 0x1, 0x163), + (0x164, 0x1, 0x165), + (0x166, 0x1, 0x167), + (0x168, 0x1, 0x169), + (0x16A, 0x1, 0x16B), + (0x16C, 0x1, 0x16D), + (0x16E, 0x1, 0x16F), + (0x170, 0x1, 0x171), + (0x172, 0x1, 0x173), + (0x174, 0x1, 0x175), + (0x176, 0x1, 0x177), + (0x178, 0x1, 0xFF), + (0x179, 0x1, 0x17A), + (0x17B, 0x1, 0x17C), + (0x17D, 0x1, 0x17E), + (0x181, 0x1, 0x253), + (0x182, 0x1, 0x183), + (0x184, 0x1, 0x185), + (0x186, 0x1, 0x254), + (0x187, 0x1, 0x188), + (0x189, 0x1, 0x256), + (0x18A, 0x1, 0x257), + (0x18B, 0x1, 0x18C), + (0x18E, 0x1, 0x1DD), + (0x18F, 0x1, 0x259), + (0x190, 0x1, 0x25B), + (0x191, 0x1, 0x192), + (0x193, 0x1, 0x260), + (0x194, 0x1, 0x263), + (0x196, 0x1, 0x269), + (0x197, 0x1, 0x268), + (0x198, 0x1, 0x199), + (0x19C, 0x1, 0x26F), + (0x19D, 0x1, 0x272), + (0x19F, 0x1, 0x275), + (0x1A0, 0x1, 0x1A1), + (0x1A2, 0x1, 0x1A3), + (0x1A4, 0x1, 0x1A5), + (0x1A6, 0x1, 0x280), + (0x1A7, 0x1, 0x1A8), + (0x1A9, 0x1, 0x283), + (0x1AC, 0x1, 0x1AD), + (0x1AE, 0x1, 0x288), + (0x1AF, 0x1, 0x1B0), + (0x1B1, 0x1, 0x28A), + (0x1B2, 0x1, 0x28B), + (0x1B3, 0x1, 0x1B4), + (0x1B5, 0x1, 0x1B6), + (0x1B7, 0x1, 0x292), + (0x1B8, 0x1, 0x1B9), + (0x1BC, 0x1, 0x1BD), + (0x1C4, 0x1, 0x1C6), + (0x1C5, 0x1, 0x1C6), + (0x1C7, 0x1, 0x1C9), + (0x1C8, 0x1, 0x1C9), + (0x1CA, 0x1, 0x1CC), + (0x1CB, 0x1, 0x1CC), + (0x1CD, 0x1, 0x1CE), + (0x1CF, 0x1, 0x1D0), + (0x1D1, 0x1, 0x1D2), + (0x1D3, 0x1, 0x1D4), + (0x1D5, 0x1, 0x1D6), + (0x1D7, 0x1, 0x1D8), + (0x1D9, 0x1, 0x1DA), + (0x1DB, 0x1, 0x1DC), + (0x1DE, 0x1, 0x1DF), + (0x1E0, 0x1, 0x1E1), + (0x1E2, 0x1, 0x1E3), + (0x1E4, 0x1, 0x1E5), + (0x1E6, 0x1, 0x1E7), + (0x1E8, 0x1, 0x1E9), + (0x1EA, 0x1, 0x1EB), + (0x1EC, 0x1, 0x1ED), + (0x1EE, 0x1, 0x1EF), + (0x1F1, 0x1, 0x1F3), + (0x1F2, 0x1, 0x1F3), + (0x1F4, 0x1, 0x1F5), + (0x1F6, 0x1, 0x195), + (0x1F7, 0x1, 0x1BF), + (0x1F8, 0x1, 0x1F9), + (0x1FA, 0x1, 0x1FB), + (0x1FC, 0x1, 0x1FD), + (0x1FE, 0x1, 0x1FF), + (0x200, 0x1, 0x201), + (0x202, 0x1, 0x203), + (0x204, 0x1, 0x205), + (0x206, 0x1, 0x207), + (0x208, 0x1, 0x209), + (0x20A, 0x1, 0x20B), + (0x20C, 0x1, 0x20D), + (0x20E, 0x1, 0x20F), + (0x210, 0x1, 0x211), + (0x212, 0x1, 0x213), + (0x214, 0x1, 0x215), + (0x216, 0x1, 0x217), + (0x218, 0x1, 0x219), + (0x21A, 0x1, 0x21B), + (0x21C, 0x1, 0x21D), + (0x21E, 0x1, 0x21F), + (0x220, 0x1, 0x19E), + (0x222, 0x1, 0x223), + (0x224, 0x1, 0x225), + (0x226, 0x1, 0x227), + (0x228, 0x1, 0x229), + (0x22A, 0x1, 0x22B), + (0x22C, 0x1, 0x22D), + (0x22E, 0x1, 0x22F), + (0x230, 0x1, 0x231), + (0x232, 0x1, 0x233), + (0x23A, 0x1, 0x2C65), + (0x23B, 0x1, 0x23C), + (0x23D, 0x1, 0x19A), + (0x23E, 0x1, 0x2C66), + (0x241, 0x1, 0x242), + (0x243, 0x1, 0x180), + (0x244, 0x1, 0x289), + (0x245, 0x1, 0x28C), + (0x246, 0x1, 0x247), + (0x248, 0x1, 0x249), + (0x24A, 0x1, 0x24B), + (0x24C, 0x1, 0x24D), + (0x24E, 0x1, 0x24F), + (0x370, 0x1, 0x371), + (0x372, 0x1, 0x373), + (0x376, 0x1, 0x377), + (0x37F, 0x1, 0x3F3), + (0x386, 0x1, 0x3AC), + (0x388, 0x1, 0x3AD), + (0x389, 0x1, 0x3AE), + (0x38A, 0x1, 0x3AF), + (0x38C, 0x1, 0x3CC), + (0x38E, 0x1, 0x3CD), + (0x38F, 0x1, 0x3CE), + (0x391, 0x1, 0x3B1), + (0x392, 0x1, 0x3B2), + (0x393, 0x1, 0x3B3), + (0x394, 0x1, 0x3B4), + (0x395, 0x1, 0x3B5), + (0x396, 0x1, 0x3B6), + (0x397, 0x1, 0x3B7), + (0x398, 0x1, 0x3B8), + (0x399, 0x1, 0x3B9), + (0x39A, 0x1, 0x3BA), + (0x39B, 0x1, 0x3BB), + (0x39C, 0x1, 0x3BC), + (0x39D, 0x1, 0x3BD), + (0x39E, 0x1, 0x3BE), + (0x39F, 0x1, 0x3BF), + (0x3A0, 0x1, 0x3C0), + (0x3A1, 0x1, 0x3C1), + (0x3A3, 0x1, 0x3C3), + (0x3A4, 0x1, 0x3C4), + (0x3A5, 0x1, 0x3C5), + (0x3A6, 0x1, 0x3C6), + (0x3A7, 0x1, 0x3C7), + (0x3A8, 0x1, 0x3C8), + (0x3A9, 0x1, 0x3C9), + (0x3AA, 0x1, 0x3CA), + (0x3AB, 0x1, 0x3CB), + (0x3CF, 0x1, 0x3D7), + (0x3D8, 0x1, 0x3D9), + (0x3DA, 0x1, 0x3DB), + (0x3DC, 0x1, 0x3DD), + (0x3DE, 0x1, 0x3DF), + (0x3E0, 0x1, 0x3E1), + (0x3E2, 0x1, 0x3E3), + (0x3E4, 0x1, 0x3E5), + (0x3E6, 0x1, 0x3E7), + (0x3E8, 0x1, 0x3E9), + (0x3EA, 0x1, 0x3EB), + (0x3EC, 0x1, 0x3ED), + (0x3EE, 0x1, 0x3EF), + (0x3F4, 0x1, 0x3B8), + (0x3F7, 0x1, 0x3F8), + (0x3F9, 0x1, 0x3F2), + (0x3FA, 0x1, 0x3FB), + (0x3FD, 0x1, 0x37B), + (0x3FE, 0x1, 0x37C), + (0x3FF, 0x1, 0x37D), + (0x400, 0x1, 0x450), + (0x401, 0x1, 0x451), + (0x402, 0x1, 0x452), + (0x403, 0x1, 0x453), + (0x404, 0x1, 0x454), + (0x405, 0x1, 0x455), + (0x406, 0x1, 0x456), + (0x407, 0x1, 0x457), + (0x408, 0x1, 0x458), + (0x409, 0x1, 0x459), + (0x40A, 0x1, 0x45A), + (0x40B, 0x1, 0x45B), + (0x40C, 0x1, 0x45C), + (0x40D, 0x1, 0x45D), + (0x40E, 0x1, 0x45E), + (0x40F, 0x1, 0x45F), + (0x410, 0x1, 0x430), + (0x411, 0x1, 0x431), + (0x412, 0x1, 0x432), + (0x413, 0x1, 0x433), + (0x414, 0x1, 0x434), + (0x415, 0x1, 0x435), + (0x416, 0x1, 0x436), + (0x417, 0x1, 0x437), + (0x418, 0x1, 0x438), + (0x419, 0x1, 0x439), + (0x41A, 0x1, 0x43A), + (0x41B, 0x1, 0x43B), + (0x41C, 0x1, 0x43C), + (0x41D, 0x1, 0x43D), + (0x41E, 0x1, 0x43E), + (0x41F, 0x1, 0x43F), + (0x420, 0x1, 0x440), + (0x421, 0x1, 0x441), + (0x422, 0x1, 0x442), + (0x423, 0x1, 0x443), + (0x424, 0x1, 0x444), + (0x425, 0x1, 0x445), + (0x426, 0x1, 0x446), + (0x427, 0x1, 0x447), + (0x428, 0x1, 0x448), + (0x429, 0x1, 0x449), + (0x42A, 0x1, 0x44A), + (0x42B, 0x1, 0x44B), + (0x42C, 0x1, 0x44C), + (0x42D, 0x1, 0x44D), + (0x42E, 0x1, 0x44E), + (0x42F, 0x1, 0x44F), + (0x460, 0x1, 0x461), + (0x462, 0x1, 0x463), + (0x464, 0x1, 0x465), + (0x466, 0x1, 0x467), + (0x468, 0x1, 0x469), + (0x46A, 0x1, 0x46B), + (0x46C, 0x1, 0x46D), + (0x46E, 0x1, 0x46F), + (0x470, 0x1, 0x471), + (0x472, 0x1, 0x473), + (0x474, 0x1, 0x475), + (0x476, 0x1, 0x477), + (0x478, 0x1, 0x479), + (0x47A, 0x1, 0x47B), + (0x47C, 0x1, 0x47D), + (0x47E, 0x1, 0x47F), + (0x480, 0x1, 0x481), + (0x48A, 0x1, 0x48B), + (0x48C, 0x1, 0x48D), + (0x48E, 0x1, 0x48F), + (0x490, 0x1, 0x491), + (0x492, 0x1, 0x493), + (0x494, 0x1, 0x495), + (0x496, 0x1, 0x497), + (0x498, 0x1, 0x499), + (0x49A, 0x1, 0x49B), + (0x49C, 0x1, 0x49D), + (0x49E, 0x1, 0x49F), + (0x4A0, 0x1, 0x4A1), + (0x4A2, 0x1, 0x4A3), + (0x4A4, 0x1, 0x4A5), + (0x4A6, 0x1, 0x4A7), + (0x4A8, 0x1, 0x4A9), + (0x4AA, 0x1, 0x4AB), + (0x4AC, 0x1, 0x4AD), + (0x4AE, 0x1, 0x4AF), + (0x4B0, 0x1, 0x4B1), + (0x4B2, 0x1, 0x4B3), + (0x4B4, 0x1, 0x4B5), + (0x4B6, 0x1, 0x4B7), + (0x4B8, 0x1, 0x4B9), + (0x4BA, 0x1, 0x4BB), + (0x4BC, 0x1, 0x4BD), + (0x4BE, 0x1, 0x4BF), + (0x4C0, 0x1, 0x4CF), + (0x4C1, 0x1, 0x4C2), + (0x4C3, 0x1, 0x4C4), + (0x4C5, 0x1, 0x4C6), + (0x4C7, 0x1, 0x4C8), + (0x4C9, 0x1, 0x4CA), + (0x4CB, 0x1, 0x4CC), + (0x4CD, 0x1, 0x4CE), + (0x4D0, 0x1, 0x4D1), + (0x4D2, 0x1, 0x4D3), + (0x4D4, 0x1, 0x4D5), + (0x4D6, 0x1, 0x4D7), + (0x4D8, 0x1, 0x4D9), + (0x4DA, 0x1, 0x4DB), + (0x4DC, 0x1, 0x4DD), + (0x4DE, 0x1, 0x4DF), + (0x4E0, 0x1, 0x4E1), + (0x4E2, 0x1, 0x4E3), + (0x4E4, 0x1, 0x4E5), + (0x4E6, 0x1, 0x4E7), + (0x4E8, 0x1, 0x4E9), + (0x4EA, 0x1, 0x4EB), + (0x4EC, 0x1, 0x4ED), + (0x4EE, 0x1, 0x4EF), + (0x4F0, 0x1, 0x4F1), + (0x4F2, 0x1, 0x4F3), + (0x4F4, 0x1, 0x4F5), + (0x4F6, 0x1, 0x4F7), + (0x4F8, 0x1, 0x4F9), + (0x4FA, 0x1, 0x4FB), + (0x4FC, 0x1, 0x4FD), + (0x4FE, 0x1, 0x4FF), + (0x500, 0x1, 0x501), + (0x502, 0x1, 0x503), + (0x504, 0x1, 0x505), + (0x506, 0x1, 0x507), + (0x508, 0x1, 0x509), + (0x50A, 0x1, 0x50B), + (0x50C, 0x1, 0x50D), + (0x50E, 0x1, 0x50F), + (0x510, 0x1, 0x511), + (0x512, 0x1, 0x513), + (0x514, 0x1, 0x515), + (0x516, 0x1, 0x517), + (0x518, 0x1, 0x519), + (0x51A, 0x1, 0x51B), + (0x51C, 0x1, 0x51D), + (0x51E, 0x1, 0x51F), + (0x520, 0x1, 0x521), + (0x522, 0x1, 0x523), + (0x524, 0x1, 0x525), + (0x526, 0x1, 0x527), + (0x528, 0x1, 0x529), + (0x52A, 0x1, 0x52B), + (0x52C, 0x1, 0x52D), + (0x52E, 0x1, 0x52F), + (0x531, 0x1, 0x561), + (0x532, 0x1, 0x562), + (0x533, 0x1, 0x563), + (0x534, 0x1, 0x564), + (0x535, 0x1, 0x565), + (0x536, 0x1, 0x566), + (0x537, 0x1, 0x567), + (0x538, 0x1, 0x568), + (0x539, 0x1, 0x569), + (0x53A, 0x1, 0x56A), + (0x53B, 0x1, 0x56B), + (0x53C, 0x1, 0x56C), + (0x53D, 0x1, 0x56D), + (0x53E, 0x1, 0x56E), + (0x53F, 0x1, 0x56F), + (0x540, 0x1, 0x570), + (0x541, 0x1, 0x571), + (0x542, 0x1, 0x572), + (0x543, 0x1, 0x573), + (0x544, 0x1, 0x574), + (0x545, 0x1, 0x575), + (0x546, 0x1, 0x576), + (0x547, 0x1, 0x577), + (0x548, 0x1, 0x578), + (0x549, 0x1, 0x579), + (0x54A, 0x1, 0x57A), + (0x54B, 0x1, 0x57B), + (0x54C, 0x1, 0x57C), + (0x54D, 0x1, 0x57D), + (0x54E, 0x1, 0x57E), + (0x54F, 0x1, 0x57F), + (0x550, 0x1, 0x580), + (0x551, 0x1, 0x581), + (0x552, 0x1, 0x582), + (0x553, 0x1, 0x583), + (0x554, 0x1, 0x584), + (0x555, 0x1, 0x585), + (0x556, 0x1, 0x586), + (0x10A0, 0x1, 0x2D00), + (0x10A1, 0x1, 0x2D01), + (0x10A2, 0x1, 0x2D02), + (0x10A3, 0x1, 0x2D03), + (0x10A4, 0x1, 0x2D04), + (0x10A5, 0x1, 0x2D05), + (0x10A6, 0x1, 0x2D06), + (0x10A7, 0x1, 0x2D07), + (0x10A8, 0x1, 0x2D08), + (0x10A9, 0x1, 0x2D09), + (0x10AA, 0x1, 0x2D0A), + (0x10AB, 0x1, 0x2D0B), + (0x10AC, 0x1, 0x2D0C), + (0x10AD, 0x1, 0x2D0D), + (0x10AE, 0x1, 0x2D0E), + (0x10AF, 0x1, 0x2D0F), + (0x10B0, 0x1, 0x2D10), + (0x10B1, 0x1, 0x2D11), + (0x10B2, 0x1, 0x2D12), + (0x10B3, 0x1, 0x2D13), + (0x10B4, 0x1, 0x2D14), + (0x10B5, 0x1, 0x2D15), + (0x10B6, 0x1, 0x2D16), + (0x10B7, 0x1, 0x2D17), + (0x10B8, 0x1, 0x2D18), + (0x10B9, 0x1, 0x2D19), + (0x10BA, 0x1, 0x2D1A), + (0x10BB, 0x1, 0x2D1B), + (0x10BC, 0x1, 0x2D1C), + (0x10BD, 0x1, 0x2D1D), + (0x10BE, 0x1, 0x2D1E), + (0x10BF, 0x1, 0x2D1F), + (0x10C0, 0x1, 0x2D20), + (0x10C1, 0x1, 0x2D21), + (0x10C2, 0x1, 0x2D22), + (0x10C3, 0x1, 0x2D23), + (0x10C4, 0x1, 0x2D24), + (0x10C5, 0x1, 0x2D25), + (0x10C7, 0x1, 0x2D27), + (0x10CD, 0x1, 0x2D2D), + (0x13A0, 0x1, 0xAB70), + (0x13A1, 0x1, 0xAB71), + (0x13A2, 0x1, 0xAB72), + (0x13A3, 0x1, 0xAB73), + (0x13A4, 0x1, 0xAB74), + (0x13A5, 0x1, 0xAB75), + (0x13A6, 0x1, 0xAB76), + (0x13A7, 0x1, 0xAB77), + (0x13A8, 0x1, 0xAB78), + (0x13A9, 0x1, 0xAB79), + (0x13AA, 0x1, 0xAB7A), + (0x13AB, 0x1, 0xAB7B), + (0x13AC, 0x1, 0xAB7C), + (0x13AD, 0x1, 0xAB7D), + (0x13AE, 0x1, 0xAB7E), + (0x13AF, 0x1, 0xAB7F), + (0x13B0, 0x1, 0xAB80), + (0x13B1, 0x1, 0xAB81), + (0x13B2, 0x1, 0xAB82), + (0x13B3, 0x1, 0xAB83), + (0x13B4, 0x1, 0xAB84), + (0x13B5, 0x1, 0xAB85), + (0x13B6, 0x1, 0xAB86), + (0x13B7, 0x1, 0xAB87), + (0x13B8, 0x1, 0xAB88), + (0x13B9, 0x1, 0xAB89), + (0x13BA, 0x1, 0xAB8A), + (0x13BB, 0x1, 0xAB8B), + (0x13BC, 0x1, 0xAB8C), + (0x13BD, 0x1, 0xAB8D), + (0x13BE, 0x1, 0xAB8E), + (0x13BF, 0x1, 0xAB8F), + (0x13C0, 0x1, 0xAB90), + (0x13C1, 0x1, 0xAB91), + (0x13C2, 0x1, 0xAB92), + (0x13C3, 0x1, 0xAB93), + (0x13C4, 0x1, 0xAB94), + (0x13C5, 0x1, 0xAB95), + (0x13C6, 0x1, 0xAB96), + (0x13C7, 0x1, 0xAB97), + (0x13C8, 0x1, 0xAB98), + (0x13C9, 0x1, 0xAB99), + (0x13CA, 0x1, 0xAB9A), + (0x13CB, 0x1, 0xAB9B), + (0x13CC, 0x1, 0xAB9C), + (0x13CD, 0x1, 0xAB9D), + (0x13CE, 0x1, 0xAB9E), + (0x13CF, 0x1, 0xAB9F), + (0x13D0, 0x1, 0xABA0), + (0x13D1, 0x1, 0xABA1), + (0x13D2, 0x1, 0xABA2), + (0x13D3, 0x1, 0xABA3), + (0x13D4, 0x1, 0xABA4), + (0x13D5, 0x1, 0xABA5), + (0x13D6, 0x1, 0xABA6), + (0x13D7, 0x1, 0xABA7), + (0x13D8, 0x1, 0xABA8), + (0x13D9, 0x1, 0xABA9), + (0x13DA, 0x1, 0xABAA), + (0x13DB, 0x1, 0xABAB), + (0x13DC, 0x1, 0xABAC), + (0x13DD, 0x1, 0xABAD), + (0x13DE, 0x1, 0xABAE), + (0x13DF, 0x1, 0xABAF), + (0x13E0, 0x1, 0xABB0), + (0x13E1, 0x1, 0xABB1), + (0x13E2, 0x1, 0xABB2), + (0x13E3, 0x1, 0xABB3), + (0x13E4, 0x1, 0xABB4), + (0x13E5, 0x1, 0xABB5), + (0x13E6, 0x1, 0xABB6), + (0x13E7, 0x1, 0xABB7), + (0x13E8, 0x1, 0xABB8), + (0x13E9, 0x1, 0xABB9), + (0x13EA, 0x1, 0xABBA), + (0x13EB, 0x1, 0xABBB), + (0x13EC, 0x1, 0xABBC), + (0x13ED, 0x1, 0xABBD), + (0x13EE, 0x1, 0xABBE), + (0x13EF, 0x1, 0xABBF), + (0x13F0, 0x1, 0x13F8), + (0x13F1, 0x1, 0x13F9), + (0x13F2, 0x1, 0x13FA), + (0x13F3, 0x1, 0x13FB), + (0x13F4, 0x1, 0x13FC), + (0x13F5, 0x1, 0x13FD), + (0x1C90, 0x1, 0x10D0), + (0x1C91, 0x1, 0x10D1), + (0x1C92, 0x1, 0x10D2), + (0x1C93, 0x1, 0x10D3), + (0x1C94, 0x1, 0x10D4), + (0x1C95, 0x1, 0x10D5), + (0x1C96, 0x1, 0x10D6), + (0x1C97, 0x1, 0x10D7), + (0x1C98, 0x1, 0x10D8), + (0x1C99, 0x1, 0x10D9), + (0x1C9A, 0x1, 0x10DA), + (0x1C9B, 0x1, 0x10DB), + (0x1C9C, 0x1, 0x10DC), + (0x1C9D, 0x1, 0x10DD), + (0x1C9E, 0x1, 0x10DE), + (0x1C9F, 0x1, 0x10DF), + (0x1CA0, 0x1, 0x10E0), + (0x1CA1, 0x1, 0x10E1), + (0x1CA2, 0x1, 0x10E2), + (0x1CA3, 0x1, 0x10E3), + (0x1CA4, 0x1, 0x10E4), + (0x1CA5, 0x1, 0x10E5), + (0x1CA6, 0x1, 0x10E6), + (0x1CA7, 0x1, 0x10E7), + (0x1CA8, 0x1, 0x10E8), + (0x1CA9, 0x1, 0x10E9), + (0x1CAA, 0x1, 0x10EA), + (0x1CAB, 0x1, 0x10EB), + (0x1CAC, 0x1, 0x10EC), + (0x1CAD, 0x1, 0x10ED), + (0x1CAE, 0x1, 0x10EE), + (0x1CAF, 0x1, 0x10EF), + (0x1CB0, 0x1, 0x10F0), + (0x1CB1, 0x1, 0x10F1), + (0x1CB2, 0x1, 0x10F2), + (0x1CB3, 0x1, 0x10F3), + (0x1CB4, 0x1, 0x10F4), + (0x1CB5, 0x1, 0x10F5), + (0x1CB6, 0x1, 0x10F6), + (0x1CB7, 0x1, 0x10F7), + (0x1CB8, 0x1, 0x10F8), + (0x1CB9, 0x1, 0x10F9), + (0x1CBA, 0x1, 0x10FA), + (0x1CBD, 0x1, 0x10FD), + (0x1CBE, 0x1, 0x10FE), + (0x1CBF, 0x1, 0x10FF), + (0x1E00, 0x1, 0x1E01), + (0x1E02, 0x1, 0x1E03), + (0x1E04, 0x1, 0x1E05), + (0x1E06, 0x1, 0x1E07), + (0x1E08, 0x1, 0x1E09), + (0x1E0A, 0x1, 0x1E0B), + (0x1E0C, 0x1, 0x1E0D), + (0x1E0E, 0x1, 0x1E0F), + (0x1E10, 0x1, 0x1E11), + (0x1E12, 0x1, 0x1E13), + (0x1E14, 0x1, 0x1E15), + (0x1E16, 0x1, 0x1E17), + (0x1E18, 0x1, 0x1E19), + (0x1E1A, 0x1, 0x1E1B), + (0x1E1C, 0x1, 0x1E1D), + (0x1E1E, 0x1, 0x1E1F), + (0x1E20, 0x1, 0x1E21), + (0x1E22, 0x1, 0x1E23), + (0x1E24, 0x1, 0x1E25), + (0x1E26, 0x1, 0x1E27), + (0x1E28, 0x1, 0x1E29), + (0x1E2A, 0x1, 0x1E2B), + (0x1E2C, 0x1, 0x1E2D), + (0x1E2E, 0x1, 0x1E2F), + (0x1E30, 0x1, 0x1E31), + (0x1E32, 0x1, 0x1E33), + (0x1E34, 0x1, 0x1E35), + (0x1E36, 0x1, 0x1E37), + (0x1E38, 0x1, 0x1E39), + (0x1E3A, 0x1, 0x1E3B), + (0x1E3C, 0x1, 0x1E3D), + (0x1E3E, 0x1, 0x1E3F), + (0x1E40, 0x1, 0x1E41), + (0x1E42, 0x1, 0x1E43), + (0x1E44, 0x1, 0x1E45), + (0x1E46, 0x1, 0x1E47), + (0x1E48, 0x1, 0x1E49), + (0x1E4A, 0x1, 0x1E4B), + (0x1E4C, 0x1, 0x1E4D), + (0x1E4E, 0x1, 0x1E4F), + (0x1E50, 0x1, 0x1E51), + (0x1E52, 0x1, 0x1E53), + (0x1E54, 0x1, 0x1E55), + (0x1E56, 0x1, 0x1E57), + (0x1E58, 0x1, 0x1E59), + (0x1E5A, 0x1, 0x1E5B), + (0x1E5C, 0x1, 0x1E5D), + (0x1E5E, 0x1, 0x1E5F), + (0x1E60, 0x1, 0x1E61), + (0x1E62, 0x1, 0x1E63), + (0x1E64, 0x1, 0x1E65), + (0x1E66, 0x1, 0x1E67), + (0x1E68, 0x1, 0x1E69), + (0x1E6A, 0x1, 0x1E6B), + (0x1E6C, 0x1, 0x1E6D), + (0x1E6E, 0x1, 0x1E6F), + (0x1E70, 0x1, 0x1E71), + (0x1E72, 0x1, 0x1E73), + (0x1E74, 0x1, 0x1E75), + (0x1E76, 0x1, 0x1E77), + (0x1E78, 0x1, 0x1E79), + (0x1E7A, 0x1, 0x1E7B), + (0x1E7C, 0x1, 0x1E7D), + (0x1E7E, 0x1, 0x1E7F), + (0x1E80, 0x1, 0x1E81), + (0x1E82, 0x1, 0x1E83), + (0x1E84, 0x1, 0x1E85), + (0x1E86, 0x1, 0x1E87), + (0x1E88, 0x1, 0x1E89), + (0x1E8A, 0x1, 0x1E8B), + (0x1E8C, 0x1, 0x1E8D), + (0x1E8E, 0x1, 0x1E8F), + (0x1E90, 0x1, 0x1E91), + (0x1E92, 0x1, 0x1E93), + (0x1E94, 0x1, 0x1E95), + (0x1E9E, 0x1, 0xDF), + (0x1EA0, 0x1, 0x1EA1), + (0x1EA2, 0x1, 0x1EA3), + (0x1EA4, 0x1, 0x1EA5), + (0x1EA6, 0x1, 0x1EA7), + (0x1EA8, 0x1, 0x1EA9), + (0x1EAA, 0x1, 0x1EAB), + (0x1EAC, 0x1, 0x1EAD), + (0x1EAE, 0x1, 0x1EAF), + (0x1EB0, 0x1, 0x1EB1), + (0x1EB2, 0x1, 0x1EB3), + (0x1EB4, 0x1, 0x1EB5), + (0x1EB6, 0x1, 0x1EB7), + (0x1EB8, 0x1, 0x1EB9), + (0x1EBA, 0x1, 0x1EBB), + (0x1EBC, 0x1, 0x1EBD), + (0x1EBE, 0x1, 0x1EBF), + (0x1EC0, 0x1, 0x1EC1), + (0x1EC2, 0x1, 0x1EC3), + (0x1EC4, 0x1, 0x1EC5), + (0x1EC6, 0x1, 0x1EC7), + (0x1EC8, 0x1, 0x1EC9), + (0x1ECA, 0x1, 0x1ECB), + (0x1ECC, 0x1, 0x1ECD), + (0x1ECE, 0x1, 0x1ECF), + (0x1ED0, 0x1, 0x1ED1), + (0x1ED2, 0x1, 0x1ED3), + (0x1ED4, 0x1, 0x1ED5), + (0x1ED6, 0x1, 0x1ED7), + (0x1ED8, 0x1, 0x1ED9), + (0x1EDA, 0x1, 0x1EDB), + (0x1EDC, 0x1, 0x1EDD), + (0x1EDE, 0x1, 0x1EDF), + (0x1EE0, 0x1, 0x1EE1), + (0x1EE2, 0x1, 0x1EE3), + (0x1EE4, 0x1, 0x1EE5), + (0x1EE6, 0x1, 0x1EE7), + (0x1EE8, 0x1, 0x1EE9), + (0x1EEA, 0x1, 0x1EEB), + (0x1EEC, 0x1, 0x1EED), + (0x1EEE, 0x1, 0x1EEF), + (0x1EF0, 0x1, 0x1EF1), + (0x1EF2, 0x1, 0x1EF3), + (0x1EF4, 0x1, 0x1EF5), + (0x1EF6, 0x1, 0x1EF7), + (0x1EF8, 0x1, 0x1EF9), + (0x1EFA, 0x1, 0x1EFB), + (0x1EFC, 0x1, 0x1EFD), + (0x1EFE, 0x1, 0x1EFF), + (0x1F08, 0x1, 0x1F00), + (0x1F09, 0x1, 0x1F01), + (0x1F0A, 0x1, 0x1F02), + (0x1F0B, 0x1, 0x1F03), + (0x1F0C, 0x1, 0x1F04), + (0x1F0D, 0x1, 0x1F05), + (0x1F0E, 0x1, 0x1F06), + (0x1F0F, 0x1, 0x1F07), + (0x1F18, 0x1, 0x1F10), + (0x1F19, 0x1, 0x1F11), + (0x1F1A, 0x1, 0x1F12), + (0x1F1B, 0x1, 0x1F13), + (0x1F1C, 0x1, 0x1F14), + (0x1F1D, 0x1, 0x1F15), + (0x1F28, 0x1, 0x1F20), + (0x1F29, 0x1, 0x1F21), + (0x1F2A, 0x1, 0x1F22), + (0x1F2B, 0x1, 0x1F23), + (0x1F2C, 0x1, 0x1F24), + (0x1F2D, 0x1, 0x1F25), + (0x1F2E, 0x1, 0x1F26), + (0x1F2F, 0x1, 0x1F27), + (0x1F38, 0x1, 0x1F30), + (0x1F39, 0x1, 0x1F31), + (0x1F3A, 0x1, 0x1F32), + (0x1F3B, 0x1, 0x1F33), + (0x1F3C, 0x1, 0x1F34), + (0x1F3D, 0x1, 0x1F35), + (0x1F3E, 0x1, 0x1F36), + (0x1F3F, 0x1, 0x1F37), + (0x1F48, 0x1, 0x1F40), + (0x1F49, 0x1, 0x1F41), + (0x1F4A, 0x1, 0x1F42), + (0x1F4B, 0x1, 0x1F43), + (0x1F4C, 0x1, 0x1F44), + (0x1F4D, 0x1, 0x1F45), + (0x1F59, 0x1, 0x1F51), + (0x1F5B, 0x1, 0x1F53), + (0x1F5D, 0x1, 0x1F55), + (0x1F5F, 0x1, 0x1F57), + (0x1F68, 0x1, 0x1F60), + (0x1F69, 0x1, 0x1F61), + (0x1F6A, 0x1, 0x1F62), + (0x1F6B, 0x1, 0x1F63), + (0x1F6C, 0x1, 0x1F64), + (0x1F6D, 0x1, 0x1F65), + (0x1F6E, 0x1, 0x1F66), + (0x1F6F, 0x1, 0x1F67), + (0x1F88, 0x1, 0x1F80), + (0x1F89, 0x1, 0x1F81), + (0x1F8A, 0x1, 0x1F82), + (0x1F8B, 0x1, 0x1F83), + (0x1F8C, 0x1, 0x1F84), + (0x1F8D, 0x1, 0x1F85), + (0x1F8E, 0x1, 0x1F86), + (0x1F8F, 0x1, 0x1F87), + (0x1F98, 0x1, 0x1F90), + (0x1F99, 0x1, 0x1F91), + (0x1F9A, 0x1, 0x1F92), + (0x1F9B, 0x1, 0x1F93), + (0x1F9C, 0x1, 0x1F94), + (0x1F9D, 0x1, 0x1F95), + (0x1F9E, 0x1, 0x1F96), + (0x1F9F, 0x1, 0x1F97), + (0x1FA8, 0x1, 0x1FA0), + (0x1FA9, 0x1, 0x1FA1), + (0x1FAA, 0x1, 0x1FA2), + (0x1FAB, 0x1, 0x1FA3), + (0x1FAC, 0x1, 0x1FA4), + (0x1FAD, 0x1, 0x1FA5), + (0x1FAE, 0x1, 0x1FA6), + (0x1FAF, 0x1, 0x1FA7), + (0x1FB8, 0x1, 0x1FB0), + (0x1FB9, 0x1, 0x1FB1), + (0x1FBA, 0x1, 0x1F70), + (0x1FBB, 0x1, 0x1F71), + (0x1FBC, 0x1, 0x1FB3), + (0x1FC8, 0x1, 0x1F72), + (0x1FC9, 0x1, 0x1F73), + (0x1FCA, 0x1, 0x1F74), + (0x1FCB, 0x1, 0x1F75), + (0x1FCC, 0x1, 0x1FC3), + (0x1FD8, 0x1, 0x1FD0), + (0x1FD9, 0x1, 0x1FD1), + (0x1FDA, 0x1, 0x1F76), + (0x1FDB, 0x1, 0x1F77), + (0x1FE8, 0x1, 0x1FE0), + (0x1FE9, 0x1, 0x1FE1), + (0x1FEA, 0x1, 0x1F7A), + (0x1FEB, 0x1, 0x1F7B), + (0x1FEC, 0x1, 0x1FE5), + (0x1FF8, 0x1, 0x1F78), + (0x1FF9, 0x1, 0x1F79), + (0x1FFA, 0x1, 0x1F7C), + (0x1FFB, 0x1, 0x1F7D), + (0x1FFC, 0x1, 0x1FF3), + (0x2126, 0x1, 0x3C9), + (0x212A, 0x1, 0x6B), + (0x212B, 0x1, 0xE5), + (0x2132, 0x1, 0x214E), + (0x2160, 0x1, 0x2170), + (0x2161, 0x1, 0x2171), + (0x2162, 0x1, 0x2172), + (0x2163, 0x1, 0x2173), + (0x2164, 0x1, 0x2174), + (0x2165, 0x1, 0x2175), + (0x2166, 0x1, 0x2176), + (0x2167, 0x1, 0x2177), + (0x2168, 0x1, 0x2178), + (0x2169, 0x1, 0x2179), + (0x216A, 0x1, 0x217A), + (0x216B, 0x1, 0x217B), + (0x216C, 0x1, 0x217C), + (0x216D, 0x1, 0x217D), + (0x216E, 0x1, 0x217E), + (0x216F, 0x1, 0x217F), + (0x2183, 0x1, 0x2184), + (0x24B6, 0x1, 0x24D0), + (0x24B7, 0x1, 0x24D1), + (0x24B8, 0x1, 0x24D2), + (0x24B9, 0x1, 0x24D3), + (0x24BA, 0x1, 0x24D4), + (0x24BB, 0x1, 0x24D5), + (0x24BC, 0x1, 0x24D6), + (0x24BD, 0x1, 0x24D7), + (0x24BE, 0x1, 0x24D8), + (0x24BF, 0x1, 0x24D9), + (0x24C0, 0x1, 0x24DA), + (0x24C1, 0x1, 0x24DB), + (0x24C2, 0x1, 0x24DC), + (0x24C3, 0x1, 0x24DD), + (0x24C4, 0x1, 0x24DE), + (0x24C5, 0x1, 0x24DF), + (0x24C6, 0x1, 0x24E0), + (0x24C7, 0x1, 0x24E1), + (0x24C8, 0x1, 0x24E2), + (0x24C9, 0x1, 0x24E3), + (0x24CA, 0x1, 0x24E4), + (0x24CB, 0x1, 0x24E5), + (0x24CC, 0x1, 0x24E6), + (0x24CD, 0x1, 0x24E7), + (0x24CE, 0x1, 0x24E8), + (0x24CF, 0x1, 0x24E9), + (0x2C00, 0x1, 0x2C30), + (0x2C01, 0x1, 0x2C31), + (0x2C02, 0x1, 0x2C32), + (0x2C03, 0x1, 0x2C33), + (0x2C04, 0x1, 0x2C34), + (0x2C05, 0x1, 0x2C35), + (0x2C06, 0x1, 0x2C36), + (0x2C07, 0x1, 0x2C37), + (0x2C08, 0x1, 0x2C38), + (0x2C09, 0x1, 0x2C39), + (0x2C0A, 0x1, 0x2C3A), + (0x2C0B, 0x1, 0x2C3B), + (0x2C0C, 0x1, 0x2C3C), + (0x2C0D, 0x1, 0x2C3D), + (0x2C0E, 0x1, 0x2C3E), + (0x2C0F, 0x1, 0x2C3F), + (0x2C10, 0x1, 0x2C40), + (0x2C11, 0x1, 0x2C41), + (0x2C12, 0x1, 0x2C42), + (0x2C13, 0x1, 0x2C43), + (0x2C14, 0x1, 0x2C44), + (0x2C15, 0x1, 0x2C45), + (0x2C16, 0x1, 0x2C46), + (0x2C17, 0x1, 0x2C47), + (0x2C18, 0x1, 0x2C48), + (0x2C19, 0x1, 0x2C49), + (0x2C1A, 0x1, 0x2C4A), + (0x2C1B, 0x1, 0x2C4B), + (0x2C1C, 0x1, 0x2C4C), + (0x2C1D, 0x1, 0x2C4D), + (0x2C1E, 0x1, 0x2C4E), + (0x2C1F, 0x1, 0x2C4F), + (0x2C20, 0x1, 0x2C50), + (0x2C21, 0x1, 0x2C51), + (0x2C22, 0x1, 0x2C52), + (0x2C23, 0x1, 0x2C53), + (0x2C24, 0x1, 0x2C54), + (0x2C25, 0x1, 0x2C55), + (0x2C26, 0x1, 0x2C56), + (0x2C27, 0x1, 0x2C57), + (0x2C28, 0x1, 0x2C58), + (0x2C29, 0x1, 0x2C59), + (0x2C2A, 0x1, 0x2C5A), + (0x2C2B, 0x1, 0x2C5B), + (0x2C2C, 0x1, 0x2C5C), + (0x2C2D, 0x1, 0x2C5D), + (0x2C2E, 0x1, 0x2C5E), + (0x2C2F, 0x1, 0x2C5F), + (0x2C60, 0x1, 0x2C61), + (0x2C62, 0x1, 0x26B), + (0x2C63, 0x1, 0x1D7D), + (0x2C64, 0x1, 0x27D), + (0x2C67, 0x1, 0x2C68), + (0x2C69, 0x1, 0x2C6A), + (0x2C6B, 0x1, 0x2C6C), + (0x2C6D, 0x1, 0x251), + (0x2C6E, 0x1, 0x271), + (0x2C6F, 0x1, 0x250), + (0x2C70, 0x1, 0x252), + (0x2C72, 0x1, 0x2C73), + (0x2C75, 0x1, 0x2C76), + (0x2C7E, 0x1, 0x23F), + (0x2C7F, 0x1, 0x240), + (0x2C80, 0x1, 0x2C81), + (0x2C82, 0x1, 0x2C83), + (0x2C84, 0x1, 0x2C85), + (0x2C86, 0x1, 0x2C87), + (0x2C88, 0x1, 0x2C89), + (0x2C8A, 0x1, 0x2C8B), + (0x2C8C, 0x1, 0x2C8D), + (0x2C8E, 0x1, 0x2C8F), + (0x2C90, 0x1, 0x2C91), + (0x2C92, 0x1, 0x2C93), + (0x2C94, 0x1, 0x2C95), + (0x2C96, 0x1, 0x2C97), + (0x2C98, 0x1, 0x2C99), + (0x2C9A, 0x1, 0x2C9B), + (0x2C9C, 0x1, 0x2C9D), + (0x2C9E, 0x1, 0x2C9F), + (0x2CA0, 0x1, 0x2CA1), + (0x2CA2, 0x1, 0x2CA3), + (0x2CA4, 0x1, 0x2CA5), + (0x2CA6, 0x1, 0x2CA7), + (0x2CA8, 0x1, 0x2CA9), + (0x2CAA, 0x1, 0x2CAB), + (0x2CAC, 0x1, 0x2CAD), + (0x2CAE, 0x1, 0x2CAF), + (0x2CB0, 0x1, 0x2CB1), + (0x2CB2, 0x1, 0x2CB3), + (0x2CB4, 0x1, 0x2CB5), + (0x2CB6, 0x1, 0x2CB7), + (0x2CB8, 0x1, 0x2CB9), + (0x2CBA, 0x1, 0x2CBB), + (0x2CBC, 0x1, 0x2CBD), + (0x2CBE, 0x1, 0x2CBF), + (0x2CC0, 0x1, 0x2CC1), + (0x2CC2, 0x1, 0x2CC3), + (0x2CC4, 0x1, 0x2CC5), + (0x2CC6, 0x1, 0x2CC7), + (0x2CC8, 0x1, 0x2CC9), + (0x2CCA, 0x1, 0x2CCB), + (0x2CCC, 0x1, 0x2CCD), + (0x2CCE, 0x1, 0x2CCF), + (0x2CD0, 0x1, 0x2CD1), + (0x2CD2, 0x1, 0x2CD3), + (0x2CD4, 0x1, 0x2CD5), + (0x2CD6, 0x1, 0x2CD7), + (0x2CD8, 0x1, 0x2CD9), + (0x2CDA, 0x1, 0x2CDB), + (0x2CDC, 0x1, 0x2CDD), + (0x2CDE, 0x1, 0x2CDF), + (0x2CE0, 0x1, 0x2CE1), + (0x2CE2, 0x1, 0x2CE3), + (0x2CEB, 0x1, 0x2CEC), + (0x2CED, 0x1, 0x2CEE), + (0x2CF2, 0x1, 0x2CF3), + (0xA640, 0x1, 0xA641), + (0xA642, 0x1, 0xA643), + (0xA644, 0x1, 0xA645), + (0xA646, 0x1, 0xA647), + (0xA648, 0x1, 0xA649), + (0xA64A, 0x1, 0xA64B), + (0xA64C, 0x1, 0xA64D), + (0xA64E, 0x1, 0xA64F), + (0xA650, 0x1, 0xA651), + (0xA652, 0x1, 0xA653), + (0xA654, 0x1, 0xA655), + (0xA656, 0x1, 0xA657), + (0xA658, 0x1, 0xA659), + (0xA65A, 0x1, 0xA65B), + (0xA65C, 0x1, 0xA65D), + (0xA65E, 0x1, 0xA65F), + (0xA660, 0x1, 0xA661), + (0xA662, 0x1, 0xA663), + (0xA664, 0x1, 0xA665), + (0xA666, 0x1, 0xA667), + (0xA668, 0x1, 0xA669), + (0xA66A, 0x1, 0xA66B), + (0xA66C, 0x1, 0xA66D), + (0xA680, 0x1, 0xA681), + (0xA682, 0x1, 0xA683), + (0xA684, 0x1, 0xA685), + (0xA686, 0x1, 0xA687), + (0xA688, 0x1, 0xA689), + (0xA68A, 0x1, 0xA68B), + (0xA68C, 0x1, 0xA68D), + (0xA68E, 0x1, 0xA68F), + (0xA690, 0x1, 0xA691), + (0xA692, 0x1, 0xA693), + (0xA694, 0x1, 0xA695), + (0xA696, 0x1, 0xA697), + (0xA698, 0x1, 0xA699), + (0xA69A, 0x1, 0xA69B), + (0xA722, 0x1, 0xA723), + (0xA724, 0x1, 0xA725), + (0xA726, 0x1, 0xA727), + (0xA728, 0x1, 0xA729), + (0xA72A, 0x1, 0xA72B), + (0xA72C, 0x1, 0xA72D), + (0xA72E, 0x1, 0xA72F), + (0xA732, 0x1, 0xA733), + (0xA734, 0x1, 0xA735), + (0xA736, 0x1, 0xA737), + (0xA738, 0x1, 0xA739), + (0xA73A, 0x1, 0xA73B), + (0xA73C, 0x1, 0xA73D), + (0xA73E, 0x1, 0xA73F), + (0xA740, 0x1, 0xA741), + (0xA742, 0x1, 0xA743), + (0xA744, 0x1, 0xA745), + (0xA746, 0x1, 0xA747), + (0xA748, 0x1, 0xA749), + (0xA74A, 0x1, 0xA74B), + (0xA74C, 0x1, 0xA74D), + (0xA74E, 0x1, 0xA74F), + (0xA750, 0x1, 0xA751), + (0xA752, 0x1, 0xA753), + (0xA754, 0x1, 0xA755), + (0xA756, 0x1, 0xA757), + (0xA758, 0x1, 0xA759), + (0xA75A, 0x1, 0xA75B), + (0xA75C, 0x1, 0xA75D), + (0xA75E, 0x1, 0xA75F), + (0xA760, 0x1, 0xA761), + (0xA762, 0x1, 0xA763), + (0xA764, 0x1, 0xA765), + (0xA766, 0x1, 0xA767), + (0xA768, 0x1, 0xA769), + (0xA76A, 0x1, 0xA76B), + (0xA76C, 0x1, 0xA76D), + (0xA76E, 0x1, 0xA76F), + (0xA779, 0x1, 0xA77A), + (0xA77B, 0x1, 0xA77C), + (0xA77D, 0x1, 0x1D79), + (0xA77E, 0x1, 0xA77F), + (0xA780, 0x1, 0xA781), + (0xA782, 0x1, 0xA783), + (0xA784, 0x1, 0xA785), + (0xA786, 0x1, 0xA787), + (0xA78B, 0x1, 0xA78C), + (0xA78D, 0x1, 0x265), + (0xA790, 0x1, 0xA791), + (0xA792, 0x1, 0xA793), + (0xA796, 0x1, 0xA797), + (0xA798, 0x1, 0xA799), + (0xA79A, 0x1, 0xA79B), + (0xA79C, 0x1, 0xA79D), + (0xA79E, 0x1, 0xA79F), + (0xA7A0, 0x1, 0xA7A1), + (0xA7A2, 0x1, 0xA7A3), + (0xA7A4, 0x1, 0xA7A5), + (0xA7A6, 0x1, 0xA7A7), + (0xA7A8, 0x1, 0xA7A9), + (0xA7AA, 0x1, 0x266), + (0xA7AB, 0x1, 0x25C), + (0xA7AC, 0x1, 0x261), + (0xA7AD, 0x1, 0x26C), + (0xA7AE, 0x1, 0x26A), + (0xA7B0, 0x1, 0x29E), + (0xA7B1, 0x1, 0x287), + (0xA7B2, 0x1, 0x29D), + (0xA7B3, 0x1, 0xAB53), + (0xA7B4, 0x1, 0xA7B5), + (0xA7B6, 0x1, 0xA7B7), + (0xA7B8, 0x1, 0xA7B9), + (0xA7BA, 0x1, 0xA7BB), + (0xA7BC, 0x1, 0xA7BD), + (0xA7BE, 0x1, 0xA7BF), + (0xA7C0, 0x1, 0xA7C1), + (0xA7C2, 0x1, 0xA7C3), + (0xA7C4, 0x1, 0xA794), + (0xA7C5, 0x1, 0x282), + (0xA7C6, 0x1, 0x1D8E), + (0xA7C7, 0x1, 0xA7C8), + (0xA7C9, 0x1, 0xA7CA), + (0xA7D0, 0x1, 0xA7D1), + (0xA7D6, 0x1, 0xA7D7), + (0xA7D8, 0x1, 0xA7D9), + (0xA7F5, 0x1, 0xA7F6), + (0xFF21, 0x1, 0xFF41), + (0xFF22, 0x1, 0xFF42), + (0xFF23, 0x1, 0xFF43), + (0xFF24, 0x1, 0xFF44), + (0xFF25, 0x1, 0xFF45), + (0xFF26, 0x1, 0xFF46), + (0xFF27, 0x1, 0xFF47), + (0xFF28, 0x1, 0xFF48), + (0xFF29, 0x1, 0xFF49), + (0xFF2A, 0x1, 0xFF4A), + (0xFF2B, 0x1, 0xFF4B), + (0xFF2C, 0x1, 0xFF4C), + (0xFF2D, 0x1, 0xFF4D), + (0xFF2E, 0x1, 0xFF4E), + (0xFF2F, 0x1, 0xFF4F), + (0xFF30, 0x1, 0xFF50), + (0xFF31, 0x1, 0xFF51), + (0xFF32, 0x1, 0xFF52), + (0xFF33, 0x1, 0xFF53), + (0xFF34, 0x1, 0xFF54), + (0xFF35, 0x1, 0xFF55), + (0xFF36, 0x1, 0xFF56), + (0xFF37, 0x1, 0xFF57), + (0xFF38, 0x1, 0xFF58), + (0xFF39, 0x1, 0xFF59), + (0xFF3A, 0x1, 0xFF5A), + (0x10400, 0x1, 0x10428), + (0x10401, 0x1, 0x10429), + (0x10402, 0x1, 0x1042A), + (0x10403, 0x1, 0x1042B), + (0x10404, 0x1, 0x1042C), + (0x10405, 0x1, 0x1042D), + (0x10406, 0x1, 0x1042E), + (0x10407, 0x1, 0x1042F), + (0x10408, 0x1, 0x10430), + (0x10409, 0x1, 0x10431), + (0x1040A, 0x1, 0x10432), + (0x1040B, 0x1, 0x10433), + (0x1040C, 0x1, 0x10434), + (0x1040D, 0x1, 0x10435), + (0x1040E, 0x1, 0x10436), + (0x1040F, 0x1, 0x10437), + (0x10410, 0x1, 0x10438), + (0x10411, 0x1, 0x10439), + (0x10412, 0x1, 0x1043A), + (0x10413, 0x1, 0x1043B), + (0x10414, 0x1, 0x1043C), + (0x10415, 0x1, 0x1043D), + (0x10416, 0x1, 0x1043E), + (0x10417, 0x1, 0x1043F), + (0x10418, 0x1, 0x10440), + (0x10419, 0x1, 0x10441), + (0x1041A, 0x1, 0x10442), + (0x1041B, 0x1, 0x10443), + (0x1041C, 0x1, 0x10444), + (0x1041D, 0x1, 0x10445), + (0x1041E, 0x1, 0x10446), + (0x1041F, 0x1, 0x10447), + (0x10420, 0x1, 0x10448), + (0x10421, 0x1, 0x10449), + (0x10422, 0x1, 0x1044A), + (0x10423, 0x1, 0x1044B), + (0x10424, 0x1, 0x1044C), + (0x10425, 0x1, 0x1044D), + (0x10426, 0x1, 0x1044E), + (0x10427, 0x1, 0x1044F), + (0x104B0, 0x1, 0x104D8), + (0x104B1, 0x1, 0x104D9), + (0x104B2, 0x1, 0x104DA), + (0x104B3, 0x1, 0x104DB), + (0x104B4, 0x1, 0x104DC), + (0x104B5, 0x1, 0x104DD), + (0x104B6, 0x1, 0x104DE), + (0x104B7, 0x1, 0x104DF), + (0x104B8, 0x1, 0x104E0), + (0x104B9, 0x1, 0x104E1), + (0x104BA, 0x1, 0x104E2), + (0x104BB, 0x1, 0x104E3), + (0x104BC, 0x1, 0x104E4), + (0x104BD, 0x1, 0x104E5), + (0x104BE, 0x1, 0x104E6), + (0x104BF, 0x1, 0x104E7), + (0x104C0, 0x1, 0x104E8), + (0x104C1, 0x1, 0x104E9), + (0x104C2, 0x1, 0x104EA), + (0x104C3, 0x1, 0x104EB), + (0x104C4, 0x1, 0x104EC), + (0x104C5, 0x1, 0x104ED), + (0x104C6, 0x1, 0x104EE), + (0x104C7, 0x1, 0x104EF), + (0x104C8, 0x1, 0x104F0), + (0x104C9, 0x1, 0x104F1), + (0x104CA, 0x1, 0x104F2), + (0x104CB, 0x1, 0x104F3), + (0x104CC, 0x1, 0x104F4), + (0x104CD, 0x1, 0x104F5), + (0x104CE, 0x1, 0x104F6), + (0x104CF, 0x1, 0x104F7), + (0x104D0, 0x1, 0x104F8), + (0x104D1, 0x1, 0x104F9), + (0x104D2, 0x1, 0x104FA), + (0x104D3, 0x1, 0x104FB), + (0x10570, 0x1, 0x10597), + (0x10571, 0x1, 0x10598), + (0x10572, 0x1, 0x10599), + (0x10573, 0x1, 0x1059A), + (0x10574, 0x1, 0x1059B), + (0x10575, 0x1, 0x1059C), + (0x10576, 0x1, 0x1059D), + (0x10577, 0x1, 0x1059E), + (0x10578, 0x1, 0x1059F), + (0x10579, 0x1, 0x105A0), + (0x1057A, 0x1, 0x105A1), + (0x1057C, 0x1, 0x105A3), + (0x1057D, 0x1, 0x105A4), + (0x1057E, 0x1, 0x105A5), + (0x1057F, 0x1, 0x105A6), + (0x10580, 0x1, 0x105A7), + (0x10581, 0x1, 0x105A8), + (0x10582, 0x1, 0x105A9), + (0x10583, 0x1, 0x105AA), + (0x10584, 0x1, 0x105AB), + (0x10585, 0x1, 0x105AC), + (0x10586, 0x1, 0x105AD), + (0x10587, 0x1, 0x105AE), + (0x10588, 0x1, 0x105AF), + (0x10589, 0x1, 0x105B0), + (0x1058A, 0x1, 0x105B1), + (0x1058C, 0x1, 0x105B3), + (0x1058D, 0x1, 0x105B4), + (0x1058E, 0x1, 0x105B5), + (0x1058F, 0x1, 0x105B6), + (0x10590, 0x1, 0x105B7), + (0x10591, 0x1, 0x105B8), + (0x10592, 0x1, 0x105B9), + (0x10594, 0x1, 0x105BB), + (0x10595, 0x1, 0x105BC), + (0x10C80, 0x1, 0x10CC0), + (0x10C81, 0x1, 0x10CC1), + (0x10C82, 0x1, 0x10CC2), + (0x10C83, 0x1, 0x10CC3), + (0x10C84, 0x1, 0x10CC4), + (0x10C85, 0x1, 0x10CC5), + (0x10C86, 0x1, 0x10CC6), + (0x10C87, 0x1, 0x10CC7), + (0x10C88, 0x1, 0x10CC8), + (0x10C89, 0x1, 0x10CC9), + (0x10C8A, 0x1, 0x10CCA), + (0x10C8B, 0x1, 0x10CCB), + (0x10C8C, 0x1, 0x10CCC), + (0x10C8D, 0x1, 0x10CCD), + (0x10C8E, 0x1, 0x10CCE), + (0x10C8F, 0x1, 0x10CCF), + (0x10C90, 0x1, 0x10CD0), + (0x10C91, 0x1, 0x10CD1), + (0x10C92, 0x1, 0x10CD2), + (0x10C93, 0x1, 0x10CD3), + (0x10C94, 0x1, 0x10CD4), + (0x10C95, 0x1, 0x10CD5), + (0x10C96, 0x1, 0x10CD6), + (0x10C97, 0x1, 0x10CD7), + (0x10C98, 0x1, 0x10CD8), + (0x10C99, 0x1, 0x10CD9), + (0x10C9A, 0x1, 0x10CDA), + (0x10C9B, 0x1, 0x10CDB), + (0x10C9C, 0x1, 0x10CDC), + (0x10C9D, 0x1, 0x10CDD), + (0x10C9E, 0x1, 0x10CDE), + (0x10C9F, 0x1, 0x10CDF), + (0x10CA0, 0x1, 0x10CE0), + (0x10CA1, 0x1, 0x10CE1), + (0x10CA2, 0x1, 0x10CE2), + (0x10CA3, 0x1, 0x10CE3), + (0x10CA4, 0x1, 0x10CE4), + (0x10CA5, 0x1, 0x10CE5), + (0x10CA6, 0x1, 0x10CE6), + (0x10CA7, 0x1, 0x10CE7), + (0x10CA8, 0x1, 0x10CE8), + (0x10CA9, 0x1, 0x10CE9), + (0x10CAA, 0x1, 0x10CEA), + (0x10CAB, 0x1, 0x10CEB), + (0x10CAC, 0x1, 0x10CEC), + (0x10CAD, 0x1, 0x10CED), + (0x10CAE, 0x1, 0x10CEE), + (0x10CAF, 0x1, 0x10CEF), + (0x10CB0, 0x1, 0x10CF0), + (0x10CB1, 0x1, 0x10CF1), + (0x10CB2, 0x1, 0x10CF2), + (0x118A0, 0x1, 0x118C0), + (0x118A1, 0x1, 0x118C1), + (0x118A2, 0x1, 0x118C2), + (0x118A3, 0x1, 0x118C3), + (0x118A4, 0x1, 0x118C4), + (0x118A5, 0x1, 0x118C5), + (0x118A6, 0x1, 0x118C6), + (0x118A7, 0x1, 0x118C7), + (0x118A8, 0x1, 0x118C8), + (0x118A9, 0x1, 0x118C9), + (0x118AA, 0x1, 0x118CA), + (0x118AB, 0x1, 0x118CB), + (0x118AC, 0x1, 0x118CC), + (0x118AD, 0x1, 0x118CD), + (0x118AE, 0x1, 0x118CE), + (0x118AF, 0x1, 0x118CF), + (0x118B0, 0x1, 0x118D0), + (0x118B1, 0x1, 0x118D1), + (0x118B2, 0x1, 0x118D2), + (0x118B3, 0x1, 0x118D3), + (0x118B4, 0x1, 0x118D4), + (0x118B5, 0x1, 0x118D5), + (0x118B6, 0x1, 0x118D6), + (0x118B7, 0x1, 0x118D7), + (0x118B8, 0x1, 0x118D8), + (0x118B9, 0x1, 0x118D9), + (0x118BA, 0x1, 0x118DA), + (0x118BB, 0x1, 0x118DB), + (0x118BC, 0x1, 0x118DC), + (0x118BD, 0x1, 0x118DD), + (0x118BE, 0x1, 0x118DE), + (0x118BF, 0x1, 0x118DF), + (0x16E40, 0x1, 0x16E60), + (0x16E41, 0x1, 0x16E61), + (0x16E42, 0x1, 0x16E62), + (0x16E43, 0x1, 0x16E63), + (0x16E44, 0x1, 0x16E64), + (0x16E45, 0x1, 0x16E65), + (0x16E46, 0x1, 0x16E66), + (0x16E47, 0x1, 0x16E67), + (0x16E48, 0x1, 0x16E68), + (0x16E49, 0x1, 0x16E69), + (0x16E4A, 0x1, 0x16E6A), + (0x16E4B, 0x1, 0x16E6B), + (0x16E4C, 0x1, 0x16E6C), + (0x16E4D, 0x1, 0x16E6D), + (0x16E4E, 0x1, 0x16E6E), + (0x16E4F, 0x1, 0x16E6F), + (0x16E50, 0x1, 0x16E70), + (0x16E51, 0x1, 0x16E71), + (0x16E52, 0x1, 0x16E72), + (0x16E53, 0x1, 0x16E73), + (0x16E54, 0x1, 0x16E74), + (0x16E55, 0x1, 0x16E75), + (0x16E56, 0x1, 0x16E76), + (0x16E57, 0x1, 0x16E77), + (0x16E58, 0x1, 0x16E78), + (0x16E59, 0x1, 0x16E79), + (0x16E5A, 0x1, 0x16E7A), + (0x16E5B, 0x1, 0x16E7B), + (0x16E5C, 0x1, 0x16E7C), + (0x16E5D, 0x1, 0x16E7D), + (0x16E5E, 0x1, 0x16E7E), + (0x16E5F, 0x1, 0x16E7F), + (0x1E900, 0x1, 0x1E922), + (0x1E901, 0x1, 0x1E923), + (0x1E902, 0x1, 0x1E924), + (0x1E903, 0x1, 0x1E925), + (0x1E904, 0x1, 0x1E926), + (0x1E905, 0x1, 0x1E927), + (0x1E906, 0x1, 0x1E928), + (0x1E907, 0x1, 0x1E929), + (0x1E908, 0x1, 0x1E92A), + (0x1E909, 0x1, 0x1E92B), + (0x1E90A, 0x1, 0x1E92C), + (0x1E90B, 0x1, 0x1E92D), + (0x1E90C, 0x1, 0x1E92E), + (0x1E90D, 0x1, 0x1E92F), + (0x1E90E, 0x1, 0x1E930), + (0x1E90F, 0x1, 0x1E931), + (0x1E910, 0x1, 0x1E932), + (0x1E911, 0x1, 0x1E933), + (0x1E912, 0x1, 0x1E934), + (0x1E913, 0x1, 0x1E935), + (0x1E914, 0x1, 0x1E936), + (0x1E915, 0x1, 0x1E937), + (0x1E916, 0x1, 0x1E938), + (0x1E917, 0x1, 0x1E939), + (0x1E918, 0x1, 0x1E93A), + (0x1E919, 0x1, 0x1E93B), + (0x1E91A, 0x1, 0x1E93C), + (0x1E91B, 0x1, 0x1E93D), + (0x1E91C, 0x1, 0x1E93E), + (0x1E91D, 0x1, 0x1E93F), + (0x1E91E, 0x1, 0x1E940), + (0x1E91F, 0x1, 0x1E941), + (0x1E920, 0x1, 0x1E942), + (0x1E921, 0x1, 0x1E943), +) + +UPPER_ROWS = ( + (0x61, 0x1, 0x41), + (0x62, 0x1, 0x42), + (0x63, 0x1, 0x43), + (0x64, 0x1, 0x44), + (0x65, 0x1, 0x45), + (0x66, 0x1, 0x46), + (0x67, 0x1, 0x47), + (0x68, 0x1, 0x48), + (0x69, 0x1, 0x49), + (0x6A, 0x1, 0x4A), + (0x6B, 0x1, 0x4B), + (0x6C, 0x1, 0x4C), + (0x6D, 0x1, 0x4D), + (0x6E, 0x1, 0x4E), + (0x6F, 0x1, 0x4F), + (0x70, 0x1, 0x50), + (0x71, 0x1, 0x51), + (0x72, 0x1, 0x52), + (0x73, 0x1, 0x53), + (0x74, 0x1, 0x54), + (0x75, 0x1, 0x55), + (0x76, 0x1, 0x56), + (0x77, 0x1, 0x57), + (0x78, 0x1, 0x58), + (0x79, 0x1, 0x59), + (0x7A, 0x1, 0x5A), + (0xB5, 0x1, 0x39C), + (0xDF, 0x2, 0x53, 0x53), + (0xE0, 0x1, 0xC0), + (0xE1, 0x1, 0xC1), + (0xE2, 0x1, 0xC2), + (0xE3, 0x1, 0xC3), + (0xE4, 0x1, 0xC4), + (0xE5, 0x1, 0xC5), + (0xE6, 0x1, 0xC6), + (0xE7, 0x1, 0xC7), + (0xE8, 0x1, 0xC8), + (0xE9, 0x1, 0xC9), + (0xEA, 0x1, 0xCA), + (0xEB, 0x1, 0xCB), + (0xEC, 0x1, 0xCC), + (0xED, 0x1, 0xCD), + (0xEE, 0x1, 0xCE), + (0xEF, 0x1, 0xCF), + (0xF0, 0x1, 0xD0), + (0xF1, 0x1, 0xD1), + (0xF2, 0x1, 0xD2), + (0xF3, 0x1, 0xD3), + (0xF4, 0x1, 0xD4), + (0xF5, 0x1, 0xD5), + (0xF6, 0x1, 0xD6), + (0xF8, 0x1, 0xD8), + (0xF9, 0x1, 0xD9), + (0xFA, 0x1, 0xDA), + (0xFB, 0x1, 0xDB), + (0xFC, 0x1, 0xDC), + (0xFD, 0x1, 0xDD), + (0xFE, 0x1, 0xDE), + (0xFF, 0x1, 0x178), + (0x101, 0x1, 0x100), + (0x103, 0x1, 0x102), + (0x105, 0x1, 0x104), + (0x107, 0x1, 0x106), + (0x109, 0x1, 0x108), + (0x10B, 0x1, 0x10A), + (0x10D, 0x1, 0x10C), + (0x10F, 0x1, 0x10E), + (0x111, 0x1, 0x110), + (0x113, 0x1, 0x112), + (0x115, 0x1, 0x114), + (0x117, 0x1, 0x116), + (0x119, 0x1, 0x118), + (0x11B, 0x1, 0x11A), + (0x11D, 0x1, 0x11C), + (0x11F, 0x1, 0x11E), + (0x121, 0x1, 0x120), + (0x123, 0x1, 0x122), + (0x125, 0x1, 0x124), + (0x127, 0x1, 0x126), + (0x129, 0x1, 0x128), + (0x12B, 0x1, 0x12A), + (0x12D, 0x1, 0x12C), + (0x12F, 0x1, 0x12E), + (0x131, 0x1, 0x49), + (0x133, 0x1, 0x132), + (0x135, 0x1, 0x134), + (0x137, 0x1, 0x136), + (0x13A, 0x1, 0x139), + (0x13C, 0x1, 0x13B), + (0x13E, 0x1, 0x13D), + (0x140, 0x1, 0x13F), + (0x142, 0x1, 0x141), + (0x144, 0x1, 0x143), + (0x146, 0x1, 0x145), + (0x148, 0x1, 0x147), + (0x149, 0x2, 0x2BC, 0x4E), + (0x14B, 0x1, 0x14A), + (0x14D, 0x1, 0x14C), + (0x14F, 0x1, 0x14E), + (0x151, 0x1, 0x150), + (0x153, 0x1, 0x152), + (0x155, 0x1, 0x154), + (0x157, 0x1, 0x156), + (0x159, 0x1, 0x158), + (0x15B, 0x1, 0x15A), + (0x15D, 0x1, 0x15C), + (0x15F, 0x1, 0x15E), + (0x161, 0x1, 0x160), + (0x163, 0x1, 0x162), + (0x165, 0x1, 0x164), + (0x167, 0x1, 0x166), + (0x169, 0x1, 0x168), + (0x16B, 0x1, 0x16A), + (0x16D, 0x1, 0x16C), + (0x16F, 0x1, 0x16E), + (0x171, 0x1, 0x170), + (0x173, 0x1, 0x172), + (0x175, 0x1, 0x174), + (0x177, 0x1, 0x176), + (0x17A, 0x1, 0x179), + (0x17C, 0x1, 0x17B), + (0x17E, 0x1, 0x17D), + (0x17F, 0x1, 0x53), + (0x180, 0x1, 0x243), + (0x183, 0x1, 0x182), + (0x185, 0x1, 0x184), + (0x188, 0x1, 0x187), + (0x18C, 0x1, 0x18B), + (0x192, 0x1, 0x191), + (0x195, 0x1, 0x1F6), + (0x199, 0x1, 0x198), + (0x19A, 0x1, 0x23D), + (0x19E, 0x1, 0x220), + (0x1A1, 0x1, 0x1A0), + (0x1A3, 0x1, 0x1A2), + (0x1A5, 0x1, 0x1A4), + (0x1A8, 0x1, 0x1A7), + (0x1AD, 0x1, 0x1AC), + (0x1B0, 0x1, 0x1AF), + (0x1B4, 0x1, 0x1B3), + (0x1B6, 0x1, 0x1B5), + (0x1B9, 0x1, 0x1B8), + (0x1BD, 0x1, 0x1BC), + (0x1BF, 0x1, 0x1F7), + (0x1C5, 0x1, 0x1C4), + (0x1C6, 0x1, 0x1C4), + (0x1C8, 0x1, 0x1C7), + (0x1C9, 0x1, 0x1C7), + (0x1CB, 0x1, 0x1CA), + (0x1CC, 0x1, 0x1CA), + (0x1CE, 0x1, 0x1CD), + (0x1D0, 0x1, 0x1CF), + (0x1D2, 0x1, 0x1D1), + (0x1D4, 0x1, 0x1D3), + (0x1D6, 0x1, 0x1D5), + (0x1D8, 0x1, 0x1D7), + (0x1DA, 0x1, 0x1D9), + (0x1DC, 0x1, 0x1DB), + (0x1DD, 0x1, 0x18E), + (0x1DF, 0x1, 0x1DE), + (0x1E1, 0x1, 0x1E0), + (0x1E3, 0x1, 0x1E2), + (0x1E5, 0x1, 0x1E4), + (0x1E7, 0x1, 0x1E6), + (0x1E9, 0x1, 0x1E8), + (0x1EB, 0x1, 0x1EA), + (0x1ED, 0x1, 0x1EC), + (0x1EF, 0x1, 0x1EE), + (0x1F0, 0x2, 0x4A, 0x30C), + (0x1F2, 0x1, 0x1F1), + (0x1F3, 0x1, 0x1F1), + (0x1F5, 0x1, 0x1F4), + (0x1F9, 0x1, 0x1F8), + (0x1FB, 0x1, 0x1FA), + (0x1FD, 0x1, 0x1FC), + (0x1FF, 0x1, 0x1FE), + (0x201, 0x1, 0x200), + (0x203, 0x1, 0x202), + (0x205, 0x1, 0x204), + (0x207, 0x1, 0x206), + (0x209, 0x1, 0x208), + (0x20B, 0x1, 0x20A), + (0x20D, 0x1, 0x20C), + (0x20F, 0x1, 0x20E), + (0x211, 0x1, 0x210), + (0x213, 0x1, 0x212), + (0x215, 0x1, 0x214), + (0x217, 0x1, 0x216), + (0x219, 0x1, 0x218), + (0x21B, 0x1, 0x21A), + (0x21D, 0x1, 0x21C), + (0x21F, 0x1, 0x21E), + (0x223, 0x1, 0x222), + (0x225, 0x1, 0x224), + (0x227, 0x1, 0x226), + (0x229, 0x1, 0x228), + (0x22B, 0x1, 0x22A), + (0x22D, 0x1, 0x22C), + (0x22F, 0x1, 0x22E), + (0x231, 0x1, 0x230), + (0x233, 0x1, 0x232), + (0x23C, 0x1, 0x23B), + (0x23F, 0x1, 0x2C7E), + (0x240, 0x1, 0x2C7F), + (0x242, 0x1, 0x241), + (0x247, 0x1, 0x246), + (0x249, 0x1, 0x248), + (0x24B, 0x1, 0x24A), + (0x24D, 0x1, 0x24C), + (0x24F, 0x1, 0x24E), + (0x250, 0x1, 0x2C6F), + (0x251, 0x1, 0x2C6D), + (0x252, 0x1, 0x2C70), + (0x253, 0x1, 0x181), + (0x254, 0x1, 0x186), + (0x256, 0x1, 0x189), + (0x257, 0x1, 0x18A), + (0x259, 0x1, 0x18F), + (0x25B, 0x1, 0x190), + (0x25C, 0x1, 0xA7AB), + (0x260, 0x1, 0x193), + (0x261, 0x1, 0xA7AC), + (0x263, 0x1, 0x194), + (0x265, 0x1, 0xA78D), + (0x266, 0x1, 0xA7AA), + (0x268, 0x1, 0x197), + (0x269, 0x1, 0x196), + (0x26A, 0x1, 0xA7AE), + (0x26B, 0x1, 0x2C62), + (0x26C, 0x1, 0xA7AD), + (0x26F, 0x1, 0x19C), + (0x271, 0x1, 0x2C6E), + (0x272, 0x1, 0x19D), + (0x275, 0x1, 0x19F), + (0x27D, 0x1, 0x2C64), + (0x280, 0x1, 0x1A6), + (0x282, 0x1, 0xA7C5), + (0x283, 0x1, 0x1A9), + (0x287, 0x1, 0xA7B1), + (0x288, 0x1, 0x1AE), + (0x289, 0x1, 0x244), + (0x28A, 0x1, 0x1B1), + (0x28B, 0x1, 0x1B2), + (0x28C, 0x1, 0x245), + (0x292, 0x1, 0x1B7), + (0x29D, 0x1, 0xA7B2), + (0x29E, 0x1, 0xA7B0), + (0x345, 0x1, 0x399), + (0x371, 0x1, 0x370), + (0x373, 0x1, 0x372), + (0x377, 0x1, 0x376), + (0x37B, 0x1, 0x3FD), + (0x37C, 0x1, 0x3FE), + (0x37D, 0x1, 0x3FF), + (0x390, 0x3, 0x399, 0x308, 0x301), + (0x3AC, 0x1, 0x386), + (0x3AD, 0x1, 0x388), + (0x3AE, 0x1, 0x389), + (0x3AF, 0x1, 0x38A), + (0x3B0, 0x3, 0x3A5, 0x308, 0x301), + (0x3B1, 0x1, 0x391), + (0x3B2, 0x1, 0x392), + (0x3B3, 0x1, 0x393), + (0x3B4, 0x1, 0x394), + (0x3B5, 0x1, 0x395), + (0x3B6, 0x1, 0x396), + (0x3B7, 0x1, 0x397), + (0x3B8, 0x1, 0x398), + (0x3B9, 0x1, 0x399), + (0x3BA, 0x1, 0x39A), + (0x3BB, 0x1, 0x39B), + (0x3BC, 0x1, 0x39C), + (0x3BD, 0x1, 0x39D), + (0x3BE, 0x1, 0x39E), + (0x3BF, 0x1, 0x39F), + (0x3C0, 0x1, 0x3A0), + (0x3C1, 0x1, 0x3A1), + (0x3C2, 0x1, 0x3A3), + (0x3C3, 0x1, 0x3A3), + (0x3C4, 0x1, 0x3A4), + (0x3C5, 0x1, 0x3A5), + (0x3C6, 0x1, 0x3A6), + (0x3C7, 0x1, 0x3A7), + (0x3C8, 0x1, 0x3A8), + (0x3C9, 0x1, 0x3A9), + (0x3CA, 0x1, 0x3AA), + (0x3CB, 0x1, 0x3AB), + (0x3CC, 0x1, 0x38C), + (0x3CD, 0x1, 0x38E), + (0x3CE, 0x1, 0x38F), + (0x3D0, 0x1, 0x392), + (0x3D1, 0x1, 0x398), + (0x3D5, 0x1, 0x3A6), + (0x3D6, 0x1, 0x3A0), + (0x3D7, 0x1, 0x3CF), + (0x3D9, 0x1, 0x3D8), + (0x3DB, 0x1, 0x3DA), + (0x3DD, 0x1, 0x3DC), + (0x3DF, 0x1, 0x3DE), + (0x3E1, 0x1, 0x3E0), + (0x3E3, 0x1, 0x3E2), + (0x3E5, 0x1, 0x3E4), + (0x3E7, 0x1, 0x3E6), + (0x3E9, 0x1, 0x3E8), + (0x3EB, 0x1, 0x3EA), + (0x3ED, 0x1, 0x3EC), + (0x3EF, 0x1, 0x3EE), + (0x3F0, 0x1, 0x39A), + (0x3F1, 0x1, 0x3A1), + (0x3F2, 0x1, 0x3F9), + (0x3F3, 0x1, 0x37F), + (0x3F5, 0x1, 0x395), + (0x3F8, 0x1, 0x3F7), + (0x3FB, 0x1, 0x3FA), + (0x430, 0x1, 0x410), + (0x431, 0x1, 0x411), + (0x432, 0x1, 0x412), + (0x433, 0x1, 0x413), + (0x434, 0x1, 0x414), + (0x435, 0x1, 0x415), + (0x436, 0x1, 0x416), + (0x437, 0x1, 0x417), + (0x438, 0x1, 0x418), + (0x439, 0x1, 0x419), + (0x43A, 0x1, 0x41A), + (0x43B, 0x1, 0x41B), + (0x43C, 0x1, 0x41C), + (0x43D, 0x1, 0x41D), + (0x43E, 0x1, 0x41E), + (0x43F, 0x1, 0x41F), + (0x440, 0x1, 0x420), + (0x441, 0x1, 0x421), + (0x442, 0x1, 0x422), + (0x443, 0x1, 0x423), + (0x444, 0x1, 0x424), + (0x445, 0x1, 0x425), + (0x446, 0x1, 0x426), + (0x447, 0x1, 0x427), + (0x448, 0x1, 0x428), + (0x449, 0x1, 0x429), + (0x44A, 0x1, 0x42A), + (0x44B, 0x1, 0x42B), + (0x44C, 0x1, 0x42C), + (0x44D, 0x1, 0x42D), + (0x44E, 0x1, 0x42E), + (0x44F, 0x1, 0x42F), + (0x450, 0x1, 0x400), + (0x451, 0x1, 0x401), + (0x452, 0x1, 0x402), + (0x453, 0x1, 0x403), + (0x454, 0x1, 0x404), + (0x455, 0x1, 0x405), + (0x456, 0x1, 0x406), + (0x457, 0x1, 0x407), + (0x458, 0x1, 0x408), + (0x459, 0x1, 0x409), + (0x45A, 0x1, 0x40A), + (0x45B, 0x1, 0x40B), + (0x45C, 0x1, 0x40C), + (0x45D, 0x1, 0x40D), + (0x45E, 0x1, 0x40E), + (0x45F, 0x1, 0x40F), + (0x461, 0x1, 0x460), + (0x463, 0x1, 0x462), + (0x465, 0x1, 0x464), + (0x467, 0x1, 0x466), + (0x469, 0x1, 0x468), + (0x46B, 0x1, 0x46A), + (0x46D, 0x1, 0x46C), + (0x46F, 0x1, 0x46E), + (0x471, 0x1, 0x470), + (0x473, 0x1, 0x472), + (0x475, 0x1, 0x474), + (0x477, 0x1, 0x476), + (0x479, 0x1, 0x478), + (0x47B, 0x1, 0x47A), + (0x47D, 0x1, 0x47C), + (0x47F, 0x1, 0x47E), + (0x481, 0x1, 0x480), + (0x48B, 0x1, 0x48A), + (0x48D, 0x1, 0x48C), + (0x48F, 0x1, 0x48E), + (0x491, 0x1, 0x490), + (0x493, 0x1, 0x492), + (0x495, 0x1, 0x494), + (0x497, 0x1, 0x496), + (0x499, 0x1, 0x498), + (0x49B, 0x1, 0x49A), + (0x49D, 0x1, 0x49C), + (0x49F, 0x1, 0x49E), + (0x4A1, 0x1, 0x4A0), + (0x4A3, 0x1, 0x4A2), + (0x4A5, 0x1, 0x4A4), + (0x4A7, 0x1, 0x4A6), + (0x4A9, 0x1, 0x4A8), + (0x4AB, 0x1, 0x4AA), + (0x4AD, 0x1, 0x4AC), + (0x4AF, 0x1, 0x4AE), + (0x4B1, 0x1, 0x4B0), + (0x4B3, 0x1, 0x4B2), + (0x4B5, 0x1, 0x4B4), + (0x4B7, 0x1, 0x4B6), + (0x4B9, 0x1, 0x4B8), + (0x4BB, 0x1, 0x4BA), + (0x4BD, 0x1, 0x4BC), + (0x4BF, 0x1, 0x4BE), + (0x4C2, 0x1, 0x4C1), + (0x4C4, 0x1, 0x4C3), + (0x4C6, 0x1, 0x4C5), + (0x4C8, 0x1, 0x4C7), + (0x4CA, 0x1, 0x4C9), + (0x4CC, 0x1, 0x4CB), + (0x4CE, 0x1, 0x4CD), + (0x4CF, 0x1, 0x4C0), + (0x4D1, 0x1, 0x4D0), + (0x4D3, 0x1, 0x4D2), + (0x4D5, 0x1, 0x4D4), + (0x4D7, 0x1, 0x4D6), + (0x4D9, 0x1, 0x4D8), + (0x4DB, 0x1, 0x4DA), + (0x4DD, 0x1, 0x4DC), + (0x4DF, 0x1, 0x4DE), + (0x4E1, 0x1, 0x4E0), + (0x4E3, 0x1, 0x4E2), + (0x4E5, 0x1, 0x4E4), + (0x4E7, 0x1, 0x4E6), + (0x4E9, 0x1, 0x4E8), + (0x4EB, 0x1, 0x4EA), + (0x4ED, 0x1, 0x4EC), + (0x4EF, 0x1, 0x4EE), + (0x4F1, 0x1, 0x4F0), + (0x4F3, 0x1, 0x4F2), + (0x4F5, 0x1, 0x4F4), + (0x4F7, 0x1, 0x4F6), + (0x4F9, 0x1, 0x4F8), + (0x4FB, 0x1, 0x4FA), + (0x4FD, 0x1, 0x4FC), + (0x4FF, 0x1, 0x4FE), + (0x501, 0x1, 0x500), + (0x503, 0x1, 0x502), + (0x505, 0x1, 0x504), + (0x507, 0x1, 0x506), + (0x509, 0x1, 0x508), + (0x50B, 0x1, 0x50A), + (0x50D, 0x1, 0x50C), + (0x50F, 0x1, 0x50E), + (0x511, 0x1, 0x510), + (0x513, 0x1, 0x512), + (0x515, 0x1, 0x514), + (0x517, 0x1, 0x516), + (0x519, 0x1, 0x518), + (0x51B, 0x1, 0x51A), + (0x51D, 0x1, 0x51C), + (0x51F, 0x1, 0x51E), + (0x521, 0x1, 0x520), + (0x523, 0x1, 0x522), + (0x525, 0x1, 0x524), + (0x527, 0x1, 0x526), + (0x529, 0x1, 0x528), + (0x52B, 0x1, 0x52A), + (0x52D, 0x1, 0x52C), + (0x52F, 0x1, 0x52E), + (0x561, 0x1, 0x531), + (0x562, 0x1, 0x532), + (0x563, 0x1, 0x533), + (0x564, 0x1, 0x534), + (0x565, 0x1, 0x535), + (0x566, 0x1, 0x536), + (0x567, 0x1, 0x537), + (0x568, 0x1, 0x538), + (0x569, 0x1, 0x539), + (0x56A, 0x1, 0x53A), + (0x56B, 0x1, 0x53B), + (0x56C, 0x1, 0x53C), + (0x56D, 0x1, 0x53D), + (0x56E, 0x1, 0x53E), + (0x56F, 0x1, 0x53F), + (0x570, 0x1, 0x540), + (0x571, 0x1, 0x541), + (0x572, 0x1, 0x542), + (0x573, 0x1, 0x543), + (0x574, 0x1, 0x544), + (0x575, 0x1, 0x545), + (0x576, 0x1, 0x546), + (0x577, 0x1, 0x547), + (0x578, 0x1, 0x548), + (0x579, 0x1, 0x549), + (0x57A, 0x1, 0x54A), + (0x57B, 0x1, 0x54B), + (0x57C, 0x1, 0x54C), + (0x57D, 0x1, 0x54D), + (0x57E, 0x1, 0x54E), + (0x57F, 0x1, 0x54F), + (0x580, 0x1, 0x550), + (0x581, 0x1, 0x551), + (0x582, 0x1, 0x552), + (0x583, 0x1, 0x553), + (0x584, 0x1, 0x554), + (0x585, 0x1, 0x555), + (0x586, 0x1, 0x556), + (0x587, 0x2, 0x535, 0x552), + (0x10D0, 0x1, 0x1C90), + (0x10D1, 0x1, 0x1C91), + (0x10D2, 0x1, 0x1C92), + (0x10D3, 0x1, 0x1C93), + (0x10D4, 0x1, 0x1C94), + (0x10D5, 0x1, 0x1C95), + (0x10D6, 0x1, 0x1C96), + (0x10D7, 0x1, 0x1C97), + (0x10D8, 0x1, 0x1C98), + (0x10D9, 0x1, 0x1C99), + (0x10DA, 0x1, 0x1C9A), + (0x10DB, 0x1, 0x1C9B), + (0x10DC, 0x1, 0x1C9C), + (0x10DD, 0x1, 0x1C9D), + (0x10DE, 0x1, 0x1C9E), + (0x10DF, 0x1, 0x1C9F), + (0x10E0, 0x1, 0x1CA0), + (0x10E1, 0x1, 0x1CA1), + (0x10E2, 0x1, 0x1CA2), + (0x10E3, 0x1, 0x1CA3), + (0x10E4, 0x1, 0x1CA4), + (0x10E5, 0x1, 0x1CA5), + (0x10E6, 0x1, 0x1CA6), + (0x10E7, 0x1, 0x1CA7), + (0x10E8, 0x1, 0x1CA8), + (0x10E9, 0x1, 0x1CA9), + (0x10EA, 0x1, 0x1CAA), + (0x10EB, 0x1, 0x1CAB), + (0x10EC, 0x1, 0x1CAC), + (0x10ED, 0x1, 0x1CAD), + (0x10EE, 0x1, 0x1CAE), + (0x10EF, 0x1, 0x1CAF), + (0x10F0, 0x1, 0x1CB0), + (0x10F1, 0x1, 0x1CB1), + (0x10F2, 0x1, 0x1CB2), + (0x10F3, 0x1, 0x1CB3), + (0x10F4, 0x1, 0x1CB4), + (0x10F5, 0x1, 0x1CB5), + (0x10F6, 0x1, 0x1CB6), + (0x10F7, 0x1, 0x1CB7), + (0x10F8, 0x1, 0x1CB8), + (0x10F9, 0x1, 0x1CB9), + (0x10FA, 0x1, 0x1CBA), + (0x10FD, 0x1, 0x1CBD), + (0x10FE, 0x1, 0x1CBE), + (0x10FF, 0x1, 0x1CBF), + (0x13F8, 0x1, 0x13F0), + (0x13F9, 0x1, 0x13F1), + (0x13FA, 0x1, 0x13F2), + (0x13FB, 0x1, 0x13F3), + (0x13FC, 0x1, 0x13F4), + (0x13FD, 0x1, 0x13F5), + (0x1C80, 0x1, 0x412), + (0x1C81, 0x1, 0x414), + (0x1C82, 0x1, 0x41E), + (0x1C83, 0x1, 0x421), + (0x1C84, 0x1, 0x422), + (0x1C85, 0x1, 0x422), + (0x1C86, 0x1, 0x42A), + (0x1C87, 0x1, 0x462), + (0x1C88, 0x1, 0xA64A), + (0x1D79, 0x1, 0xA77D), + (0x1D7D, 0x1, 0x2C63), + (0x1D8E, 0x1, 0xA7C6), + (0x1E01, 0x1, 0x1E00), + (0x1E03, 0x1, 0x1E02), + (0x1E05, 0x1, 0x1E04), + (0x1E07, 0x1, 0x1E06), + (0x1E09, 0x1, 0x1E08), + (0x1E0B, 0x1, 0x1E0A), + (0x1E0D, 0x1, 0x1E0C), + (0x1E0F, 0x1, 0x1E0E), + (0x1E11, 0x1, 0x1E10), + (0x1E13, 0x1, 0x1E12), + (0x1E15, 0x1, 0x1E14), + (0x1E17, 0x1, 0x1E16), + (0x1E19, 0x1, 0x1E18), + (0x1E1B, 0x1, 0x1E1A), + (0x1E1D, 0x1, 0x1E1C), + (0x1E1F, 0x1, 0x1E1E), + (0x1E21, 0x1, 0x1E20), + (0x1E23, 0x1, 0x1E22), + (0x1E25, 0x1, 0x1E24), + (0x1E27, 0x1, 0x1E26), + (0x1E29, 0x1, 0x1E28), + (0x1E2B, 0x1, 0x1E2A), + (0x1E2D, 0x1, 0x1E2C), + (0x1E2F, 0x1, 0x1E2E), + (0x1E31, 0x1, 0x1E30), + (0x1E33, 0x1, 0x1E32), + (0x1E35, 0x1, 0x1E34), + (0x1E37, 0x1, 0x1E36), + (0x1E39, 0x1, 0x1E38), + (0x1E3B, 0x1, 0x1E3A), + (0x1E3D, 0x1, 0x1E3C), + (0x1E3F, 0x1, 0x1E3E), + (0x1E41, 0x1, 0x1E40), + (0x1E43, 0x1, 0x1E42), + (0x1E45, 0x1, 0x1E44), + (0x1E47, 0x1, 0x1E46), + (0x1E49, 0x1, 0x1E48), + (0x1E4B, 0x1, 0x1E4A), + (0x1E4D, 0x1, 0x1E4C), + (0x1E4F, 0x1, 0x1E4E), + (0x1E51, 0x1, 0x1E50), + (0x1E53, 0x1, 0x1E52), + (0x1E55, 0x1, 0x1E54), + (0x1E57, 0x1, 0x1E56), + (0x1E59, 0x1, 0x1E58), + (0x1E5B, 0x1, 0x1E5A), + (0x1E5D, 0x1, 0x1E5C), + (0x1E5F, 0x1, 0x1E5E), + (0x1E61, 0x1, 0x1E60), + (0x1E63, 0x1, 0x1E62), + (0x1E65, 0x1, 0x1E64), + (0x1E67, 0x1, 0x1E66), + (0x1E69, 0x1, 0x1E68), + (0x1E6B, 0x1, 0x1E6A), + (0x1E6D, 0x1, 0x1E6C), + (0x1E6F, 0x1, 0x1E6E), + (0x1E71, 0x1, 0x1E70), + (0x1E73, 0x1, 0x1E72), + (0x1E75, 0x1, 0x1E74), + (0x1E77, 0x1, 0x1E76), + (0x1E79, 0x1, 0x1E78), + (0x1E7B, 0x1, 0x1E7A), + (0x1E7D, 0x1, 0x1E7C), + (0x1E7F, 0x1, 0x1E7E), + (0x1E81, 0x1, 0x1E80), + (0x1E83, 0x1, 0x1E82), + (0x1E85, 0x1, 0x1E84), + (0x1E87, 0x1, 0x1E86), + (0x1E89, 0x1, 0x1E88), + (0x1E8B, 0x1, 0x1E8A), + (0x1E8D, 0x1, 0x1E8C), + (0x1E8F, 0x1, 0x1E8E), + (0x1E91, 0x1, 0x1E90), + (0x1E93, 0x1, 0x1E92), + (0x1E95, 0x1, 0x1E94), + (0x1E96, 0x2, 0x48, 0x331), + (0x1E97, 0x2, 0x54, 0x308), + (0x1E98, 0x2, 0x57, 0x30A), + (0x1E99, 0x2, 0x59, 0x30A), + (0x1E9A, 0x2, 0x41, 0x2BE), + (0x1E9B, 0x1, 0x1E60), + (0x1EA1, 0x1, 0x1EA0), + (0x1EA3, 0x1, 0x1EA2), + (0x1EA5, 0x1, 0x1EA4), + (0x1EA7, 0x1, 0x1EA6), + (0x1EA9, 0x1, 0x1EA8), + (0x1EAB, 0x1, 0x1EAA), + (0x1EAD, 0x1, 0x1EAC), + (0x1EAF, 0x1, 0x1EAE), + (0x1EB1, 0x1, 0x1EB0), + (0x1EB3, 0x1, 0x1EB2), + (0x1EB5, 0x1, 0x1EB4), + (0x1EB7, 0x1, 0x1EB6), + (0x1EB9, 0x1, 0x1EB8), + (0x1EBB, 0x1, 0x1EBA), + (0x1EBD, 0x1, 0x1EBC), + (0x1EBF, 0x1, 0x1EBE), + (0x1EC1, 0x1, 0x1EC0), + (0x1EC3, 0x1, 0x1EC2), + (0x1EC5, 0x1, 0x1EC4), + (0x1EC7, 0x1, 0x1EC6), + (0x1EC9, 0x1, 0x1EC8), + (0x1ECB, 0x1, 0x1ECA), + (0x1ECD, 0x1, 0x1ECC), + (0x1ECF, 0x1, 0x1ECE), + (0x1ED1, 0x1, 0x1ED0), + (0x1ED3, 0x1, 0x1ED2), + (0x1ED5, 0x1, 0x1ED4), + (0x1ED7, 0x1, 0x1ED6), + (0x1ED9, 0x1, 0x1ED8), + (0x1EDB, 0x1, 0x1EDA), + (0x1EDD, 0x1, 0x1EDC), + (0x1EDF, 0x1, 0x1EDE), + (0x1EE1, 0x1, 0x1EE0), + (0x1EE3, 0x1, 0x1EE2), + (0x1EE5, 0x1, 0x1EE4), + (0x1EE7, 0x1, 0x1EE6), + (0x1EE9, 0x1, 0x1EE8), + (0x1EEB, 0x1, 0x1EEA), + (0x1EED, 0x1, 0x1EEC), + (0x1EEF, 0x1, 0x1EEE), + (0x1EF1, 0x1, 0x1EF0), + (0x1EF3, 0x1, 0x1EF2), + (0x1EF5, 0x1, 0x1EF4), + (0x1EF7, 0x1, 0x1EF6), + (0x1EF9, 0x1, 0x1EF8), + (0x1EFB, 0x1, 0x1EFA), + (0x1EFD, 0x1, 0x1EFC), + (0x1EFF, 0x1, 0x1EFE), + (0x1F00, 0x1, 0x1F08), + (0x1F01, 0x1, 0x1F09), + (0x1F02, 0x1, 0x1F0A), + (0x1F03, 0x1, 0x1F0B), + (0x1F04, 0x1, 0x1F0C), + (0x1F05, 0x1, 0x1F0D), + (0x1F06, 0x1, 0x1F0E), + (0x1F07, 0x1, 0x1F0F), + (0x1F10, 0x1, 0x1F18), + (0x1F11, 0x1, 0x1F19), + (0x1F12, 0x1, 0x1F1A), + (0x1F13, 0x1, 0x1F1B), + (0x1F14, 0x1, 0x1F1C), + (0x1F15, 0x1, 0x1F1D), + (0x1F20, 0x1, 0x1F28), + (0x1F21, 0x1, 0x1F29), + (0x1F22, 0x1, 0x1F2A), + (0x1F23, 0x1, 0x1F2B), + (0x1F24, 0x1, 0x1F2C), + (0x1F25, 0x1, 0x1F2D), + (0x1F26, 0x1, 0x1F2E), + (0x1F27, 0x1, 0x1F2F), + (0x1F30, 0x1, 0x1F38), + (0x1F31, 0x1, 0x1F39), + (0x1F32, 0x1, 0x1F3A), + (0x1F33, 0x1, 0x1F3B), + (0x1F34, 0x1, 0x1F3C), + (0x1F35, 0x1, 0x1F3D), + (0x1F36, 0x1, 0x1F3E), + (0x1F37, 0x1, 0x1F3F), + (0x1F40, 0x1, 0x1F48), + (0x1F41, 0x1, 0x1F49), + (0x1F42, 0x1, 0x1F4A), + (0x1F43, 0x1, 0x1F4B), + (0x1F44, 0x1, 0x1F4C), + (0x1F45, 0x1, 0x1F4D), + (0x1F50, 0x2, 0x3A5, 0x313), + (0x1F51, 0x1, 0x1F59), + (0x1F52, 0x3, 0x3A5, 0x313, 0x300), + (0x1F53, 0x1, 0x1F5B), + (0x1F54, 0x3, 0x3A5, 0x313, 0x301), + (0x1F55, 0x1, 0x1F5D), + (0x1F56, 0x3, 0x3A5, 0x313, 0x342), + (0x1F57, 0x1, 0x1F5F), + (0x1F60, 0x1, 0x1F68), + (0x1F61, 0x1, 0x1F69), + (0x1F62, 0x1, 0x1F6A), + (0x1F63, 0x1, 0x1F6B), + (0x1F64, 0x1, 0x1F6C), + (0x1F65, 0x1, 0x1F6D), + (0x1F66, 0x1, 0x1F6E), + (0x1F67, 0x1, 0x1F6F), + (0x1F70, 0x1, 0x1FBA), + (0x1F71, 0x1, 0x1FBB), + (0x1F72, 0x1, 0x1FC8), + (0x1F73, 0x1, 0x1FC9), + (0x1F74, 0x1, 0x1FCA), + (0x1F75, 0x1, 0x1FCB), + (0x1F76, 0x1, 0x1FDA), + (0x1F77, 0x1, 0x1FDB), + (0x1F78, 0x1, 0x1FF8), + (0x1F79, 0x1, 0x1FF9), + (0x1F7A, 0x1, 0x1FEA), + (0x1F7B, 0x1, 0x1FEB), + (0x1F7C, 0x1, 0x1FFA), + (0x1F7D, 0x1, 0x1FFB), + (0x1F80, 0x2, 0x1F08, 0x399), + (0x1F81, 0x2, 0x1F09, 0x399), + (0x1F82, 0x2, 0x1F0A, 0x399), + (0x1F83, 0x2, 0x1F0B, 0x399), + (0x1F84, 0x2, 0x1F0C, 0x399), + (0x1F85, 0x2, 0x1F0D, 0x399), + (0x1F86, 0x2, 0x1F0E, 0x399), + (0x1F87, 0x2, 0x1F0F, 0x399), + (0x1F88, 0x2, 0x1F08, 0x399), + (0x1F89, 0x2, 0x1F09, 0x399), + (0x1F8A, 0x2, 0x1F0A, 0x399), + (0x1F8B, 0x2, 0x1F0B, 0x399), + (0x1F8C, 0x2, 0x1F0C, 0x399), + (0x1F8D, 0x2, 0x1F0D, 0x399), + (0x1F8E, 0x2, 0x1F0E, 0x399), + (0x1F8F, 0x2, 0x1F0F, 0x399), + (0x1F90, 0x2, 0x1F28, 0x399), + (0x1F91, 0x2, 0x1F29, 0x399), + (0x1F92, 0x2, 0x1F2A, 0x399), + (0x1F93, 0x2, 0x1F2B, 0x399), + (0x1F94, 0x2, 0x1F2C, 0x399), + (0x1F95, 0x2, 0x1F2D, 0x399), + (0x1F96, 0x2, 0x1F2E, 0x399), + (0x1F97, 0x2, 0x1F2F, 0x399), + (0x1F98, 0x2, 0x1F28, 0x399), + (0x1F99, 0x2, 0x1F29, 0x399), + (0x1F9A, 0x2, 0x1F2A, 0x399), + (0x1F9B, 0x2, 0x1F2B, 0x399), + (0x1F9C, 0x2, 0x1F2C, 0x399), + (0x1F9D, 0x2, 0x1F2D, 0x399), + (0x1F9E, 0x2, 0x1F2E, 0x399), + (0x1F9F, 0x2, 0x1F2F, 0x399), + (0x1FA0, 0x2, 0x1F68, 0x399), + (0x1FA1, 0x2, 0x1F69, 0x399), + (0x1FA2, 0x2, 0x1F6A, 0x399), + (0x1FA3, 0x2, 0x1F6B, 0x399), + (0x1FA4, 0x2, 0x1F6C, 0x399), + (0x1FA5, 0x2, 0x1F6D, 0x399), + (0x1FA6, 0x2, 0x1F6E, 0x399), + (0x1FA7, 0x2, 0x1F6F, 0x399), + (0x1FA8, 0x2, 0x1F68, 0x399), + (0x1FA9, 0x2, 0x1F69, 0x399), + (0x1FAA, 0x2, 0x1F6A, 0x399), + (0x1FAB, 0x2, 0x1F6B, 0x399), + (0x1FAC, 0x2, 0x1F6C, 0x399), + (0x1FAD, 0x2, 0x1F6D, 0x399), + (0x1FAE, 0x2, 0x1F6E, 0x399), + (0x1FAF, 0x2, 0x1F6F, 0x399), + (0x1FB0, 0x1, 0x1FB8), + (0x1FB1, 0x1, 0x1FB9), + (0x1FB2, 0x2, 0x1FBA, 0x399), + (0x1FB3, 0x2, 0x391, 0x399), + (0x1FB4, 0x2, 0x386, 0x399), + (0x1FB6, 0x2, 0x391, 0x342), + (0x1FB7, 0x3, 0x391, 0x342, 0x399), + (0x1FBC, 0x2, 0x391, 0x399), + (0x1FBE, 0x1, 0x399), + (0x1FC2, 0x2, 0x1FCA, 0x399), + (0x1FC3, 0x2, 0x397, 0x399), + (0x1FC4, 0x2, 0x389, 0x399), + (0x1FC6, 0x2, 0x397, 0x342), + (0x1FC7, 0x3, 0x397, 0x342, 0x399), + (0x1FCC, 0x2, 0x397, 0x399), + (0x1FD0, 0x1, 0x1FD8), + (0x1FD1, 0x1, 0x1FD9), + (0x1FD2, 0x3, 0x399, 0x308, 0x300), + (0x1FD3, 0x3, 0x399, 0x308, 0x301), + (0x1FD6, 0x2, 0x399, 0x342), + (0x1FD7, 0x3, 0x399, 0x308, 0x342), + (0x1FE0, 0x1, 0x1FE8), + (0x1FE1, 0x1, 0x1FE9), + (0x1FE2, 0x3, 0x3A5, 0x308, 0x300), + (0x1FE3, 0x3, 0x3A5, 0x308, 0x301), + (0x1FE4, 0x2, 0x3A1, 0x313), + (0x1FE5, 0x1, 0x1FEC), + (0x1FE6, 0x2, 0x3A5, 0x342), + (0x1FE7, 0x3, 0x3A5, 0x308, 0x342), + (0x1FF2, 0x2, 0x1FFA, 0x399), + (0x1FF3, 0x2, 0x3A9, 0x399), + (0x1FF4, 0x2, 0x38F, 0x399), + (0x1FF6, 0x2, 0x3A9, 0x342), + (0x1FF7, 0x3, 0x3A9, 0x342, 0x399), + (0x1FFC, 0x2, 0x3A9, 0x399), + (0x214E, 0x1, 0x2132), + (0x2170, 0x1, 0x2160), + (0x2171, 0x1, 0x2161), + (0x2172, 0x1, 0x2162), + (0x2173, 0x1, 0x2163), + (0x2174, 0x1, 0x2164), + (0x2175, 0x1, 0x2165), + (0x2176, 0x1, 0x2166), + (0x2177, 0x1, 0x2167), + (0x2178, 0x1, 0x2168), + (0x2179, 0x1, 0x2169), + (0x217A, 0x1, 0x216A), + (0x217B, 0x1, 0x216B), + (0x217C, 0x1, 0x216C), + (0x217D, 0x1, 0x216D), + (0x217E, 0x1, 0x216E), + (0x217F, 0x1, 0x216F), + (0x2184, 0x1, 0x2183), + (0x24D0, 0x1, 0x24B6), + (0x24D1, 0x1, 0x24B7), + (0x24D2, 0x1, 0x24B8), + (0x24D3, 0x1, 0x24B9), + (0x24D4, 0x1, 0x24BA), + (0x24D5, 0x1, 0x24BB), + (0x24D6, 0x1, 0x24BC), + (0x24D7, 0x1, 0x24BD), + (0x24D8, 0x1, 0x24BE), + (0x24D9, 0x1, 0x24BF), + (0x24DA, 0x1, 0x24C0), + (0x24DB, 0x1, 0x24C1), + (0x24DC, 0x1, 0x24C2), + (0x24DD, 0x1, 0x24C3), + (0x24DE, 0x1, 0x24C4), + (0x24DF, 0x1, 0x24C5), + (0x24E0, 0x1, 0x24C6), + (0x24E1, 0x1, 0x24C7), + (0x24E2, 0x1, 0x24C8), + (0x24E3, 0x1, 0x24C9), + (0x24E4, 0x1, 0x24CA), + (0x24E5, 0x1, 0x24CB), + (0x24E6, 0x1, 0x24CC), + (0x24E7, 0x1, 0x24CD), + (0x24E8, 0x1, 0x24CE), + (0x24E9, 0x1, 0x24CF), + (0x2C30, 0x1, 0x2C00), + (0x2C31, 0x1, 0x2C01), + (0x2C32, 0x1, 0x2C02), + (0x2C33, 0x1, 0x2C03), + (0x2C34, 0x1, 0x2C04), + (0x2C35, 0x1, 0x2C05), + (0x2C36, 0x1, 0x2C06), + (0x2C37, 0x1, 0x2C07), + (0x2C38, 0x1, 0x2C08), + (0x2C39, 0x1, 0x2C09), + (0x2C3A, 0x1, 0x2C0A), + (0x2C3B, 0x1, 0x2C0B), + (0x2C3C, 0x1, 0x2C0C), + (0x2C3D, 0x1, 0x2C0D), + (0x2C3E, 0x1, 0x2C0E), + (0x2C3F, 0x1, 0x2C0F), + (0x2C40, 0x1, 0x2C10), + (0x2C41, 0x1, 0x2C11), + (0x2C42, 0x1, 0x2C12), + (0x2C43, 0x1, 0x2C13), + (0x2C44, 0x1, 0x2C14), + (0x2C45, 0x1, 0x2C15), + (0x2C46, 0x1, 0x2C16), + (0x2C47, 0x1, 0x2C17), + (0x2C48, 0x1, 0x2C18), + (0x2C49, 0x1, 0x2C19), + (0x2C4A, 0x1, 0x2C1A), + (0x2C4B, 0x1, 0x2C1B), + (0x2C4C, 0x1, 0x2C1C), + (0x2C4D, 0x1, 0x2C1D), + (0x2C4E, 0x1, 0x2C1E), + (0x2C4F, 0x1, 0x2C1F), + (0x2C50, 0x1, 0x2C20), + (0x2C51, 0x1, 0x2C21), + (0x2C52, 0x1, 0x2C22), + (0x2C53, 0x1, 0x2C23), + (0x2C54, 0x1, 0x2C24), + (0x2C55, 0x1, 0x2C25), + (0x2C56, 0x1, 0x2C26), + (0x2C57, 0x1, 0x2C27), + (0x2C58, 0x1, 0x2C28), + (0x2C59, 0x1, 0x2C29), + (0x2C5A, 0x1, 0x2C2A), + (0x2C5B, 0x1, 0x2C2B), + (0x2C5C, 0x1, 0x2C2C), + (0x2C5D, 0x1, 0x2C2D), + (0x2C5E, 0x1, 0x2C2E), + (0x2C5F, 0x1, 0x2C2F), + (0x2C61, 0x1, 0x2C60), + (0x2C65, 0x1, 0x23A), + (0x2C66, 0x1, 0x23E), + (0x2C68, 0x1, 0x2C67), + (0x2C6A, 0x1, 0x2C69), + (0x2C6C, 0x1, 0x2C6B), + (0x2C73, 0x1, 0x2C72), + (0x2C76, 0x1, 0x2C75), + (0x2C81, 0x1, 0x2C80), + (0x2C83, 0x1, 0x2C82), + (0x2C85, 0x1, 0x2C84), + (0x2C87, 0x1, 0x2C86), + (0x2C89, 0x1, 0x2C88), + (0x2C8B, 0x1, 0x2C8A), + (0x2C8D, 0x1, 0x2C8C), + (0x2C8F, 0x1, 0x2C8E), + (0x2C91, 0x1, 0x2C90), + (0x2C93, 0x1, 0x2C92), + (0x2C95, 0x1, 0x2C94), + (0x2C97, 0x1, 0x2C96), + (0x2C99, 0x1, 0x2C98), + (0x2C9B, 0x1, 0x2C9A), + (0x2C9D, 0x1, 0x2C9C), + (0x2C9F, 0x1, 0x2C9E), + (0x2CA1, 0x1, 0x2CA0), + (0x2CA3, 0x1, 0x2CA2), + (0x2CA5, 0x1, 0x2CA4), + (0x2CA7, 0x1, 0x2CA6), + (0x2CA9, 0x1, 0x2CA8), + (0x2CAB, 0x1, 0x2CAA), + (0x2CAD, 0x1, 0x2CAC), + (0x2CAF, 0x1, 0x2CAE), + (0x2CB1, 0x1, 0x2CB0), + (0x2CB3, 0x1, 0x2CB2), + (0x2CB5, 0x1, 0x2CB4), + (0x2CB7, 0x1, 0x2CB6), + (0x2CB9, 0x1, 0x2CB8), + (0x2CBB, 0x1, 0x2CBA), + (0x2CBD, 0x1, 0x2CBC), + (0x2CBF, 0x1, 0x2CBE), + (0x2CC1, 0x1, 0x2CC0), + (0x2CC3, 0x1, 0x2CC2), + (0x2CC5, 0x1, 0x2CC4), + (0x2CC7, 0x1, 0x2CC6), + (0x2CC9, 0x1, 0x2CC8), + (0x2CCB, 0x1, 0x2CCA), + (0x2CCD, 0x1, 0x2CCC), + (0x2CCF, 0x1, 0x2CCE), + (0x2CD1, 0x1, 0x2CD0), + (0x2CD3, 0x1, 0x2CD2), + (0x2CD5, 0x1, 0x2CD4), + (0x2CD7, 0x1, 0x2CD6), + (0x2CD9, 0x1, 0x2CD8), + (0x2CDB, 0x1, 0x2CDA), + (0x2CDD, 0x1, 0x2CDC), + (0x2CDF, 0x1, 0x2CDE), + (0x2CE1, 0x1, 0x2CE0), + (0x2CE3, 0x1, 0x2CE2), + (0x2CEC, 0x1, 0x2CEB), + (0x2CEE, 0x1, 0x2CED), + (0x2CF3, 0x1, 0x2CF2), + (0x2D00, 0x1, 0x10A0), + (0x2D01, 0x1, 0x10A1), + (0x2D02, 0x1, 0x10A2), + (0x2D03, 0x1, 0x10A3), + (0x2D04, 0x1, 0x10A4), + (0x2D05, 0x1, 0x10A5), + (0x2D06, 0x1, 0x10A6), + (0x2D07, 0x1, 0x10A7), + (0x2D08, 0x1, 0x10A8), + (0x2D09, 0x1, 0x10A9), + (0x2D0A, 0x1, 0x10AA), + (0x2D0B, 0x1, 0x10AB), + (0x2D0C, 0x1, 0x10AC), + (0x2D0D, 0x1, 0x10AD), + (0x2D0E, 0x1, 0x10AE), + (0x2D0F, 0x1, 0x10AF), + (0x2D10, 0x1, 0x10B0), + (0x2D11, 0x1, 0x10B1), + (0x2D12, 0x1, 0x10B2), + (0x2D13, 0x1, 0x10B3), + (0x2D14, 0x1, 0x10B4), + (0x2D15, 0x1, 0x10B5), + (0x2D16, 0x1, 0x10B6), + (0x2D17, 0x1, 0x10B7), + (0x2D18, 0x1, 0x10B8), + (0x2D19, 0x1, 0x10B9), + (0x2D1A, 0x1, 0x10BA), + (0x2D1B, 0x1, 0x10BB), + (0x2D1C, 0x1, 0x10BC), + (0x2D1D, 0x1, 0x10BD), + (0x2D1E, 0x1, 0x10BE), + (0x2D1F, 0x1, 0x10BF), + (0x2D20, 0x1, 0x10C0), + (0x2D21, 0x1, 0x10C1), + (0x2D22, 0x1, 0x10C2), + (0x2D23, 0x1, 0x10C3), + (0x2D24, 0x1, 0x10C4), + (0x2D25, 0x1, 0x10C5), + (0x2D27, 0x1, 0x10C7), + (0x2D2D, 0x1, 0x10CD), + (0xA641, 0x1, 0xA640), + (0xA643, 0x1, 0xA642), + (0xA645, 0x1, 0xA644), + (0xA647, 0x1, 0xA646), + (0xA649, 0x1, 0xA648), + (0xA64B, 0x1, 0xA64A), + (0xA64D, 0x1, 0xA64C), + (0xA64F, 0x1, 0xA64E), + (0xA651, 0x1, 0xA650), + (0xA653, 0x1, 0xA652), + (0xA655, 0x1, 0xA654), + (0xA657, 0x1, 0xA656), + (0xA659, 0x1, 0xA658), + (0xA65B, 0x1, 0xA65A), + (0xA65D, 0x1, 0xA65C), + (0xA65F, 0x1, 0xA65E), + (0xA661, 0x1, 0xA660), + (0xA663, 0x1, 0xA662), + (0xA665, 0x1, 0xA664), + (0xA667, 0x1, 0xA666), + (0xA669, 0x1, 0xA668), + (0xA66B, 0x1, 0xA66A), + (0xA66D, 0x1, 0xA66C), + (0xA681, 0x1, 0xA680), + (0xA683, 0x1, 0xA682), + (0xA685, 0x1, 0xA684), + (0xA687, 0x1, 0xA686), + (0xA689, 0x1, 0xA688), + (0xA68B, 0x1, 0xA68A), + (0xA68D, 0x1, 0xA68C), + (0xA68F, 0x1, 0xA68E), + (0xA691, 0x1, 0xA690), + (0xA693, 0x1, 0xA692), + (0xA695, 0x1, 0xA694), + (0xA697, 0x1, 0xA696), + (0xA699, 0x1, 0xA698), + (0xA69B, 0x1, 0xA69A), + (0xA723, 0x1, 0xA722), + (0xA725, 0x1, 0xA724), + (0xA727, 0x1, 0xA726), + (0xA729, 0x1, 0xA728), + (0xA72B, 0x1, 0xA72A), + (0xA72D, 0x1, 0xA72C), + (0xA72F, 0x1, 0xA72E), + (0xA733, 0x1, 0xA732), + (0xA735, 0x1, 0xA734), + (0xA737, 0x1, 0xA736), + (0xA739, 0x1, 0xA738), + (0xA73B, 0x1, 0xA73A), + (0xA73D, 0x1, 0xA73C), + (0xA73F, 0x1, 0xA73E), + (0xA741, 0x1, 0xA740), + (0xA743, 0x1, 0xA742), + (0xA745, 0x1, 0xA744), + (0xA747, 0x1, 0xA746), + (0xA749, 0x1, 0xA748), + (0xA74B, 0x1, 0xA74A), + (0xA74D, 0x1, 0xA74C), + (0xA74F, 0x1, 0xA74E), + (0xA751, 0x1, 0xA750), + (0xA753, 0x1, 0xA752), + (0xA755, 0x1, 0xA754), + (0xA757, 0x1, 0xA756), + (0xA759, 0x1, 0xA758), + (0xA75B, 0x1, 0xA75A), + (0xA75D, 0x1, 0xA75C), + (0xA75F, 0x1, 0xA75E), + (0xA761, 0x1, 0xA760), + (0xA763, 0x1, 0xA762), + (0xA765, 0x1, 0xA764), + (0xA767, 0x1, 0xA766), + (0xA769, 0x1, 0xA768), + (0xA76B, 0x1, 0xA76A), + (0xA76D, 0x1, 0xA76C), + (0xA76F, 0x1, 0xA76E), + (0xA77A, 0x1, 0xA779), + (0xA77C, 0x1, 0xA77B), + (0xA77F, 0x1, 0xA77E), + (0xA781, 0x1, 0xA780), + (0xA783, 0x1, 0xA782), + (0xA785, 0x1, 0xA784), + (0xA787, 0x1, 0xA786), + (0xA78C, 0x1, 0xA78B), + (0xA791, 0x1, 0xA790), + (0xA793, 0x1, 0xA792), + (0xA794, 0x1, 0xA7C4), + (0xA797, 0x1, 0xA796), + (0xA799, 0x1, 0xA798), + (0xA79B, 0x1, 0xA79A), + (0xA79D, 0x1, 0xA79C), + (0xA79F, 0x1, 0xA79E), + (0xA7A1, 0x1, 0xA7A0), + (0xA7A3, 0x1, 0xA7A2), + (0xA7A5, 0x1, 0xA7A4), + (0xA7A7, 0x1, 0xA7A6), + (0xA7A9, 0x1, 0xA7A8), + (0xA7B5, 0x1, 0xA7B4), + (0xA7B7, 0x1, 0xA7B6), + (0xA7B9, 0x1, 0xA7B8), + (0xA7BB, 0x1, 0xA7BA), + (0xA7BD, 0x1, 0xA7BC), + (0xA7BF, 0x1, 0xA7BE), + (0xA7C1, 0x1, 0xA7C0), + (0xA7C3, 0x1, 0xA7C2), + (0xA7C8, 0x1, 0xA7C7), + (0xA7CA, 0x1, 0xA7C9), + (0xA7D1, 0x1, 0xA7D0), + (0xA7D7, 0x1, 0xA7D6), + (0xA7D9, 0x1, 0xA7D8), + (0xA7F6, 0x1, 0xA7F5), + (0xAB53, 0x1, 0xA7B3), + (0xAB70, 0x1, 0x13A0), + (0xAB71, 0x1, 0x13A1), + (0xAB72, 0x1, 0x13A2), + (0xAB73, 0x1, 0x13A3), + (0xAB74, 0x1, 0x13A4), + (0xAB75, 0x1, 0x13A5), + (0xAB76, 0x1, 0x13A6), + (0xAB77, 0x1, 0x13A7), + (0xAB78, 0x1, 0x13A8), + (0xAB79, 0x1, 0x13A9), + (0xAB7A, 0x1, 0x13AA), + (0xAB7B, 0x1, 0x13AB), + (0xAB7C, 0x1, 0x13AC), + (0xAB7D, 0x1, 0x13AD), + (0xAB7E, 0x1, 0x13AE), + (0xAB7F, 0x1, 0x13AF), + (0xAB80, 0x1, 0x13B0), + (0xAB81, 0x1, 0x13B1), + (0xAB82, 0x1, 0x13B2), + (0xAB83, 0x1, 0x13B3), + (0xAB84, 0x1, 0x13B4), + (0xAB85, 0x1, 0x13B5), + (0xAB86, 0x1, 0x13B6), + (0xAB87, 0x1, 0x13B7), + (0xAB88, 0x1, 0x13B8), + (0xAB89, 0x1, 0x13B9), + (0xAB8A, 0x1, 0x13BA), + (0xAB8B, 0x1, 0x13BB), + (0xAB8C, 0x1, 0x13BC), + (0xAB8D, 0x1, 0x13BD), + (0xAB8E, 0x1, 0x13BE), + (0xAB8F, 0x1, 0x13BF), + (0xAB90, 0x1, 0x13C0), + (0xAB91, 0x1, 0x13C1), + (0xAB92, 0x1, 0x13C2), + (0xAB93, 0x1, 0x13C3), + (0xAB94, 0x1, 0x13C4), + (0xAB95, 0x1, 0x13C5), + (0xAB96, 0x1, 0x13C6), + (0xAB97, 0x1, 0x13C7), + (0xAB98, 0x1, 0x13C8), + (0xAB99, 0x1, 0x13C9), + (0xAB9A, 0x1, 0x13CA), + (0xAB9B, 0x1, 0x13CB), + (0xAB9C, 0x1, 0x13CC), + (0xAB9D, 0x1, 0x13CD), + (0xAB9E, 0x1, 0x13CE), + (0xAB9F, 0x1, 0x13CF), + (0xABA0, 0x1, 0x13D0), + (0xABA1, 0x1, 0x13D1), + (0xABA2, 0x1, 0x13D2), + (0xABA3, 0x1, 0x13D3), + (0xABA4, 0x1, 0x13D4), + (0xABA5, 0x1, 0x13D5), + (0xABA6, 0x1, 0x13D6), + (0xABA7, 0x1, 0x13D7), + (0xABA8, 0x1, 0x13D8), + (0xABA9, 0x1, 0x13D9), + (0xABAA, 0x1, 0x13DA), + (0xABAB, 0x1, 0x13DB), + (0xABAC, 0x1, 0x13DC), + (0xABAD, 0x1, 0x13DD), + (0xABAE, 0x1, 0x13DE), + (0xABAF, 0x1, 0x13DF), + (0xABB0, 0x1, 0x13E0), + (0xABB1, 0x1, 0x13E1), + (0xABB2, 0x1, 0x13E2), + (0xABB3, 0x1, 0x13E3), + (0xABB4, 0x1, 0x13E4), + (0xABB5, 0x1, 0x13E5), + (0xABB6, 0x1, 0x13E6), + (0xABB7, 0x1, 0x13E7), + (0xABB8, 0x1, 0x13E8), + (0xABB9, 0x1, 0x13E9), + (0xABBA, 0x1, 0x13EA), + (0xABBB, 0x1, 0x13EB), + (0xABBC, 0x1, 0x13EC), + (0xABBD, 0x1, 0x13ED), + (0xABBE, 0x1, 0x13EE), + (0xABBF, 0x1, 0x13EF), + (0xFB00, 0x2, 0x46, 0x46), + (0xFB01, 0x2, 0x46, 0x49), + (0xFB02, 0x2, 0x46, 0x4C), + (0xFB03, 0x3, 0x46, 0x46, 0x49), + (0xFB04, 0x3, 0x46, 0x46, 0x4C), + (0xFB05, 0x2, 0x53, 0x54), + (0xFB06, 0x2, 0x53, 0x54), + (0xFB13, 0x2, 0x544, 0x546), + (0xFB14, 0x2, 0x544, 0x535), + (0xFB15, 0x2, 0x544, 0x53B), + (0xFB16, 0x2, 0x54E, 0x546), + (0xFB17, 0x2, 0x544, 0x53D), + (0xFF41, 0x1, 0xFF21), + (0xFF42, 0x1, 0xFF22), + (0xFF43, 0x1, 0xFF23), + (0xFF44, 0x1, 0xFF24), + (0xFF45, 0x1, 0xFF25), + (0xFF46, 0x1, 0xFF26), + (0xFF47, 0x1, 0xFF27), + (0xFF48, 0x1, 0xFF28), + (0xFF49, 0x1, 0xFF29), + (0xFF4A, 0x1, 0xFF2A), + (0xFF4B, 0x1, 0xFF2B), + (0xFF4C, 0x1, 0xFF2C), + (0xFF4D, 0x1, 0xFF2D), + (0xFF4E, 0x1, 0xFF2E), + (0xFF4F, 0x1, 0xFF2F), + (0xFF50, 0x1, 0xFF30), + (0xFF51, 0x1, 0xFF31), + (0xFF52, 0x1, 0xFF32), + (0xFF53, 0x1, 0xFF33), + (0xFF54, 0x1, 0xFF34), + (0xFF55, 0x1, 0xFF35), + (0xFF56, 0x1, 0xFF36), + (0xFF57, 0x1, 0xFF37), + (0xFF58, 0x1, 0xFF38), + (0xFF59, 0x1, 0xFF39), + (0xFF5A, 0x1, 0xFF3A), + (0x10428, 0x1, 0x10400), + (0x10429, 0x1, 0x10401), + (0x1042A, 0x1, 0x10402), + (0x1042B, 0x1, 0x10403), + (0x1042C, 0x1, 0x10404), + (0x1042D, 0x1, 0x10405), + (0x1042E, 0x1, 0x10406), + (0x1042F, 0x1, 0x10407), + (0x10430, 0x1, 0x10408), + (0x10431, 0x1, 0x10409), + (0x10432, 0x1, 0x1040A), + (0x10433, 0x1, 0x1040B), + (0x10434, 0x1, 0x1040C), + (0x10435, 0x1, 0x1040D), + (0x10436, 0x1, 0x1040E), + (0x10437, 0x1, 0x1040F), + (0x10438, 0x1, 0x10410), + (0x10439, 0x1, 0x10411), + (0x1043A, 0x1, 0x10412), + (0x1043B, 0x1, 0x10413), + (0x1043C, 0x1, 0x10414), + (0x1043D, 0x1, 0x10415), + (0x1043E, 0x1, 0x10416), + (0x1043F, 0x1, 0x10417), + (0x10440, 0x1, 0x10418), + (0x10441, 0x1, 0x10419), + (0x10442, 0x1, 0x1041A), + (0x10443, 0x1, 0x1041B), + (0x10444, 0x1, 0x1041C), + (0x10445, 0x1, 0x1041D), + (0x10446, 0x1, 0x1041E), + (0x10447, 0x1, 0x1041F), + (0x10448, 0x1, 0x10420), + (0x10449, 0x1, 0x10421), + (0x1044A, 0x1, 0x10422), + (0x1044B, 0x1, 0x10423), + (0x1044C, 0x1, 0x10424), + (0x1044D, 0x1, 0x10425), + (0x1044E, 0x1, 0x10426), + (0x1044F, 0x1, 0x10427), + (0x104D8, 0x1, 0x104B0), + (0x104D9, 0x1, 0x104B1), + (0x104DA, 0x1, 0x104B2), + (0x104DB, 0x1, 0x104B3), + (0x104DC, 0x1, 0x104B4), + (0x104DD, 0x1, 0x104B5), + (0x104DE, 0x1, 0x104B6), + (0x104DF, 0x1, 0x104B7), + (0x104E0, 0x1, 0x104B8), + (0x104E1, 0x1, 0x104B9), + (0x104E2, 0x1, 0x104BA), + (0x104E3, 0x1, 0x104BB), + (0x104E4, 0x1, 0x104BC), + (0x104E5, 0x1, 0x104BD), + (0x104E6, 0x1, 0x104BE), + (0x104E7, 0x1, 0x104BF), + (0x104E8, 0x1, 0x104C0), + (0x104E9, 0x1, 0x104C1), + (0x104EA, 0x1, 0x104C2), + (0x104EB, 0x1, 0x104C3), + (0x104EC, 0x1, 0x104C4), + (0x104ED, 0x1, 0x104C5), + (0x104EE, 0x1, 0x104C6), + (0x104EF, 0x1, 0x104C7), + (0x104F0, 0x1, 0x104C8), + (0x104F1, 0x1, 0x104C9), + (0x104F2, 0x1, 0x104CA), + (0x104F3, 0x1, 0x104CB), + (0x104F4, 0x1, 0x104CC), + (0x104F5, 0x1, 0x104CD), + (0x104F6, 0x1, 0x104CE), + (0x104F7, 0x1, 0x104CF), + (0x104F8, 0x1, 0x104D0), + (0x104F9, 0x1, 0x104D1), + (0x104FA, 0x1, 0x104D2), + (0x104FB, 0x1, 0x104D3), + (0x10597, 0x1, 0x10570), + (0x10598, 0x1, 0x10571), + (0x10599, 0x1, 0x10572), + (0x1059A, 0x1, 0x10573), + (0x1059B, 0x1, 0x10574), + (0x1059C, 0x1, 0x10575), + (0x1059D, 0x1, 0x10576), + (0x1059E, 0x1, 0x10577), + (0x1059F, 0x1, 0x10578), + (0x105A0, 0x1, 0x10579), + (0x105A1, 0x1, 0x1057A), + (0x105A3, 0x1, 0x1057C), + (0x105A4, 0x1, 0x1057D), + (0x105A5, 0x1, 0x1057E), + (0x105A6, 0x1, 0x1057F), + (0x105A7, 0x1, 0x10580), + (0x105A8, 0x1, 0x10581), + (0x105A9, 0x1, 0x10582), + (0x105AA, 0x1, 0x10583), + (0x105AB, 0x1, 0x10584), + (0x105AC, 0x1, 0x10585), + (0x105AD, 0x1, 0x10586), + (0x105AE, 0x1, 0x10587), + (0x105AF, 0x1, 0x10588), + (0x105B0, 0x1, 0x10589), + (0x105B1, 0x1, 0x1058A), + (0x105B3, 0x1, 0x1058C), + (0x105B4, 0x1, 0x1058D), + (0x105B5, 0x1, 0x1058E), + (0x105B6, 0x1, 0x1058F), + (0x105B7, 0x1, 0x10590), + (0x105B8, 0x1, 0x10591), + (0x105B9, 0x1, 0x10592), + (0x105BB, 0x1, 0x10594), + (0x105BC, 0x1, 0x10595), + (0x10CC0, 0x1, 0x10C80), + (0x10CC1, 0x1, 0x10C81), + (0x10CC2, 0x1, 0x10C82), + (0x10CC3, 0x1, 0x10C83), + (0x10CC4, 0x1, 0x10C84), + (0x10CC5, 0x1, 0x10C85), + (0x10CC6, 0x1, 0x10C86), + (0x10CC7, 0x1, 0x10C87), + (0x10CC8, 0x1, 0x10C88), + (0x10CC9, 0x1, 0x10C89), + (0x10CCA, 0x1, 0x10C8A), + (0x10CCB, 0x1, 0x10C8B), + (0x10CCC, 0x1, 0x10C8C), + (0x10CCD, 0x1, 0x10C8D), + (0x10CCE, 0x1, 0x10C8E), + (0x10CCF, 0x1, 0x10C8F), + (0x10CD0, 0x1, 0x10C90), + (0x10CD1, 0x1, 0x10C91), + (0x10CD2, 0x1, 0x10C92), + (0x10CD3, 0x1, 0x10C93), + (0x10CD4, 0x1, 0x10C94), + (0x10CD5, 0x1, 0x10C95), + (0x10CD6, 0x1, 0x10C96), + (0x10CD7, 0x1, 0x10C97), + (0x10CD8, 0x1, 0x10C98), + (0x10CD9, 0x1, 0x10C99), + (0x10CDA, 0x1, 0x10C9A), + (0x10CDB, 0x1, 0x10C9B), + (0x10CDC, 0x1, 0x10C9C), + (0x10CDD, 0x1, 0x10C9D), + (0x10CDE, 0x1, 0x10C9E), + (0x10CDF, 0x1, 0x10C9F), + (0x10CE0, 0x1, 0x10CA0), + (0x10CE1, 0x1, 0x10CA1), + (0x10CE2, 0x1, 0x10CA2), + (0x10CE3, 0x1, 0x10CA3), + (0x10CE4, 0x1, 0x10CA4), + (0x10CE5, 0x1, 0x10CA5), + (0x10CE6, 0x1, 0x10CA6), + (0x10CE7, 0x1, 0x10CA7), + (0x10CE8, 0x1, 0x10CA8), + (0x10CE9, 0x1, 0x10CA9), + (0x10CEA, 0x1, 0x10CAA), + (0x10CEB, 0x1, 0x10CAB), + (0x10CEC, 0x1, 0x10CAC), + (0x10CED, 0x1, 0x10CAD), + (0x10CEE, 0x1, 0x10CAE), + (0x10CEF, 0x1, 0x10CAF), + (0x10CF0, 0x1, 0x10CB0), + (0x10CF1, 0x1, 0x10CB1), + (0x10CF2, 0x1, 0x10CB2), + (0x118C0, 0x1, 0x118A0), + (0x118C1, 0x1, 0x118A1), + (0x118C2, 0x1, 0x118A2), + (0x118C3, 0x1, 0x118A3), + (0x118C4, 0x1, 0x118A4), + (0x118C5, 0x1, 0x118A5), + (0x118C6, 0x1, 0x118A6), + (0x118C7, 0x1, 0x118A7), + (0x118C8, 0x1, 0x118A8), + (0x118C9, 0x1, 0x118A9), + (0x118CA, 0x1, 0x118AA), + (0x118CB, 0x1, 0x118AB), + (0x118CC, 0x1, 0x118AC), + (0x118CD, 0x1, 0x118AD), + (0x118CE, 0x1, 0x118AE), + (0x118CF, 0x1, 0x118AF), + (0x118D0, 0x1, 0x118B0), + (0x118D1, 0x1, 0x118B1), + (0x118D2, 0x1, 0x118B2), + (0x118D3, 0x1, 0x118B3), + (0x118D4, 0x1, 0x118B4), + (0x118D5, 0x1, 0x118B5), + (0x118D6, 0x1, 0x118B6), + (0x118D7, 0x1, 0x118B7), + (0x118D8, 0x1, 0x118B8), + (0x118D9, 0x1, 0x118B9), + (0x118DA, 0x1, 0x118BA), + (0x118DB, 0x1, 0x118BB), + (0x118DC, 0x1, 0x118BC), + (0x118DD, 0x1, 0x118BD), + (0x118DE, 0x1, 0x118BE), + (0x118DF, 0x1, 0x118BF), + (0x16E60, 0x1, 0x16E40), + (0x16E61, 0x1, 0x16E41), + (0x16E62, 0x1, 0x16E42), + (0x16E63, 0x1, 0x16E43), + (0x16E64, 0x1, 0x16E44), + (0x16E65, 0x1, 0x16E45), + (0x16E66, 0x1, 0x16E46), + (0x16E67, 0x1, 0x16E47), + (0x16E68, 0x1, 0x16E48), + (0x16E69, 0x1, 0x16E49), + (0x16E6A, 0x1, 0x16E4A), + (0x16E6B, 0x1, 0x16E4B), + (0x16E6C, 0x1, 0x16E4C), + (0x16E6D, 0x1, 0x16E4D), + (0x16E6E, 0x1, 0x16E4E), + (0x16E6F, 0x1, 0x16E4F), + (0x16E70, 0x1, 0x16E50), + (0x16E71, 0x1, 0x16E51), + (0x16E72, 0x1, 0x16E52), + (0x16E73, 0x1, 0x16E53), + (0x16E74, 0x1, 0x16E54), + (0x16E75, 0x1, 0x16E55), + (0x16E76, 0x1, 0x16E56), + (0x16E77, 0x1, 0x16E57), + (0x16E78, 0x1, 0x16E58), + (0x16E79, 0x1, 0x16E59), + (0x16E7A, 0x1, 0x16E5A), + (0x16E7B, 0x1, 0x16E5B), + (0x16E7C, 0x1, 0x16E5C), + (0x16E7D, 0x1, 0x16E5D), + (0x16E7E, 0x1, 0x16E5E), + (0x16E7F, 0x1, 0x16E5F), + (0x1E922, 0x1, 0x1E900), + (0x1E923, 0x1, 0x1E901), + (0x1E924, 0x1, 0x1E902), + (0x1E925, 0x1, 0x1E903), + (0x1E926, 0x1, 0x1E904), + (0x1E927, 0x1, 0x1E905), + (0x1E928, 0x1, 0x1E906), + (0x1E929, 0x1, 0x1E907), + (0x1E92A, 0x1, 0x1E908), + (0x1E92B, 0x1, 0x1E909), + (0x1E92C, 0x1, 0x1E90A), + (0x1E92D, 0x1, 0x1E90B), + (0x1E92E, 0x1, 0x1E90C), + (0x1E92F, 0x1, 0x1E90D), + (0x1E930, 0x1, 0x1E90E), + (0x1E931, 0x1, 0x1E90F), + (0x1E932, 0x1, 0x1E910), + (0x1E933, 0x1, 0x1E911), + (0x1E934, 0x1, 0x1E912), + (0x1E935, 0x1, 0x1E913), + (0x1E936, 0x1, 0x1E914), + (0x1E937, 0x1, 0x1E915), + (0x1E938, 0x1, 0x1E916), + (0x1E939, 0x1, 0x1E917), + (0x1E93A, 0x1, 0x1E918), + (0x1E93B, 0x1, 0x1E919), + (0x1E93C, 0x1, 0x1E91A), + (0x1E93D, 0x1, 0x1E91B), + (0x1E93E, 0x1, 0x1E91C), + (0x1E93F, 0x1, 0x1E91D), + (0x1E940, 0x1, 0x1E91E), + (0x1E941, 0x1, 0x1E91F), + (0x1E942, 0x1, 0x1E920), + (0x1E943, 0x1, 0x1E921), +) + +LOWER_FINAL_SIGMA_ROWS = ( + (0x3A3, 0x1, 0x3C2), +) + +CONTEXT_RANGES = ( + (0x27, 0x27, 0x2), + (0x2E, 0x2E, 0x2), + (0x30, 0x39, 0x4), + (0x3A, 0x3A, 0x2), + (0x41, 0x5A, 0x1), + (0x5E, 0x5E, 0x2), + (0x60, 0x60, 0x2), + (0x61, 0x7A, 0x1), + (0xA8, 0xA8, 0x2), + (0xAA, 0xAA, 0x1), + (0xAD, 0xAD, 0x2), + (0xAF, 0xAF, 0x2), + (0xB4, 0xB4, 0x2), + (0xB5, 0xB5, 0x1), + (0xB7, 0xB8, 0x2), + (0xBA, 0xBA, 0x1), + (0xC0, 0xD6, 0x1), + (0xD8, 0xF6, 0x1), + (0xF8, 0x1BA, 0x1), + (0x1BC, 0x1BF, 0x1), + (0x1C4, 0x293, 0x1), + (0x295, 0x2AF, 0x1), + (0x2B0, 0x2B8, 0x3), + (0x2B9, 0x2BF, 0x2), + (0x2C0, 0x2C1, 0x3), + (0x2C2, 0x2DF, 0x2), + (0x2E0, 0x2E4, 0x3), + (0x2E5, 0x344, 0x2), + (0x345, 0x345, 0x3), + (0x346, 0x36F, 0x2), + (0x370, 0x373, 0x1), + (0x374, 0x375, 0x2), + (0x376, 0x377, 0x1), + (0x37A, 0x37A, 0x3), + (0x37B, 0x37D, 0x1), + (0x37F, 0x37F, 0x1), + (0x384, 0x385, 0x2), + (0x386, 0x386, 0x1), + (0x387, 0x387, 0x2), + (0x388, 0x38A, 0x1), + (0x38C, 0x38C, 0x1), + (0x38E, 0x3A1, 0x1), + (0x3A3, 0x3F5, 0x1), + (0x3F7, 0x481, 0x1), + (0x483, 0x489, 0x2), + (0x48A, 0x52F, 0x1), + (0x531, 0x556, 0x1), + (0x559, 0x559, 0x2), + (0x55F, 0x55F, 0x2), + (0x560, 0x588, 0x1), + (0x591, 0x5BD, 0x2), + (0x5BF, 0x5BF, 0x2), + (0x5C1, 0x5C2, 0x2), + (0x5C4, 0x5C5, 0x2), + (0x5C7, 0x5C7, 0x2), + (0x5F4, 0x5F4, 0x2), + (0x600, 0x605, 0x2), + (0x610, 0x61A, 0x2), + (0x61C, 0x61C, 0x2), + (0x640, 0x640, 0x2), + (0x64B, 0x65F, 0x2), + (0x660, 0x669, 0x4), + (0x670, 0x670, 0x2), + (0x6D6, 0x6DD, 0x2), + (0x6DF, 0x6E8, 0x2), + (0x6EA, 0x6ED, 0x2), + (0x6F0, 0x6F9, 0x4), + (0x70F, 0x70F, 0x2), + (0x711, 0x711, 0x2), + (0x730, 0x74A, 0x2), + (0x7A6, 0x7B0, 0x2), + (0x7C0, 0x7C9, 0x4), + (0x7EB, 0x7F5, 0x2), + (0x7FA, 0x7FA, 0x2), + (0x7FD, 0x7FD, 0x2), + (0x816, 0x82D, 0x2), + (0x859, 0x85B, 0x2), + (0x888, 0x888, 0x2), + (0x890, 0x891, 0x2), + (0x898, 0x89F, 0x2), + (0x8C9, 0x902, 0x2), + (0x93A, 0x93A, 0x2), + (0x93C, 0x93C, 0x2), + (0x941, 0x948, 0x2), + (0x94D, 0x94D, 0x2), + (0x951, 0x957, 0x2), + (0x962, 0x963, 0x2), + (0x966, 0x96F, 0x4), + (0x971, 0x971, 0x2), + (0x981, 0x981, 0x2), + (0x9BC, 0x9BC, 0x2), + (0x9C1, 0x9C4, 0x2), + (0x9CD, 0x9CD, 0x2), + (0x9E2, 0x9E3, 0x2), + (0x9E6, 0x9EF, 0x4), + (0x9FE, 0x9FE, 0x2), + (0xA01, 0xA02, 0x2), + (0xA3C, 0xA3C, 0x2), + (0xA41, 0xA42, 0x2), + (0xA47, 0xA48, 0x2), + (0xA4B, 0xA4D, 0x2), + (0xA51, 0xA51, 0x2), + (0xA66, 0xA6F, 0x4), + (0xA70, 0xA71, 0x2), + (0xA75, 0xA75, 0x2), + (0xA81, 0xA82, 0x2), + (0xABC, 0xABC, 0x2), + (0xAC1, 0xAC5, 0x2), + (0xAC7, 0xAC8, 0x2), + (0xACD, 0xACD, 0x2), + (0xAE2, 0xAE3, 0x2), + (0xAE6, 0xAEF, 0x4), + (0xAFA, 0xAFF, 0x2), + (0xB01, 0xB01, 0x2), + (0xB3C, 0xB3C, 0x2), + (0xB3F, 0xB3F, 0x2), + (0xB41, 0xB44, 0x2), + (0xB4D, 0xB4D, 0x2), + (0xB55, 0xB56, 0x2), + (0xB62, 0xB63, 0x2), + (0xB66, 0xB6F, 0x4), + (0xB82, 0xB82, 0x2), + (0xBC0, 0xBC0, 0x2), + (0xBCD, 0xBCD, 0x2), + (0xBE6, 0xBEF, 0x4), + (0xC00, 0xC00, 0x2), + (0xC04, 0xC04, 0x2), + (0xC3C, 0xC3C, 0x2), + (0xC3E, 0xC40, 0x2), + (0xC46, 0xC48, 0x2), + (0xC4A, 0xC4D, 0x2), + (0xC55, 0xC56, 0x2), + (0xC62, 0xC63, 0x2), + (0xC66, 0xC6F, 0x4), + (0xC81, 0xC81, 0x2), + (0xCBC, 0xCBC, 0x2), + (0xCBF, 0xCBF, 0x2), + (0xCC6, 0xCC6, 0x2), + (0xCCC, 0xCCD, 0x2), + (0xCE2, 0xCE3, 0x2), + (0xCE6, 0xCEF, 0x4), + (0xD00, 0xD01, 0x2), + (0xD3B, 0xD3C, 0x2), + (0xD41, 0xD44, 0x2), + (0xD4D, 0xD4D, 0x2), + (0xD62, 0xD63, 0x2), + (0xD66, 0xD6F, 0x4), + (0xD81, 0xD81, 0x2), + (0xDCA, 0xDCA, 0x2), + (0xDD2, 0xDD4, 0x2), + (0xDD6, 0xDD6, 0x2), + (0xDE6, 0xDEF, 0x4), + (0xE31, 0xE31, 0x2), + (0xE34, 0xE3A, 0x2), + (0xE46, 0xE4E, 0x2), + (0xE50, 0xE59, 0x4), + (0xEB1, 0xEB1, 0x2), + (0xEB4, 0xEBC, 0x2), + (0xEC6, 0xEC6, 0x2), + (0xEC8, 0xECD, 0x2), + (0xED0, 0xED9, 0x4), + (0xF18, 0xF19, 0x2), + (0xF20, 0xF29, 0x4), + (0xF35, 0xF35, 0x2), + (0xF37, 0xF37, 0x2), + (0xF39, 0xF39, 0x2), + (0xF71, 0xF7E, 0x2), + (0xF80, 0xF84, 0x2), + (0xF86, 0xF87, 0x2), + (0xF8D, 0xF97, 0x2), + (0xF99, 0xFBC, 0x2), + (0xFC6, 0xFC6, 0x2), + (0x102D, 0x1030, 0x2), + (0x1032, 0x1037, 0x2), + (0x1039, 0x103A, 0x2), + (0x103D, 0x103E, 0x2), + (0x1040, 0x1049, 0x4), + (0x1058, 0x1059, 0x2), + (0x105E, 0x1060, 0x2), + (0x1071, 0x1074, 0x2), + (0x1082, 0x1082, 0x2), + (0x1085, 0x1086, 0x2), + (0x108D, 0x108D, 0x2), + (0x1090, 0x1099, 0x4), + (0x109D, 0x109D, 0x2), + (0x10A0, 0x10C5, 0x1), + (0x10C7, 0x10C7, 0x1), + (0x10CD, 0x10CD, 0x1), + (0x10D0, 0x10FA, 0x1), + (0x10FC, 0x10FC, 0x2), + (0x10FD, 0x10FF, 0x1), + (0x135D, 0x135F, 0x2), + (0x13A0, 0x13F5, 0x1), + (0x13F8, 0x13FD, 0x1), + (0x1712, 0x1714, 0x2), + (0x1732, 0x1733, 0x2), + (0x1752, 0x1753, 0x2), + (0x1772, 0x1773, 0x2), + (0x17B4, 0x17B5, 0x2), + (0x17B7, 0x17BD, 0x2), + (0x17C6, 0x17C6, 0x2), + (0x17C9, 0x17D3, 0x2), + (0x17D7, 0x17D7, 0x2), + (0x17DD, 0x17DD, 0x2), + (0x17E0, 0x17E9, 0x4), + (0x180B, 0x180F, 0x2), + (0x1810, 0x1819, 0x4), + (0x1843, 0x1843, 0x2), + (0x1885, 0x1886, 0x2), + (0x18A9, 0x18A9, 0x2), + (0x1920, 0x1922, 0x2), + (0x1927, 0x1928, 0x2), + (0x1932, 0x1932, 0x2), + (0x1939, 0x193B, 0x2), + (0x1946, 0x194F, 0x4), + (0x19D0, 0x19D9, 0x4), + (0x1A17, 0x1A18, 0x2), + (0x1A1B, 0x1A1B, 0x2), + (0x1A56, 0x1A56, 0x2), + (0x1A58, 0x1A5E, 0x2), + (0x1A60, 0x1A60, 0x2), + (0x1A62, 0x1A62, 0x2), + (0x1A65, 0x1A6C, 0x2), + (0x1A73, 0x1A7C, 0x2), + (0x1A7F, 0x1A7F, 0x2), + (0x1A80, 0x1A89, 0x4), + (0x1A90, 0x1A99, 0x4), + (0x1AA7, 0x1AA7, 0x2), + (0x1AB0, 0x1ACE, 0x2), + (0x1B00, 0x1B03, 0x2), + (0x1B34, 0x1B34, 0x2), + (0x1B36, 0x1B3A, 0x2), + (0x1B3C, 0x1B3C, 0x2), + (0x1B42, 0x1B42, 0x2), + (0x1B50, 0x1B59, 0x4), + (0x1B6B, 0x1B73, 0x2), + (0x1B80, 0x1B81, 0x2), + (0x1BA2, 0x1BA5, 0x2), + (0x1BA8, 0x1BA9, 0x2), + (0x1BAB, 0x1BAD, 0x2), + (0x1BB0, 0x1BB9, 0x4), + (0x1BE6, 0x1BE6, 0x2), + (0x1BE8, 0x1BE9, 0x2), + (0x1BED, 0x1BED, 0x2), + (0x1BEF, 0x1BF1, 0x2), + (0x1C2C, 0x1C33, 0x2), + (0x1C36, 0x1C37, 0x2), + (0x1C40, 0x1C49, 0x4), + (0x1C50, 0x1C59, 0x4), + (0x1C78, 0x1C7D, 0x2), + (0x1C80, 0x1C88, 0x1), + (0x1C90, 0x1CBA, 0x1), + (0x1CBD, 0x1CBF, 0x1), + (0x1CD0, 0x1CD2, 0x2), + (0x1CD4, 0x1CE0, 0x2), + (0x1CE2, 0x1CE8, 0x2), + (0x1CED, 0x1CED, 0x2), + (0x1CF4, 0x1CF4, 0x2), + (0x1CF8, 0x1CF9, 0x2), + (0x1D00, 0x1D2B, 0x1), + (0x1D2C, 0x1D6A, 0x3), + (0x1D6B, 0x1D77, 0x1), + (0x1D78, 0x1D78, 0x3), + (0x1D79, 0x1D9A, 0x1), + (0x1D9B, 0x1DBF, 0x3), + (0x1DC0, 0x1DFF, 0x2), + (0x1E00, 0x1F15, 0x1), + (0x1F18, 0x1F1D, 0x1), + (0x1F20, 0x1F45, 0x1), + (0x1F48, 0x1F4D, 0x1), + (0x1F50, 0x1F57, 0x1), + (0x1F59, 0x1F59, 0x1), + (0x1F5B, 0x1F5B, 0x1), + (0x1F5D, 0x1F5D, 0x1), + (0x1F5F, 0x1F7D, 0x1), + (0x1F80, 0x1FB4, 0x1), + (0x1FB6, 0x1FBC, 0x1), + (0x1FBD, 0x1FBD, 0x2), + (0x1FBE, 0x1FBE, 0x1), + (0x1FBF, 0x1FC1, 0x2), + (0x1FC2, 0x1FC4, 0x1), + (0x1FC6, 0x1FCC, 0x1), + (0x1FCD, 0x1FCF, 0x2), + (0x1FD0, 0x1FD3, 0x1), + (0x1FD6, 0x1FDB, 0x1), + (0x1FDD, 0x1FDF, 0x2), + (0x1FE0, 0x1FEC, 0x1), + (0x1FED, 0x1FEF, 0x2), + (0x1FF2, 0x1FF4, 0x1), + (0x1FF6, 0x1FFC, 0x1), + (0x1FFD, 0x1FFE, 0x2), + (0x200B, 0x200F, 0x2), + (0x2018, 0x2019, 0x2), + (0x2024, 0x2024, 0x2), + (0x2027, 0x2027, 0x2), + (0x202A, 0x202E, 0x2), + (0x2060, 0x2064, 0x2), + (0x2066, 0x206F, 0x2), + (0x2071, 0x2071, 0x3), + (0x207F, 0x207F, 0x3), + (0x2090, 0x209C, 0x3), + (0x20D0, 0x20F0, 0x2), + (0x2102, 0x2102, 0x1), + (0x2107, 0x2107, 0x1), + (0x210A, 0x2113, 0x1), + (0x2115, 0x2115, 0x1), + (0x2119, 0x211D, 0x1), + (0x2124, 0x2124, 0x1), + (0x2126, 0x2126, 0x1), + (0x2128, 0x2128, 0x1), + (0x212A, 0x212D, 0x1), + (0x212F, 0x2134, 0x1), + (0x2139, 0x2139, 0x1), + (0x213C, 0x213F, 0x1), + (0x2145, 0x2149, 0x1), + (0x214E, 0x214E, 0x1), + (0x2160, 0x217F, 0x1), + (0x2183, 0x2184, 0x1), + (0x24B6, 0x24E9, 0x1), + (0x2C00, 0x2C7B, 0x1), + (0x2C7C, 0x2C7D, 0x3), + (0x2C7E, 0x2CE4, 0x1), + (0x2CEB, 0x2CEE, 0x1), + (0x2CEF, 0x2CF1, 0x2), + (0x2CF2, 0x2CF3, 0x1), + (0x2D00, 0x2D25, 0x1), + (0x2D27, 0x2D27, 0x1), + (0x2D2D, 0x2D2D, 0x1), + (0x2D6F, 0x2D6F, 0x2), + (0x2D7F, 0x2D7F, 0x2), + (0x2DE0, 0x2DFF, 0x2), + (0x2E2F, 0x2E2F, 0x2), + (0x3005, 0x3005, 0x2), + (0x302A, 0x302D, 0x2), + (0x3031, 0x3035, 0x2), + (0x303B, 0x303B, 0x2), + (0x3099, 0x309E, 0x2), + (0x30FC, 0x30FE, 0x2), + (0xA015, 0xA015, 0x2), + (0xA4F8, 0xA4FD, 0x2), + (0xA60C, 0xA60C, 0x2), + (0xA620, 0xA629, 0x4), + (0xA640, 0xA66D, 0x1), + (0xA66F, 0xA672, 0x2), + (0xA674, 0xA67D, 0x2), + (0xA67F, 0xA67F, 0x2), + (0xA680, 0xA69B, 0x1), + (0xA69C, 0xA69D, 0x3), + (0xA69E, 0xA69F, 0x2), + (0xA6F0, 0xA6F1, 0x2), + (0xA700, 0xA721, 0x2), + (0xA722, 0xA76F, 0x1), + (0xA770, 0xA770, 0x3), + (0xA771, 0xA787, 0x1), + (0xA788, 0xA78A, 0x2), + (0xA78B, 0xA78E, 0x1), + (0xA790, 0xA7CA, 0x1), + (0xA7D0, 0xA7D1, 0x1), + (0xA7D3, 0xA7D3, 0x1), + (0xA7D5, 0xA7D9, 0x1), + (0xA7F2, 0xA7F4, 0x2), + (0xA7F5, 0xA7F6, 0x1), + (0xA7F8, 0xA7F9, 0x3), + (0xA7FA, 0xA7FA, 0x1), + (0xA802, 0xA802, 0x2), + (0xA806, 0xA806, 0x2), + (0xA80B, 0xA80B, 0x2), + (0xA825, 0xA826, 0x2), + (0xA82C, 0xA82C, 0x2), + (0xA8C4, 0xA8C5, 0x2), + (0xA8D0, 0xA8D9, 0x4), + (0xA8E0, 0xA8F1, 0x2), + (0xA8FF, 0xA8FF, 0x2), + (0xA900, 0xA909, 0x4), + (0xA926, 0xA92D, 0x2), + (0xA947, 0xA951, 0x2), + (0xA980, 0xA982, 0x2), + (0xA9B3, 0xA9B3, 0x2), + (0xA9B6, 0xA9B9, 0x2), + (0xA9BC, 0xA9BD, 0x2), + (0xA9CF, 0xA9CF, 0x2), + (0xA9D0, 0xA9D9, 0x4), + (0xA9E5, 0xA9E6, 0x2), + (0xA9F0, 0xA9F9, 0x4), + (0xAA29, 0xAA2E, 0x2), + (0xAA31, 0xAA32, 0x2), + (0xAA35, 0xAA36, 0x2), + (0xAA43, 0xAA43, 0x2), + (0xAA4C, 0xAA4C, 0x2), + (0xAA50, 0xAA59, 0x4), + (0xAA70, 0xAA70, 0x2), + (0xAA7C, 0xAA7C, 0x2), + (0xAAB0, 0xAAB0, 0x2), + (0xAAB2, 0xAAB4, 0x2), + (0xAAB7, 0xAAB8, 0x2), + (0xAABE, 0xAABF, 0x2), + (0xAAC1, 0xAAC1, 0x2), + (0xAADD, 0xAADD, 0x2), + (0xAAEC, 0xAAED, 0x2), + (0xAAF3, 0xAAF4, 0x2), + (0xAAF6, 0xAAF6, 0x2), + (0xAB30, 0xAB5A, 0x1), + (0xAB5B, 0xAB5B, 0x2), + (0xAB5C, 0xAB5F, 0x3), + (0xAB60, 0xAB68, 0x1), + (0xAB69, 0xAB6B, 0x2), + (0xAB70, 0xABBF, 0x1), + (0xABE5, 0xABE5, 0x2), + (0xABE8, 0xABE8, 0x2), + (0xABED, 0xABED, 0x2), + (0xABF0, 0xABF9, 0x4), + (0xFB00, 0xFB06, 0x1), + (0xFB13, 0xFB17, 0x1), + (0xFB1E, 0xFB1E, 0x2), + (0xFBB2, 0xFBC2, 0x2), + (0xFE00, 0xFE0F, 0x2), + (0xFE13, 0xFE13, 0x2), + (0xFE20, 0xFE2F, 0x2), + (0xFE52, 0xFE52, 0x2), + (0xFE55, 0xFE55, 0x2), + (0xFEFF, 0xFEFF, 0x2), + (0xFF07, 0xFF07, 0x2), + (0xFF0E, 0xFF0E, 0x2), + (0xFF10, 0xFF19, 0x4), + (0xFF1A, 0xFF1A, 0x2), + (0xFF21, 0xFF3A, 0x1), + (0xFF3E, 0xFF3E, 0x2), + (0xFF40, 0xFF40, 0x2), + (0xFF41, 0xFF5A, 0x1), + (0xFF70, 0xFF70, 0x2), + (0xFF9E, 0xFF9F, 0x2), + (0xFFE3, 0xFFE3, 0x2), + (0xFFF9, 0xFFFB, 0x2), + (0x101FD, 0x101FD, 0x2), + (0x102E0, 0x102E0, 0x2), + (0x10376, 0x1037A, 0x2), + (0x10400, 0x1044F, 0x1), + (0x104A0, 0x104A9, 0x4), + (0x104B0, 0x104D3, 0x1), + (0x104D8, 0x104FB, 0x1), + (0x10570, 0x1057A, 0x1), + (0x1057C, 0x1058A, 0x1), + (0x1058C, 0x10592, 0x1), + (0x10594, 0x10595, 0x1), + (0x10597, 0x105A1, 0x1), + (0x105A3, 0x105B1, 0x1), + (0x105B3, 0x105B9, 0x1), + (0x105BB, 0x105BC, 0x1), + (0x10780, 0x10780, 0x3), + (0x10781, 0x10782, 0x2), + (0x10783, 0x10785, 0x3), + (0x10787, 0x107B0, 0x3), + (0x107B2, 0x107BA, 0x3), + (0x10A01, 0x10A03, 0x2), + (0x10A05, 0x10A06, 0x2), + (0x10A0C, 0x10A0F, 0x2), + (0x10A38, 0x10A3A, 0x2), + (0x10A3F, 0x10A3F, 0x2), + (0x10AE5, 0x10AE6, 0x2), + (0x10C80, 0x10CB2, 0x1), + (0x10CC0, 0x10CF2, 0x1), + (0x10D24, 0x10D27, 0x2), + (0x10D30, 0x10D39, 0x4), + (0x10EAB, 0x10EAC, 0x2), + (0x10F46, 0x10F50, 0x2), + (0x10F82, 0x10F85, 0x2), + (0x11001, 0x11001, 0x2), + (0x11038, 0x11046, 0x2), + (0x11066, 0x1106F, 0x4), + (0x11070, 0x11070, 0x2), + (0x11073, 0x11074, 0x2), + (0x1107F, 0x11081, 0x2), + (0x110B3, 0x110B6, 0x2), + (0x110B9, 0x110BA, 0x2), + (0x110BD, 0x110BD, 0x2), + (0x110C2, 0x110C2, 0x2), + (0x110CD, 0x110CD, 0x2), + (0x110F0, 0x110F9, 0x4), + (0x11100, 0x11102, 0x2), + (0x11127, 0x1112B, 0x2), + (0x1112D, 0x11134, 0x2), + (0x11136, 0x1113F, 0x4), + (0x11173, 0x11173, 0x2), + (0x11180, 0x11181, 0x2), + (0x111B6, 0x111BE, 0x2), + (0x111C9, 0x111CC, 0x2), + (0x111CF, 0x111CF, 0x2), + (0x111D0, 0x111D9, 0x4), + (0x1122F, 0x11231, 0x2), + (0x11234, 0x11234, 0x2), + (0x11236, 0x11237, 0x2), + (0x1123E, 0x1123E, 0x2), + (0x112DF, 0x112DF, 0x2), + (0x112E3, 0x112EA, 0x2), + (0x112F0, 0x112F9, 0x4), + (0x11300, 0x11301, 0x2), + (0x1133B, 0x1133C, 0x2), + (0x11340, 0x11340, 0x2), + (0x11366, 0x1136C, 0x2), + (0x11370, 0x11374, 0x2), + (0x11438, 0x1143F, 0x2), + (0x11442, 0x11444, 0x2), + (0x11446, 0x11446, 0x2), + (0x11450, 0x11459, 0x4), + (0x1145E, 0x1145E, 0x2), + (0x114B3, 0x114B8, 0x2), + (0x114BA, 0x114BA, 0x2), + (0x114BF, 0x114C0, 0x2), + (0x114C2, 0x114C3, 0x2), + (0x114D0, 0x114D9, 0x4), + (0x115B2, 0x115B5, 0x2), + (0x115BC, 0x115BD, 0x2), + (0x115BF, 0x115C0, 0x2), + (0x115DC, 0x115DD, 0x2), + (0x11633, 0x1163A, 0x2), + (0x1163D, 0x1163D, 0x2), + (0x1163F, 0x11640, 0x2), + (0x11650, 0x11659, 0x4), + (0x116AB, 0x116AB, 0x2), + (0x116AD, 0x116AD, 0x2), + (0x116B0, 0x116B5, 0x2), + (0x116B7, 0x116B7, 0x2), + (0x116C0, 0x116C9, 0x4), + (0x1171D, 0x1171F, 0x2), + (0x11722, 0x11725, 0x2), + (0x11727, 0x1172B, 0x2), + (0x11730, 0x11739, 0x4), + (0x1182F, 0x11837, 0x2), + (0x11839, 0x1183A, 0x2), + (0x118A0, 0x118DF, 0x1), + (0x118E0, 0x118E9, 0x4), + (0x1193B, 0x1193C, 0x2), + (0x1193E, 0x1193E, 0x2), + (0x11943, 0x11943, 0x2), + (0x11950, 0x11959, 0x4), + (0x119D4, 0x119D7, 0x2), + (0x119DA, 0x119DB, 0x2), + (0x119E0, 0x119E0, 0x2), + (0x11A01, 0x11A0A, 0x2), + (0x11A33, 0x11A38, 0x2), + (0x11A3B, 0x11A3E, 0x2), + (0x11A47, 0x11A47, 0x2), + (0x11A51, 0x11A56, 0x2), + (0x11A59, 0x11A5B, 0x2), + (0x11A8A, 0x11A96, 0x2), + (0x11A98, 0x11A99, 0x2), + (0x11C30, 0x11C36, 0x2), + (0x11C38, 0x11C3D, 0x2), + (0x11C3F, 0x11C3F, 0x2), + (0x11C50, 0x11C59, 0x4), + (0x11C92, 0x11CA7, 0x2), + (0x11CAA, 0x11CB0, 0x2), + (0x11CB2, 0x11CB3, 0x2), + (0x11CB5, 0x11CB6, 0x2), + (0x11D31, 0x11D36, 0x2), + (0x11D3A, 0x11D3A, 0x2), + (0x11D3C, 0x11D3D, 0x2), + (0x11D3F, 0x11D45, 0x2), + (0x11D47, 0x11D47, 0x2), + (0x11D50, 0x11D59, 0x4), + (0x11D90, 0x11D91, 0x2), + (0x11D95, 0x11D95, 0x2), + (0x11D97, 0x11D97, 0x2), + (0x11DA0, 0x11DA9, 0x4), + (0x11EF3, 0x11EF4, 0x2), + (0x13430, 0x13438, 0x2), + (0x16A60, 0x16A69, 0x4), + (0x16AC0, 0x16AC9, 0x4), + (0x16AF0, 0x16AF4, 0x2), + (0x16B30, 0x16B36, 0x2), + (0x16B40, 0x16B43, 0x2), + (0x16B50, 0x16B59, 0x4), + (0x16E40, 0x16E7F, 0x1), + (0x16F4F, 0x16F4F, 0x2), + (0x16F8F, 0x16F9F, 0x2), + (0x16FE0, 0x16FE1, 0x2), + (0x16FE3, 0x16FE4, 0x2), + (0x1AFF0, 0x1AFF3, 0x2), + (0x1AFF5, 0x1AFFB, 0x2), + (0x1AFFD, 0x1AFFE, 0x2), + (0x1BC9D, 0x1BC9E, 0x2), + (0x1BCA0, 0x1BCA3, 0x2), + (0x1CF00, 0x1CF2D, 0x2), + (0x1CF30, 0x1CF46, 0x2), + (0x1D167, 0x1D169, 0x2), + (0x1D173, 0x1D182, 0x2), + (0x1D185, 0x1D18B, 0x2), + (0x1D1AA, 0x1D1AD, 0x2), + (0x1D242, 0x1D244, 0x2), + (0x1D400, 0x1D454, 0x1), + (0x1D456, 0x1D49C, 0x1), + (0x1D49E, 0x1D49F, 0x1), + (0x1D4A2, 0x1D4A2, 0x1), + (0x1D4A5, 0x1D4A6, 0x1), + (0x1D4A9, 0x1D4AC, 0x1), + (0x1D4AE, 0x1D4B9, 0x1), + (0x1D4BB, 0x1D4BB, 0x1), + (0x1D4BD, 0x1D4C3, 0x1), + (0x1D4C5, 0x1D505, 0x1), + (0x1D507, 0x1D50A, 0x1), + (0x1D50D, 0x1D514, 0x1), + (0x1D516, 0x1D51C, 0x1), + (0x1D51E, 0x1D539, 0x1), + (0x1D53B, 0x1D53E, 0x1), + (0x1D540, 0x1D544, 0x1), + (0x1D546, 0x1D546, 0x1), + (0x1D54A, 0x1D550, 0x1), + (0x1D552, 0x1D6A5, 0x1), + (0x1D6A8, 0x1D6C0, 0x1), + (0x1D6C2, 0x1D6DA, 0x1), + (0x1D6DC, 0x1D6FA, 0x1), + (0x1D6FC, 0x1D714, 0x1), + (0x1D716, 0x1D734, 0x1), + (0x1D736, 0x1D74E, 0x1), + (0x1D750, 0x1D76E, 0x1), + (0x1D770, 0x1D788, 0x1), + (0x1D78A, 0x1D7A8, 0x1), + (0x1D7AA, 0x1D7C2, 0x1), + (0x1D7C4, 0x1D7CB, 0x1), + (0x1D7CE, 0x1D7FF, 0x4), + (0x1DA00, 0x1DA36, 0x2), + (0x1DA3B, 0x1DA6C, 0x2), + (0x1DA75, 0x1DA75, 0x2), + (0x1DA84, 0x1DA84, 0x2), + (0x1DA9B, 0x1DA9F, 0x2), + (0x1DAA1, 0x1DAAF, 0x2), + (0x1DF00, 0x1DF09, 0x1), + (0x1DF0B, 0x1DF1E, 0x1), + (0x1E000, 0x1E006, 0x2), + (0x1E008, 0x1E018, 0x2), + (0x1E01B, 0x1E021, 0x2), + (0x1E023, 0x1E024, 0x2), + (0x1E026, 0x1E02A, 0x2), + (0x1E130, 0x1E13D, 0x2), + (0x1E140, 0x1E149, 0x4), + (0x1E2AE, 0x1E2AE, 0x2), + (0x1E2EC, 0x1E2EF, 0x2), + (0x1E2F0, 0x1E2F9, 0x4), + (0x1E8D0, 0x1E8D6, 0x2), + (0x1E900, 0x1E943, 0x1), + (0x1E944, 0x1E94B, 0x2), + (0x1E950, 0x1E959, 0x4), + (0x1F130, 0x1F149, 0x1), + (0x1F150, 0x1F169, 0x1), + (0x1F170, 0x1F189, 0x1), + (0x1F3FB, 0x1F3FF, 0x2), + (0x1FBF0, 0x1FBF9, 0x4), + (0xE0001, 0xE0001, 0x2), + (0xE0020, 0xE007F, 0x2), + (0xE0100, 0xE01EF, 0x2), +) diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index 25480bfb..b8d948f6 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -5272,6 +5272,7 @@ def _prepare_closed_trade_folds(self) -> None: def _generate_cpp(self) -> str: self._uses_enum_string_lookup = False self._uses_tv_string_units = False + self._uses_unicode_regex = False # Titles already carry assigned declaration keys. Refuse an unproved # title first, then flag any remaining shared fallback/suffix keys. self._check_input_titles() @@ -6598,7 +6599,7 @@ def _resolve_known(self, arg_str: str) -> str: if isinstance(val, (int, float)): return str(val) if isinstance(val, str): - return f'std::string("{self._cpp_string_escape(val)}")' + return self._cpp_std_string(val) # Also resolve bar field references if arg_str in BAR_FIELDS: return BAR_FIELDS[arg_str] @@ -6968,7 +6969,7 @@ def _sub(p: re.Match) -> str: return "true" if val else "false" if isinstance(val, (int, float)): return str(val) - return f'std::string("{self._cpp_string_escape(val)}")' + return self._cpp_std_string(val) return name # Identifier tokens only: a string literal's text is not identifiers diff --git a/pineforge_codegen/codegen/checked_settings.py b/pineforge_codegen/codegen/checked_settings.py index ce2b9bb9..79239101 100644 --- a/pineforge_codegen/codegen/checked_settings.py +++ b/pineforge_codegen/codegen/checked_settings.py @@ -9,7 +9,10 @@ _QUOTED_STRING = r'"(?:[^"\\\x00\r\n]|\\(?:["\\nrt]|000))*"' -_STRING_LITERAL = re.compile(rf'(?:std::string\({_QUOTED_STRING}\)|{_QUOTED_STRING})') +_STRING_LITERAL = re.compile( + rf'(?:std::string\((?P{_QUOTED_STRING})(?:, (?P0|[1-9][0-9]*))?\)' + rf'|(?P{_QUOTED_STRING}))' +) _STRING_ESCAPE = re.compile(r'\\(?:["\\nrt]|000)') _STRING_ESCAPE_VALUES = { r'\"': '"', r'\\': '\\', r'\n': '\n', r'\r': '\r', r'\t': '\t', r'\000': '\0', @@ -37,7 +40,11 @@ def _string_setting_arg(expr, lowered: str) -> str | None: ) if not known: return None - return lowered if _STRING_LITERAL.fullmatch(lowered) else None + try: + _string_setting_value(lowered) + except ValueError: + return None + return lowered def _visit_setting_arg(emitter, expr) -> str: @@ -53,16 +60,22 @@ def _visit_setting_arg(emitter, expr) -> str: def _string_setting_value(lowered: str) -> str: - """Decode the emitter's literals as native std::string(const char*) values. + """Decode counted and uncounted literals with their native string semantics. Only the serializer's escape grammar is accepted; Unicode stays literal. - The receipt's string constructors stop at the first NUL, including one - followed by digits, while an escaped backslash remains ordinary data. + A counted literal retains NUL and must carry its exact UTF-8 byte count. + Uncounted C-string conversions still stop at the first NUL. An escaped + backslash remains ordinary data in either form. """ - if not _STRING_LITERAL.fullmatch(lowered): + match = _STRING_LITERAL.fullmatch(lowered) + if match is None: raise ValueError("setting value is not an emitted C++ string literal") - literal = lowered[len("std::string("):-1] if lowered.startswith("std::string(") else lowered + literal = match["wrapped"] or match["bare"] decoded = _STRING_ESCAPE.sub(lambda match: _STRING_ESCAPE_VALUES[match.group()], literal[1:-1]) + if match["count"] is not None: + if int(match["count"]) != len(decoded.encode("utf-8")): + raise ValueError("setting value is not an emitted C++ string literal") + return decoded return decoded.partition("\0")[0] @@ -240,10 +253,7 @@ def emit_settings_members(emitter, lines: list[str], constructor: list[str]) -> option = emitter._string_input_argument(option, refuse_unproven=False) option_cpp = _visit_setting_arg(emitter, option) if getter == "get_input_string": - # An input's options stay C strings: they stop at the - # first NUL, as the default does. - option_cpp = _string_setting_arg( - option, emitter._unsized_std_string(option_cpp)) + option_cpp = _string_setting_arg(option, option_cpp) if option_cpp is None: supported = "false" else: diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index 53ea10ee..cc5548aa 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -111,10 +111,14 @@ from .drawing import DRAWING_LIFETIME_CPP, POLYLINE_HANDLE_CPP from .tv_number_format import TV_NUMBER_FORMAT_CPP from .tv_string_units import TV_STRING_UNITS_CPP, TV_STRING_UNITS_MARKER +from .unicode_regex import UNICODE_REGEX_CPP # Keep the late-selected include beside the headers the C++ allowlist reads. ENUM_STRING_INCLUDE = '#include ' +UNICODE_REGEX_INCLUDES = ( + "#include ", "#include ", "#include ", +) _TRADE_HISTORY_POLICY_MARKER = " /* __PINEFORGE_TRADE_HISTORY_POLICY__ */" @@ -135,6 +139,9 @@ def _emit_includes(self, lines: list[str]) -> None: self._uses_tv_number_format = any( (namespace == "str" and func_name in {"format", "tostring"}) or (namespace is None and func_name == "tostring") + # Namespace and method joins can render numeric elements without + # an explicit str.tostring call, including in callable bodies. + or (isinstance(node.callee, MemberAccess) and node.callee.member == "join") or (namespace == "log" and func_name in {"info", "warning", "error"} and len(node.args) > 1) for root in roots @@ -1568,7 +1575,7 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " // invalid (na under ignore_invalid_symbol, a stopped run without it).", " bool _pf_symbol_data_installed(const std::string& symbol,", " const std::string& timeframe,", - " const char* column = nullptr) const {", + " const std::string* column = nullptr) const {", " const auto facts = symbol_facts_.find(symbol);", " if (facts != symbol_facts_.end() && facts->second.valid && !*facts->second.valid)", " return true;", @@ -1579,11 +1586,16 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " if (feed.instrument != symbol || feed.tf != tf) continue;", " if (column == nullptr) return true;", " for (const auto& named : feed.columns) {", - " if (named.name == column) return true;", + " if (named.name == *column) return true;", " }", " }", " return false;", " }", + " bool _pf_symbol_data_installed(const std::string& symbol,", + " const std::string& timeframe,", + " const std::string& column) const {", + " return _pf_symbol_data_installed(symbol, timeframe, &column);", + " }", " // Its payload seeds ta.ema as TradingView does in the requested context:", " // na until `length` values, then their mean (EmaSeeding::SimpleAverage).", " struct _PFForeignEmaSeeding {", @@ -1594,14 +1606,15 @@ def _emit_foreign_security_lookups(self, lines: list[str]) -> None: " _PFForeignEmaSeeding& operator=(const _PFForeignEmaSeeding&) = delete;", " };", ]) - if any(self._security_footprint_column(info["sec_id"]) for info in self._security_eval_info): + if any(self._security_footprint_column(info["sec_id"]) is not None + for info in self._security_eval_info): lines.extend([ " // A footprint's delta: the named column of the requested bar.", - " double _pf_symbol_column(int sec_id, const char* name) const {", + " double _pf_symbol_column(int sec_id, const std::string& name) const {", " return security_column_value(sec_id, name);", " }", "#else", - " double _pf_symbol_column(int, const char*) const { return na(); }", + " double _pf_symbol_column(int, const std::string&) const { return na(); }", ]) lines.append("#endif") @@ -1615,7 +1628,7 @@ def _emit_foreign_security_registration( sec_id = info["sec_id"] symbol = self._security_tf_runtime_expr(info["symbol_node"]) column = self._security_footprint_column(sec_id) - column = f', "{self._cpp_string_escape(column)}"' if column else "" + column = f', {self._cpp_string_literal(column)}' if column is not None else "" ignore_node = info.get("ignore_invalid_node") if ignore_node is None: ignore = "false" @@ -2185,6 +2198,10 @@ def _settle_tv_string_units(self, lines: list[str]) -> None: ) from exc if self._uses_tv_string_units: lines[marker] = TV_STRING_UNITS_CPP + if self._uses_unicode_regex: + lines[marker] += "\n".join( + header for header in UNICODE_REGEX_INCLUDES if header not in lines + ) + "\n" + UNICODE_REGEX_CPP else: del lines[marker:marker + 2] diff --git a/pineforge_codegen/codegen/helpers.py b/pineforge_codegen/codegen/helpers.py index 15996b45..4c928cd1 100644 --- a/pineforge_codegen/codegen/helpers.py +++ b/pineforge_codegen/codegen/helpers.py @@ -461,6 +461,9 @@ def cpp_code_only(cpp: str) -> str: _pf_tv_trim _pf_tv_increment_digits _pf_tv_decimal _pf_tv_pattern _pf_tv_number_style _pf_u16_length _pf_u16_pos _pf_u16_decode _pf_u16_encode _pf_u16_substr + _pf_u16_find _pf_u16_contains _pf_u16_startswith _pf_u16_endswith _pf_u16_join + _pf_u16_normalize _pf_u16_concat + _pf_regex _security_helper_series_ _security_helper_series_str_ _use_precalc _var_initialized _ta_initialized_ _inputs_initialized_ _pf_lines_ _pf_boxes_ _pf_labels_ _pf_linefills_ @@ -543,6 +546,19 @@ def cpp_code_only(cpp: str) -> str: __pf_idx __pf_occ _pf_number_text _pf_substring_text _pf_substring_begin _pf_substring_end _pf_substring_error + _pf_substring_start _pf_substring_count + _pf_match_source _pf_match_pattern + _pf_search_source _pf_search_needle _pf_join_parts _pf_join_separator + _pf_StringArray _pf_text _pf_needle _pf_text_units _pf_needle_units _pf_at + _pf_parts _pf_separator _pf_separator_units _pf_join_units _pf_part_units _pf_i + _pf_case_text _pf_case_properties _pf_case_flags _pf_case_code + _pf_case_lo _pf_case_hi _pf_case_mid _pf_case_after _pf_case_codes + _pf_case_next_cased _pf_case_i _pf_case_props _pf_case_before _pf_case_lead + _pf_case_width _pf_case_valid _pf_case_k _pf_case_byte _pf_case_min + _pf_case_out _pf_case_append _pf_toupper_map _pf_tolower_map + _pf_tolower_final_map _pf_case_contextual _pf_case_row + _pf_trim_text _pf_trim_size _pf_trim_first _pf_trim_last + _pf_trim_i _pf_trim_c _pf_trim_width _pf_trim_second _pf_trim_third _pf_array_new_size _pf_array_new_value _pf_array_new_size_type _pf_array_new_error _pf_read _pf_v5_l _pf_v5_r _pf_wide_l _pf_wide_r _pf_w _pf_symbol _pf_colon _pf_color _pf_bar_back _pf_ema_seeding @@ -650,20 +666,18 @@ def _cpp_std_string(s: str) -> str: return f'std::string("{escaped}", {len(s.encode("utf-8"))})' return f'std::string("{escaped}")' - # ``_cpp_std_string``'s spelling of a literal that holds a NUL; group 1 is - # the quoted literal. - _SIZED_STD_STRING = re.compile(r'std::string\(("(?:[^"\\\r\n]|\\.)*"), \d+\)') - @staticmethod - def _unsized_std_string(cpp: str) -> str: - """``cpp`` with the byte count of a lowered NUL literal left out. - - The default and the options of an input stay C strings: the settings - receipt and the getter's default stop at the first NUL, as they are - documented to, and the manifest must equal both. A text that is not a - counted literal is returned as it is.""" - sized = NamingHelper._SIZED_STD_STRING.fullmatch(cpp.strip()) - return f"std::string({sized.group(1)})" if sized else cpp + def _cpp_string_literal(s: str) -> str: + """A literal for a length-aware C++ string consumer. + + Keep the bare literal spelling when C-string conversion is lossless; + otherwise carry the UTF-8 byte count through a std::string. Consumers + must accept that string directly, without a const-char intermediate. + This does not change a public NUL-terminated C interface. + """ + if "\0" in s: + return NamingHelper._cpp_std_string(s) + return f'"{NamingHelper._cpp_string_escape(s)}"' @staticmethod def _cpp_comment_escape(s: str) -> str: diff --git a/pineforge_codegen/codegen/input.py b/pineforge_codegen/codegen/input.py index e9b84508..1ab60a09 100644 --- a/pineforge_codegen/codegen/input.py +++ b/pineforge_codegen/codegen/input.py @@ -11,7 +11,7 @@ - ``self._resolve_callee`` (``NamingHelper``). - ``self._visit_expr`` (visitor mixin, currently on ``base.py``; used to render the title-arg fallback expression). -- ``self._cpp_string_escape`` (``NamingHelper``). +- ``self._cpp_string_literal`` (``NamingHelper``). """ from __future__ import annotations @@ -405,10 +405,6 @@ def _input_default_expression(self, node: FuncCall) -> tuple[str, int | None]: if proof is not None: return f"0x{proof.value:08X}LL", proof.value cpp = self._visit_expr(default) if default is not None else "0" - if getter == "get_input_string": - # An input's default stays a C string, cut at its first NUL as the - # settings receipt documents; the manifest equals both. - cpp = self._unsized_std_string(cpp) color = (self._literal_color_value(default) if namespace == "input" and func_name == "color" else None) return cpp, color @@ -482,13 +478,14 @@ def _generic_input_cpp_type(self, node: FuncCall) -> str: return self._GETTER_CPP_TYPE[self._input_getter_for_call(node, "input", None)] def _input_key_literal(self, title: str) -> str: - """An assigned key (or option value) as a C++ string literal. + """An assigned key (or option value) as a C++ string expression. Key assignment happens once before lowering, never during escaping. A key is arbitrary Pine string text; pasted verbatim, a quote in it ends the literal early and a backslash escapes the next character, so - the TU no longer compiles.""" - return f'"{self._cpp_string_escape(title)}"' + the TU no longer compiles. A NUL-bearing key also needs its byte count + when converted to the getter's or descriptor's std::string.""" + return self._cpp_string_literal(title) def _coerce_string_input_default(self, getter: str, default_cpp: str) -> str: """String getters take a ``std::string`` default. An unresolved default diff --git a/pineforge_codegen/codegen/security.py b/pineforge_codegen/codegen/security.py index 056756ab..ae524a4e 100644 --- a/pineforge_codegen/codegen/security.py +++ b/pineforge_codegen/codegen/security.py @@ -209,6 +209,9 @@ def __init__(self, values: dict, caller_stack: tuple[dict, ...], self.caller_stack = caller_stack self.call = call self.float_parameters: frozenset[str] = frozenset() + # Declared parameter types survive expansion to caller-authored ASTs, + # including bare na, whose own expression type is numeric. + self.parameter_types: dict[str, str] = {} # Runtime metadata only: the dict's values remain the authored ASTs, # which key TA discovery and binding signatures. self.samples: dict[str, _SecurityArgumentSample] = {} @@ -917,7 +920,7 @@ def result(sec_id): for sec_id in sites: info = self._security_eval_info[sec_id] tf = info["tf"] - arm = (f'"{self._cpp_string_escape(tf)}"' if tf is not None + arm = (self._security_tf_literal(tf) if tf is not None else info["tf_expr"]) # Missing string values denote the chart timeframe in a selection. condition = f"{target} == {arm}" @@ -925,10 +928,10 @@ def result(sec_id): condition = f"({target}.empty() || {condition})" invalid = info.get("series_tf_invalid") if invalid is not None: - message = self._cpp_string_escape(invalid) + message = self._cpp_std_string(invalid) lines.extend(( f" if ({condition}) {{", - f' pine_runtime_error(std::string("{message}"));', + f' pine_runtime_error({message});', # Retain the expression's scalar/tuple/array return type # even when every inventoried value is invalid. f" return {result(sec_id)};", @@ -1107,9 +1110,8 @@ def check(value) -> None: else substituted) def _security_tf_literal(self, tf: str) -> str: - """A timeframe string as the C++ string literal registration and the - evaluator's ``timeframe.*`` reads spell it.""" - return f'"{self._cpp_string_escape(tf)}"' + """A complete timeframe string for registration and evaluator reads.""" + return self._cpp_string_literal(tf) def _security_lookahead_value(self, node): """A computed lookahead's value before on_bar initializes its aliases.""" @@ -2682,6 +2684,14 @@ def _security_float_parameter(name, bindings) -> bool: return name in getattr(frame, "float_parameters", ()) return False + @staticmethod + def _security_parameter_cpp_type(name, bindings) -> str | None: + for frame in reversed(bindings or ()): + types = getattr(frame, "parameter_types", {}) + if name in frame or name in types: + return types.get(name) + return None + def _security_index_reads_helper_local( self, index, @@ -4958,6 +4968,13 @@ def _security_local_value( ) return expr_cpp + def _security_type_for_decl(self, node, bindings=(), body=()) -> str: + """Type an evaluator local in its requested lexical bindings.""" + if (not node.type_hint + and self._security_value_is_string(node.value, bindings, body)): + return "std::string" + return self._type_for_decl(node) + def _security_float_cpp(self, rendered, node, bindings=()) -> str: """Adapt a builder-owned value without re-evaluating it on the chart.""" if not self._security_integer_na_operand(node, bindings): @@ -4989,6 +5006,11 @@ def _security_integer_na_operand(self, node, bindings=(), depth=0) -> bool: value, stack = context if isinstance(value, ASTNode): return self._security_integer_na_operand(value, stack, depth + 1) + series_name = self._security_series_binding_target(value) + if series_name in self._security_string_series: + # The requested slot's family owns the read even when + # the chart has no active local type for this name. + return False if value in getattr(self, "_security_counted_loop_bindings", ()): # The binding identifies the counter even if an inner # declaration later shadows its authored spelling. @@ -5088,7 +5110,7 @@ def unproved(name: str) -> str: continue if not isinstance(stmt, VarDecl): continue - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, stack + (frame,)) persistent = stmt.is_var or stmt.is_varip if persistent or stmt.name in local_series: numeric = (cpp_type != "std::string" @@ -5192,11 +5214,10 @@ def _security_needs_string_series(self) -> bool: seen: set[str] = set() global_expr_map = getattr(self.ctx, "global_expr_map", {}) or {} - def is_string_decl(stmt: VarDecl) -> bool: - # The emitter's own rule (``_type_for_decl``), read in the same - # class-scope context it runs in. + def is_string_decl(stmt: VarDecl, bindings=(), body=()) -> bool: + # Use the emitter's lexical rule before declaring its series map. try: - return self._type_for_decl(stmt) == "std::string" + return self._security_type_for_decl(stmt, bindings, body) == "std::string" except Exception: return stmt.type_hint == "string" or isinstance(stmt.value, StringLiteral) @@ -5247,23 +5268,30 @@ def scan_expr(node) -> bool: if info is None or info.node is None: continue series = set(self.ctx.func_series_vars.get(info.name, set())) - if scan_body(info.node.body, series): + declarations = _SecurityHelperArgumentFrame({}, ()) + declarations.parameter_types = self._security_declared_parameter_types(info) + if scan_body(info.node.body, series, (declarations,)): return True return False - def scan_body(body, series: set[str]) -> bool: + def scan_body(body, series: set[str], bindings=(), declarations=()) -> bool: + preceding = list(declarations) for stmt in body or []: + if self._budget is not None: + self._budget.check(stmt.loc, Phase.CODEGEN) if isinstance(stmt, VarDecl): persistent = stmt.is_var or ( stmt.is_varip and id(stmt) in self._security_series_tf_local_decls) - if (persistent or stmt.name in series) and is_string_decl(stmt): + if ((persistent or stmt.name in series) + and is_string_decl(stmt, bindings, preceding)): return True if scan_expr(stmt.value): return True + preceding.append(stmt) elif isinstance(stmt, IfStmt): if (scan_expr(stmt.condition) - or scan_body(stmt.body, series) - or scan_body(stmt.else_body, series)): + or scan_body(stmt.body, series, bindings, preceding) + or scan_body(stmt.else_body, series, bindings, preceding)): return True elif scan_expr(stmt): return True @@ -5415,7 +5443,7 @@ def activate_decl() -> None: self._security_persistent_helper_series.add(series_name) cpp_type = None if is_persistent_var or self._security_string_series_declared: - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, runtime_stack_local) is_color_state = ( is_persistent_var and self._security_helper_var_is_color(stmt) ) @@ -5483,7 +5511,7 @@ def activate_decl() -> None: plan["func_info"].name, stmt.name, ) - cpp_type = self._type_for_decl(stmt) + cpp_type = self._security_type_for_decl(stmt, runtime_stack_local) expr_cpp = self._build_security_expr( sec_id, stmt.value, @@ -5544,9 +5572,9 @@ def activate_decl() -> None: ) series_name = self._security_series_binding_target(binding) if series_name is not None: - if series_name in self._security_string_series and isinstance( - stmt.value, NaLiteral): - expr_cpp = "na()" + if series_name in self._security_string_series: + expr_cpp = self._security_local_value( + expr_cpp, stmt.value, "std::string", runtime_stack_local) elif series_name in self._security_color_series: expr_cpp = self._security_color_series_store(expr_cpp) else: @@ -5569,6 +5597,9 @@ def activate_decl() -> None: current = _series_expr(series_name, "0") rhs = self._compound_assign_rhs( current, stmt.op, expr_cpp, stmt.target, stmt.value, + target_cpp_type=( + "std::string" + if series_name in self._security_string_series else None), target_is_double=( series_name not in self._security_color_series and series_name not in self._security_string_series), @@ -5593,7 +5624,8 @@ def activate_decl() -> None: rhs = None else: rhs = self._compound_assign_rhs( - binding, stmt.op, expr_cpp, stmt.target, stmt.value) + binding, stmt.op, expr_cpp, stmt.target, stmt.value, + target_cpp_type=self._security_local_cpp_types.get(binding)) if rhs is None: lines.append(f"{pad}{binding} {stmt.op} {expr_cpp};") else: @@ -6344,6 +6376,14 @@ def _security_warn_chart_call(self, node, reason: str | None = None) -> None: "TradingView evaluates it on the requested bar.", ) + def _security_declared_parameter_types(self, info) -> dict[str, str]: + specs = self.ctx.func_declared_param_type_specs.get(info.name, ()) + return { + name: self._type_spec_to_cpp(spec) + for name, spec in zip(info.node.params, specs) + if spec is not None + } + def _security_bind_helper_args( self, node: FuncCall, @@ -6382,6 +6422,7 @@ def _security_bind_helper_args( fi, index, self._callable_target_callsite_idx(fi, node)) == "double") for frame in frames: frame.float_parameters = float_parameters + frame.parameter_types = self._security_declared_parameter_types(fi) return fi, base_stack + frames if not isinstance(node.callee, Identifier): self._codegen_error( @@ -6433,6 +6474,7 @@ def _security_bind_helper_args( param for index, param in enumerate(params) if self._func_param_int_cpp_type( fi, index, self._callable_target_callsite_idx(fi, node)) == "double") + new_frame.parameter_types = self._security_declared_parameter_types(fi) return fi, base_stack + (new_frame,) def _security_helper_call_plan( @@ -6611,6 +6653,9 @@ def _security_value_is_string( nxt = depth + 1 if isinstance(node, StringLiteral): return True + if isinstance(node, Subscript): + return self._security_value_is_string( + node.object, helper_binding_stack, body, nxt) if isinstance(node, Identifier) and not self._security_identifier_is_global_binding(node): for stmt in reversed(body or []): if isinstance(stmt, VarDecl) and stmt.name == node.name: @@ -6619,6 +6664,10 @@ def _security_value_is_string( return self._security_value_is_string( stmt.value, helper_binding_stack, body, nxt ) + parameter_type = self._security_parameter_cpp_type( + node.name, helper_binding_stack) + if parameter_type is not None: + return parameter_type == "std::string" binding = self._security_lookup_helper_binding_context( node.name, helper_binding_stack ) @@ -7814,15 +7863,19 @@ def store(cpp: str) -> str: # (``/`` in double, ``%`` through std::fmod: C++ ``%`` takes no # double) and narrows its result. wide = node.op != ":=" and store(value_cpp) != value_cpp + slot_type = self._security_cpp_type_for_mutable(target_name, info) + string_concat = node.op == "+=" and slot_type == "std::string" def combined(target_read: str) -> str: - return (self._compound_assign_rhs(target_read, node.op, value_cpp) + operand = store(value_cpp) if string_concat else value_cpp + return (self._compound_assign_rhs( + target_read, node.op, operand, target_cpp_type=slot_type) or f"({target_read} {node.op[0]} {value_cpp})") if getattr(info, "is_series", False): if node.op == ":=": lines.append(f"{pad}{state_name}.update({store(value_cpp)});") - elif wide: + elif wide or string_concat: lines.append(f"{pad}{state_name}.update(" f"{store(combined(f'{state_name}[0]'))});") else: @@ -7832,7 +7885,7 @@ def combined(target_read: str) -> str: if node.op == ":=": lines.append(f"{pad}{state_name} = {store(value_cpp)};") - elif wide: + elif wide or string_concat: lines.append(f"{pad}{state_name} = {store(combined(state_name))};") else: lines.append(f"{pad}{state_name} {node.op} {value_cpp};") @@ -9275,7 +9328,7 @@ def _build_security_expr( if column is not None and self._security_foreign(sec_id): # request.footprint(...) of another symbol: the delta its feed # records for the requested bar (the value its delta() reads). - return f'_pf_symbol_column({sec_id}, "{self._cpp_string_escape(column)}")' + return f'_pf_symbol_column({sec_id}, {self._cpp_string_literal(column)})' if resolving is None: resolving = set() @@ -9319,6 +9372,10 @@ def _build_security_expr( ) if self._security_float_parameter(expr_node.name, helper_binding_stack): rendered = self._security_float_cpp(rendered, bound, bound_stack) + if self._security_parameter_cpp_type( + expr_node.name, helper_binding_stack) == "std::string": + rendered = self._security_local_value( + rendered, bound, "std::string", bound_stack) return rendered bar_fields = { **SECURITY_BAR_FIELD_EXPRS, @@ -9401,6 +9458,9 @@ def _build_security_expr( ) resolving.remove(expr_node.name) symbol = self._variable_symbol(expr_node.name) + if self._infer_type(expr_node) == "std::string": + resolved = self._security_local_value( + resolved, value, "std::string", ()) if (symbol is not None and self._global_float_na_selection(expr_node.name, symbol.pine_type)): resolved = self._security_float_cpp(resolved, value) @@ -9985,6 +10045,12 @@ def _build_security_expr( def lower(left: str, right: str) -> str: cpp_ops = {"and": "&&", "or": "||"} op = cpp_ops.get(expr_node.op, expr_node.op) + if (expr_node.op == "+" + and self._security_value_is_string( + expr_node, helper_binding_stack, [])): + # The builder has resolved helper bindings to their + # requested string values, rather than chart locals. + return self._string_concat_cpp(left, right) if expr_node.op in ("and", "or"): left = self._coerce_bool_expr(left, expr_node.left) right = self._coerce_bool_expr(right, expr_node.right) @@ -10208,6 +10274,9 @@ def argument(value): argument = self._build_security_expr( sec_id, argument_node, ta_range, ta_results, resolving, security_mutable_names, frame.caller_stack, emitted_lines) + if frame.parameter_types.get(parameter) == "std::string": + argument = self._security_local_value( + argument, argument_node, "std::string", frame.caller_stack) name = self._security_next_inline_name(sec_id, func_name, parameter) emitted_lines.append(f" const auto {name} = {argument};") frame.samples[parameter] = _SecurityArgumentSample(name, None) @@ -10943,11 +11012,14 @@ def build(value): return True if isinstance(node, Assignment) and id(node.target) in context["reads"]: name = context["reads"][id(node.target)] + cpp_type = context["types"].get(name) value = build(node.value) + if node.op == "+=" and cpp_type == "std::string": + value = self._security_local_value(value, node.value, cpp_type, args[5]) rhs = (value if node.op == ":=" - else self._compound_assign_rhs(name, node.op, value)) + else self._compound_assign_rhs( + name, node.op, value, target_cpp_type=cpp_type)) if rhs is not None: - cpp_type = context["types"].get(name) if node.op == ":=": rhs = self._security_local_value(rhs, node.value, cpp_type, args[5]) elif cpp_type in ("int", "int64_t"): diff --git a/pineforge_codegen/codegen/tables.py b/pineforge_codegen/codegen/tables.py index baa0914a..78155048 100644 --- a/pineforge_codegen/codegen/tables.py +++ b/pineforge_codegen/codegen/tables.py @@ -26,6 +26,8 @@ evaluate_args_once, na_preserving_int_cast, pine_index_int_cast, pine_truth_cast, ) +from .unicode_case import _unicode_case_expr +from .unicode_regex import _str_match_expr _PROVEN_INT_ARG = re.compile( @@ -1071,6 +1073,46 @@ def _array_sort_indices(a: str, args: list[str]) -> str: # Methods called as ``array.method(arr, ...)`` or ``arr.method(...)``. +def _array_join(a: str, args: list[str], *, stringify: str | None = "pine_str_tostring_tv") -> str: + """Join after binding receiver and separator once, in parameter order. + + String elements retain the UTF-16 join, including adjacent surrogate + halves. Numeric elements use the existing Pine formatter; other element + families keep their previous conversion until their domains are measured. + """ + default = ARRAY_OPTIONAL_PARAMS["join"]["separator"] + separator = args[0] if args else f'std::string("{default}")' + if stringify is None: + result = "return _pf_u16_join(_pf_join_parts, _pf_join_separator); " + else: + formatter = "" + if stringify == "pine_str_tostring_tv": + # Keep every integer digit rather than passing a present int64 + # through the formatter's double overload. Test the sentinel in + # its original type; missing integers and floats share Pine's + # existing missing-number spelling. + formatter = ( + "const auto _pf_join_number = [](const auto _pf_join_value) { " + "if constexpr (std::is_integral_v) { " + "if (!is_na(_pf_join_value)) return std::to_string(_pf_join_value); " + "} return pine_str_tostring_tv(_pf_join_value); }; " + ) + stringify = "_pf_join_number" + result = ( + f"{formatter}std::string _pf_join_result; " + "for (std::size_t _pf_join_i = 0; _pf_join_i < _pf_join_parts.size(); " + "++_pf_join_i) { " + "if (_pf_join_i != 0) _pf_join_result += _pf_join_separator; " + f"_pf_join_result += {stringify}(_pf_join_parts[_pf_join_i]); " + "} return _pf_u16_normalize(_pf_join_result); " + ) + return ( + f"[&](){{ const auto& _pf_join_parts = ({a}); " + f"const std::string _pf_join_separator = ({separator}); " + f"{result}}}()" + ) + + ARRAY_METHODS = { "get": _checked_array_get, "set": _checked_array_set, @@ -1126,7 +1168,7 @@ def _array_sort_indices(a: str, args: list[str]) -> str: "percentile_nearest_rank": lambda a, args: f"[&](){{ if({a}.empty()) return na(); auto __pf_c={a}; std::sort(__pf_c.begin(),__pf_c.end()); double __pf_p=({args[0]}); if(is_na(__pf_p)) return (double)__pf_c.front(); double __pf_rank=std::ceil((__pf_p/100.0)*__pf_c.size()); if(!std::isfinite(__pf_rank)) return na(); int __pf_r=(int)std::clamp(__pf_rank,1.0,(double)__pf_c.size()); return (double)__pf_c[__pf_r-1]; }}()", "percentrank": _checked_array_percentrank, "abs": lambda a, args: f"[&](){{ std::vector __pf_r; for(auto v:{a})__pf_r.push_back(std::abs(v)); return __pf_r; }}()", - "join": lambda a, args: "[&](){{ std::string __pf_r; for(size_t __pf_i=0;__pf_i<{arr}.size();__pf_i++){{ if(__pf_i>0)__pf_r+={sep}; __pf_r+=std::to_string({arr}[__pf_i]); }} return __pf_r; }}()".format(arr=a, sep=args[0] if args else 'std::string(",")'), + "join": _array_join, "standardize": lambda a, args: f"[&](){{ double __pf_m=std::accumulate({a}.begin(),{a}.end(),0.0)/{a}.size(); double __pf_s=0; for(auto v:{a})__pf_s+=(v-__pf_m)*(v-__pf_m); __pf_s=std::sqrt(__pf_s/{a}.size()); std::vector __pf_r; for(auto v:{a})__pf_r.push_back(__pf_s==0?1.0:(v-__pf_m)/__pf_s); return __pf_r; }}()", "covariance": lambda a, args: f"[&](){{ auto&&__pf_b=({args[0]}); int __pf_n=std::min({a}.size(),__pf_b.size()); if(__pf_n<=0) return na(); double __pf_ma=0,__pf_mb=0; for(int __pf_i=0;__pf_i<__pf_n;__pf_i++){{__pf_ma+={a}[__pf_i];__pf_mb+=__pf_b[__pf_i];}} __pf_ma/=__pf_n;__pf_mb/=__pf_n; double __pf_c=0; for(int __pf_i=0;__pf_i<__pf_n;__pf_i++)__pf_c+=({a}[__pf_i]-__pf_ma)*(__pf_b[__pf_i]-__pf_mb); return __pf_c/__pf_n; }}()", "binary_search": lambda a, args: f"[&](){{ auto __pf_it=std::lower_bound({a}.begin(),{a}.end(),{args[0]}); return (__pf_it!={a}.end()&&*__pf_it=={args[0]})?(double)(__pf_it-{a}.begin()):-1.0; }}()", @@ -1198,7 +1240,7 @@ def journal_array_method(a: str, method: str, args: list[str], journal: str, "concat": (0,), } -# Pine parameter order for the checked-index subset. Keeping the receiver +# Pine parameter order for keyword-aware array lowerings. Keeping the receiver # (``id``) separate lets method syntax merge ``a.get(index = i)`` while the # namespace-functional path merges ``array.get(id = a, index = i)``. CHECKED_ARRAY_METHOD_KWARGS: dict[str, list[str]] = { @@ -1217,6 +1259,7 @@ def journal_array_method(a: str, method: str, args: list[str], journal: str, "sort_indices": ["order"], "indexof": ["value"], "unshift": ["value"], + "join": ["separator"], } MAP_METHODS = { @@ -1370,9 +1413,11 @@ def _matrix_functional_receiver(method: str, node): "sum": {"id2": None}, "concat": {"vertical": None}, } -ARRAY_OPTIONAL_PARAMS: dict[str, dict[str, int | None]] = { +ARRAY_OPTIONAL_PARAMS: dict[str, dict[str, int | str | None]] = { "fill": {"index_from": 0, "index_to": None}, "sort": {"order": None}, + # Independently measured namespace and method calls omit the separator. + "join": {"separator": ""}, } # Matrix mutators whose established C++ lowering returns ``void``. A Pine @@ -1581,6 +1626,59 @@ def _str_split_expr(args: list[str]) -> str: ) +def _str_trim_expr(args: list[str]) -> str: + """Trim the measured whitespace at the ends, keeping the original bytes.""" + # Match only complete admitted UTF-8 sequences. Every other byte, + # including NUL or malformed UTF-8, extends the retained byte bounds. + return ( + "[](const std::string _pf_trim_text) { " + "const std::size_t _pf_trim_size = _pf_trim_text.size(); " + "std::size_t _pf_trim_first = _pf_trim_size; " + "std::size_t _pf_trim_last = 0; " + "for (std::size_t _pf_trim_i = 0; _pf_trim_i < _pf_trim_size;) { " + "const unsigned char _pf_trim_c = " + "static_cast(_pf_trim_text[_pf_trim_i]); " + "std::size_t _pf_trim_width = 0; " + "if (_pf_trim_c == 0x20u || " + "(_pf_trim_c >= 0x09u && _pf_trim_c <= 0x0Du)) { " + "_pf_trim_width = 1; " + "} else if (_pf_trim_size - _pf_trim_i >= 2 && " + "_pf_trim_c == 0xC2u && " + "static_cast(_pf_trim_text[_pf_trim_i + 1]) == 0xA0u) { " + "_pf_trim_width = 2; " + "} else if (_pf_trim_size - _pf_trim_i >= 3) { " + "const unsigned char _pf_trim_second = " + "static_cast(_pf_trim_text[_pf_trim_i + 1]); " + "const unsigned char _pf_trim_third = " + "static_cast(_pf_trim_text[_pf_trim_i + 2]); " + "if ((_pf_trim_c == 0xE1u && _pf_trim_second == 0x9Au && " + "_pf_trim_third == 0x80u) || " + "(_pf_trim_c == 0xE2u && _pf_trim_second == 0x80u && " + "(_pf_trim_third == 0x83u || _pf_trim_third == 0xA8u)) || " + "(_pf_trim_c == 0xE3u && _pf_trim_second == 0x80u && " + "_pf_trim_third == 0x80u) || " + "(_pf_trim_c == 0xEFu && _pf_trim_second == 0xBBu && " + "_pf_trim_third == 0xBFu)) { " + "_pf_trim_width = 3; } } " + "if (_pf_trim_width != 0) { _pf_trim_i += _pf_trim_width; } " + "else { " + "if (_pf_trim_first == _pf_trim_size) _pf_trim_first = _pf_trim_i; " + "_pf_trim_last = ++_pf_trim_i; } } " + "return _pf_trim_first == _pf_trim_size ? std::string() " + ": _pf_trim_text.substr(_pf_trim_first, _pf_trim_last - _pf_trim_first); " + f"}}(({args[0]}))" + ) + + +def _str_u16_search_expr(function: str, args: list[str]) -> str: + """Bind source, then needle, once each in parameter order.""" + return ( + f"[&](){{ const std::string _pf_search_source = ({args[0]}); " + f"const std::string _pf_search_needle = ({args[1]}); " + f"return _pf_u16_{function}(_pf_search_source, _pf_search_needle); }}()" + ) + + STR_FUNC_MAP = { "tostring": None, # handled separately (already works) "tonumber": lambda args: ( @@ -1591,31 +1689,28 @@ def _str_split_expr(args: list[str]) -> str: f"}}(({args[0]}))" ), "length": lambda args: f"_pf_u16_length({args[0]})", - "contains": lambda args: f"({args[0]}.find({args[1]}) != std::string::npos)", - "startswith": lambda args: f"({args[0]}.substr(0, {args[1]}.length()) == {args[1]})", - "endswith": lambda args: f"({args[0]}.length() >= {args[1]}.length() && {args[0]}.compare({args[0]}.length() - {args[1]}.length(), {args[1]}.length(), {args[1]}) == 0)", - # The two statements keep the source before the needle in evaluation order. - "pos": lambda args: f"[&](){{ const std::string __pf_s = ({args[0]}); const std::string __pf_t = ({args[1]}); return _pf_u16_pos(__pf_s, __pf_t); }}()", + "contains": lambda args: _str_u16_search_expr("contains", args), + "startswith": lambda args: _str_u16_search_expr("startswith", args), + "endswith": lambda args: _str_u16_search_expr("endswith", args), + "pos": lambda args: _str_u16_search_expr("pos", args), "substring": None, # handled separately (2 vs 3 args) "replace_all": lambda args: f"[&](){{ std::string __pf_s={args[0]}; size_t __pf_p=0; while((__pf_p=__pf_s.find({args[1]},__pf_p))!=std::string::npos){{ __pf_s.replace(__pf_p,{args[1]}.length(),{args[2]}); __pf_p+={args[2]}.length(); }} return __pf_s; }}()", "replace": None, # handled separately (3 vs 4 args) - "lower": lambda args: f"[&](){{ std::string __pf_s={args[0]}; std::transform(__pf_s.begin(),__pf_s.end(),__pf_s.begin(),::tolower); return __pf_s; }}()", - "upper": lambda args: f"[&](){{ std::string __pf_s={args[0]}; std::transform(__pf_s.begin(),__pf_s.end(),__pf_s.begin(),::toupper); return __pf_s; }}()", - "trim": lambda args: f'[&](){{ std::string __pf_s={args[0]}; __pf_s.erase(0,__pf_s.find_first_not_of(" \\t\\n\\r")); __pf_s.erase(__pf_s.find_last_not_of(" \\t\\n\\r")+1); return __pf_s; }}()', + "lower": lambda args: _unicode_case_expr(args[0], upper=False), + "upper": lambda args: _unicode_case_expr(args[0], upper=True), + "trim": _str_trim_expr, "repeat": lambda args: ( f"[&](){{ std::string __pf_r; for(int __pf_i=0;__pf_i<{na_preserving_int_cast(args[1])};__pf_i++) __pf_r+={args[0]}; return __pf_r; }}()" if len(args) < 3 else f"[&](){{ std::string __pf_r; for(int __pf_i=0;__pf_i<{na_preserving_int_cast(args[1])};__pf_i++){{ if(__pf_i>0) __pf_r+={args[2]}; __pf_r+={args[0]}; }} return __pf_r; }}()" ), - "match": lambda args: f'pine_str_match({args[0]}, {args[1]})', + "match": _str_match_expr, "split": _str_split_expr, "format": None, # handled separately } # The STR_FUNC_MAP arguments a template reads more than once, or once per # loop iteration: evaluate_args_once binds each that is not a plain read. STR_ARGS_READ_REPEATEDLY = { - "startswith": (1,), - "endswith": (0, 1), "replace_all": (1, 2), "repeat": (0, 1, 2), } diff --git a/pineforge_codegen/codegen/tv_string_units.py b/pineforge_codegen/codegen/tv_string_units.py index 32a3192b..6ef291c5 100644 --- a/pineforge_codegen/codegen/tv_string_units.py +++ b/pineforge_codegen/codegen/tv_string_units.py @@ -4,9 +4,11 @@ the Basic Multilingual Plane (an emoji) is two units, every other character is one. The generated code keeps text as UTF-8, whose byte counts differ from the unit counts for every character that is not plain ASCII (an accent is two -bytes, a CJK character three, an emoji four). ``str.length``, ``str.pos`` and -``str.substring`` count and cut through these translation-unit-local helpers; -the engine is not involved. +bytes, a CJK character three, an emoji four). String measurement, search and +substring use these translation-unit-local helpers. String-array joins also +use the same units, so adjacent surrogate halves recompose across parts and +separators. Concatenation uses the same decode/re-encode path, preserving +unpaired halves; the engine is not involved. A cut between the two units of an emoji leaves a lone surrogate. UTF-8 has no form for it, so each half is kept as its own three-byte sequence, the form @@ -35,12 +37,6 @@ return static_cast(units); } -inline int _pf_u16_pos(const std::string& text, const std::string& needle) { - const std::size_t at = text.find(needle); - if (at == std::string::npos) return -1; - return _pf_u16_length(text.substr(0, at)); -} - inline std::vector _pf_u16_decode(const std::string& text) { std::vector units; units.reserve(text.size()); @@ -68,6 +64,46 @@ return units; } +inline std::size_t _pf_u16_find(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + if (_pf_needle_units.empty()) return 0; + const auto _pf_at = std::search( + _pf_text_units.begin(), _pf_text_units.end(), + _pf_needle_units.begin(), _pf_needle_units.end()); + if (_pf_at == _pf_text_units.end()) return std::string::npos; + return static_cast(_pf_at - _pf_text_units.begin()); +} + +inline int _pf_u16_pos(const std::string& _pf_text, const std::string& _pf_needle) { + const std::size_t _pf_at = _pf_u16_find(_pf_text, _pf_needle); + return _pf_at == std::string::npos ? na() : static_cast(_pf_at); +} + +inline bool _pf_u16_contains(const std::string& _pf_text, + const std::string& _pf_needle) { + return _pf_u16_find(_pf_text, _pf_needle) != std::string::npos; +} + +inline bool _pf_u16_startswith(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + return _pf_needle_units.size() <= _pf_text_units.size() + && std::equal(_pf_needle_units.begin(), _pf_needle_units.end(), + _pf_text_units.begin()); +} + +inline bool _pf_u16_endswith(const std::string& _pf_text, + const std::string& _pf_needle) { + const auto _pf_text_units = _pf_u16_decode(_pf_text); + const auto _pf_needle_units = _pf_u16_decode(_pf_needle); + return _pf_needle_units.size() <= _pf_text_units.size() + && std::equal(_pf_needle_units.rbegin(), _pf_needle_units.rend(), + _pf_text_units.rbegin()); +} + // The units from begin to end, as UTF-8. A high surrogate that the cut leaves // without its low one (or the reverse) is a lone half, kept as three bytes. inline std::string _pf_u16_encode(const std::vector& units, std::size_t begin, @@ -102,6 +138,31 @@ return text; } +inline std::string _pf_u16_normalize(const std::string& text) { + const std::vector units = _pf_u16_decode(text); + return _pf_u16_encode(units, 0, units.size()); +} + +inline std::string _pf_u16_concat(const std::string& left, const std::string& right) { + return _pf_u16_normalize(left + right); +} + +template +inline std::string _pf_u16_join(const _pf_StringArray& _pf_parts, + const std::string& _pf_separator) { + const auto _pf_separator_units = _pf_u16_decode(_pf_separator); + std::vector _pf_join_units; + for (std::size_t _pf_i = 0; _pf_i < _pf_parts.size(); ++_pf_i) { + if (_pf_i != 0) + _pf_join_units.insert(_pf_join_units.end(), + _pf_separator_units.begin(), _pf_separator_units.end()); + const auto _pf_part_units = _pf_u16_decode(_pf_parts[_pf_i]); + _pf_join_units.insert(_pf_join_units.end(), + _pf_part_units.begin(), _pf_part_units.end()); + } + return _pf_u16_encode(_pf_join_units, 0, _pf_join_units.size()); +} + // std::string::substr in units: past the end it throws std::out_of_range with // the text substr gives (the units are counted), and a count beyond the end is // cut to the end. diff --git a/pineforge_codegen/codegen/types.py b/pineforge_codegen/codegen/types.py index 0a649e1b..da3f0a87 100644 --- a/pineforge_codegen/codegen/types.py +++ b/pineforge_codegen/codegen/types.py @@ -50,6 +50,7 @@ from ..cpp_names import cpp_identifier_fragment from ..errors import CodegenInvariantError, Phase from ..external_requests import UNPINNED_ANNOTATION +from ..intrinsic_binding import array_method_domain_error from ..limits import iter_ast_nodes from ..loop_types import FLOAT_REASSIGNMENT, LoopTypeSources, for_counter_type from ..matrix_overloads import matrix_sum_has_rhs @@ -83,6 +84,7 @@ PINE_TYPE_TO_CPP, STRATEGY_DIRECTION_VALUES, TA_RETURNS_BOOL, + _array_join, _matrix_functional_receiver, checked_array_slice, ARRAY_JOURNAL_DELTA_METHODS, @@ -3532,6 +3534,9 @@ def _array_method_expr( if method in ("covariance", "concat") and args: args = [f"_pf_array_id({args[0]})", *args[1:]] spec = spec or TypeSpec.array(TypeSpec.primitive("float")) + problem = array_method_domain_error(method, spec) + if problem is not None: + self._codegen_error(node, problem[0], hint=problem[1]) arr_cpp_type = self._type_spec_to_cpp(spec) elem_cpp = self._type_spec_to_cpp(spec.element) if spec.element is not None else "double" if method == "copy": @@ -3563,13 +3568,14 @@ def _array_method_expr( recv, args, result_type=arr_cpp_type, view_function=view_function, ) - elif method == "join" and elem_cpp == "std::string": - join_args = list(args) or ['std::string(",")'] - lower_receiver = lambda recv: evaluate_args_once( - join_args, ARRAY_ARGS_READ_REPEATEDLY["join"], - lambda a: f"[&](){{ std::string __pf_r; for(size_t __pf_i=0;__pf_i<{recv}.size();__pf_i++){{ if(__pf_i>0)__pf_r+={a[0]}; __pf_r+={recv}[__pf_i]; }} return __pf_r; }}()", - "_pf_array_a", + elif method == "join": + self._uses_tv_string_units = True + stringify = ( + None if elem_cpp == "std::string" + else "pine_str_tostring_tv" if elem_cpp in ("double", "int", "int64_t") + else "std::to_string" ) + lower_receiver = lambda recv: _array_join(recv, args, stringify=stringify) else: numeric_only = { "sum", "avg", "min", "max", "range", "stdev", "variance", "median", @@ -3959,12 +3965,12 @@ def promote_wide_int(cpp_type: str) -> str: _dret = self._drawing_call_return_cpp(node.value) if _dret is not None: return _dret - local = self._builtin_local_symbol(node) - if local is not None: - if local.type_spec is not None: - return promote_wide_int(self._type_spec_to_cpp(local.type_spec)) - if local.pine_type not in (PineType.UNKNOWN, PineType.VOID): - return promote_wide_int(PINE_TYPE_TO_CPP[local.pine_type]) + declared = self._builtin_declaration_symbol(node) + if declared is not None: + if declared.type_spec is not None: + return promote_wide_int(self._type_spec_to_cpp(declared.type_spec)) + if declared.pine_type not in (PineType.UNKNOWN, PineType.VOID): + return promote_wide_int(PINE_TYPE_TO_CPP[declared.pine_type]) # Analyzer scopes are popped before codegen, so ctx.symbols.resolve may # find a same-named global instead of this active callable's plain # local. The local binding is known from the emitted body inventory; @@ -4069,32 +4075,45 @@ def _series_handle_target(self, name: str) -> str | None: return None return DRAWING_TYPE_TO_CPP.get(udt, udt) - def _builtin_local_symbol(self, node): - """Recover a builtin-shadowing declaration from its own callable scope.""" + def _builtin_declaration_symbol(self, node): + """Recover this builtin-named variable's exact authored declaration.""" if node.name not in sigs.BUILTIN_VARIABLES: return None - declaration = ( - node if isinstance(node, VarDecl) - else self.ctx.identifier_binding_nodes.get(id(node)) - ) - if declaration is None: - return None - symbols = getattr(self, "_builtin_local_symbols", None) + if isinstance(node, VarDecl): + declaration = id(node) + else: + binding = ( + self.ctx.identifier_decl_bindings[id(node)] + if id(node) in self.ctx.identifier_decl_bindings + else ValueBindings.declaration_of(node) + ) + if binding is None: + return None + declaration = binding[0] + symbols = getattr(self, "_builtin_declaration_symbols", None) if symbols is None: - symbols = self._builtin_local_symbols = {} + symbols = self._builtin_declaration_symbols = {} + local_symbols = self._builtin_local_symbols = {} for scope in self.ctx.symbols.all_scopes: owner = scope while (owner is not None and not owner.name.startswith(("func_", "method_"))): owner = owner.parent - if owner is None: - continue for symbol in scope.symbols.values(): declared = getattr(symbol, "_pf_decl_node", None) if (symbol.name in sigs.BUILTIN_VARIABLES and isinstance(declared, VarDecl)): symbols[id(declared)] = symbol - return symbols.get(id(declaration)) + if owner is not None: + local_symbols[id(declared)] = symbol + return symbols.get(declaration) + + def _builtin_local_symbol(self, node): + """Recover a builtin-shadowing declaration from its own callable scope.""" + symbol = self._builtin_declaration_symbol(node) + if symbol is None: + return None + return self._builtin_local_symbols.get(id(symbol._pf_decl_node)) def _variable_symbol(self, name: str): """The symbol the variable ``name`` resolves to. @@ -6319,9 +6338,9 @@ def _infer_type( lexical_tuple_type = self._lexical_tuple_cpp_type(node) if lexical_tuple_type is not None: return lexical_tuple_type - local = self._builtin_local_symbol(node) - if local is not None: - return self._type_for_decl(local._pf_decl_node) + declared = self._builtin_declaration_symbol(node) + if declared is not None: + return self._type_for_decl(declared._pf_decl_node) binding = self.ctx.identifier_decl_bindings.get(id(node)) if (binding is not None and binding[0] in self.ctx.primitive_block_shadow_types): @@ -6335,10 +6354,10 @@ def _infer_type( ) if tuple_type is not None: return tuple_type - if not tuple_binding and node.name in ("time", "time_close", "timenow"): + builtin_read = self._reads_builtin_variable(node) + if builtin_read and node.name in ("time", "time_close", "timenow"): return "int64_t" - if (node.name in BAR_FIELDS - or (node.name in BAR_BUILTINS and not tuple_binding)): + if builtin_read: return "double" if (node.name in self._known_vars and not self._known_var_is_lexically_shadowed(node.name)): diff --git a/pineforge_codegen/codegen/unicode_case.py b/pineforge_codegen/codegen/unicode_case.py new file mode 100644 index 00000000..1137157e --- /dev/null +++ b/pineforge_codegen/codegen/unicode_case.py @@ -0,0 +1,252 @@ +"""Emit Unicode casing over runtime UTF-8 strings. + +Mappings and properties come only from the generated, checked-in Unicode +14.0.0 data module. scripts/gen_unicode_case_data.py binds the three exact +official UCD inputs; no host Unicode database or locale supplies data. +The data pin identifies this implementation, not TradingView's whole dialect. +Locale-specific casing and normalization are not applied. + +DerivedCoreProperties supplies Cased and Case_Ignorable. UnicodeData supplies +Nd. SpecialCasing supplies full expansions and the Final_Sigma override. +Ignoring takes precedence for overlapping properties. The cased context also +crosses decimal-number runs without making digits cased or case-ignorable. + +The captureless lambda receives its argument outside its local scope, once. +Adjacent WTF-8 high/low surrogates form one code point before casing. NUL, +isolated surrogate pieces and malformed bytes survive. +""" + +from __future__ import annotations + +from functools import lru_cache + + +_DATA_VERSION = "14.0.0" +_SOURCE_SHA256 = ( + ("UnicodeData.txt", "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +_SOURCE_BYTE_COUNTS = ( + ("UnicodeData.txt", 1897793), + ("SpecialCasing.txt", 16830), + ("DerivedCoreProperties.txt", 1040794), +) + + +@lru_cache(maxsize=1) +def _case_data(): + # Keep unrelated translation paths importable while regenerating data. + # A missing/mismatched module is a build defect, never a host fallback. + from . import _unicode_case_data as data + + if (data.DATA_FORMAT != 1 or data.DATA_VERSION != _DATA_VERSION + or data.SOURCE_SHA256 != _SOURCE_SHA256 + or data.SOURCE_BYTE_COUNTS != _SOURCE_BYTE_COUNTS): + raise RuntimeError("regenerate Unicode casing data from the pinned UCD inputs") + return data + + +def _context_flags(char: str) -> int: + point = ord(char) + ranges = _context_ranges() + low, high = 0, len(ranges) + while low < high: + mid = low + (high - low) // 2 + if ranges[mid][1] < point: + low = mid + 1 + else: + high = mid + if low < len(ranges) and ranges[low][0] <= point: + return ranges[low][2] + return 0 + + +@lru_cache(maxsize=1) +def _context_ranges() -> tuple[tuple[int, int, int], ...]: + return _case_data().CONTEXT_RANGES + + +_CONTEXT_CPP = r""" + static constexpr uint32_t _pf_tolower_final_map[][@FINAL_WIDTH@] = { +@FINAL_MAPPINGS@ + }; + static constexpr uint32_t _pf_case_properties[][3] = { +@PROPERTIES@ + }; + const auto _pf_case_flags = [](uint32_t _pf_case_code) -> unsigned { + std::size_t _pf_case_lo = 0; + std::size_t _pf_case_hi = sizeof(_pf_case_properties) / sizeof(_pf_case_properties[0]); + while (_pf_case_lo < _pf_case_hi) { + const std::size_t _pf_case_mid = _pf_case_lo + (_pf_case_hi - _pf_case_lo) / 2; + if (_pf_case_properties[_pf_case_mid][1] < _pf_case_code) + _pf_case_lo = _pf_case_mid + 1; + else + _pf_case_hi = _pf_case_mid; + } + if (_pf_case_lo < sizeof(_pf_case_properties) / sizeof(_pf_case_properties[0]) + && _pf_case_properties[_pf_case_lo][0] <= _pf_case_code) + return _pf_case_properties[_pf_case_lo][2]; + return 0; + }; + std::vector _pf_case_after(_pf_case_codes.size()); + bool _pf_case_next_cased = false; + for (std::size_t _pf_case_i = _pf_case_codes.size(); _pf_case_i > 0;) { + --_pf_case_i; + _pf_case_after[_pf_case_i] = static_cast(_pf_case_next_cased); + const unsigned _pf_case_props = _pf_case_flags(_pf_case_codes[_pf_case_i]); + if (!(_pf_case_props & 6u)) _pf_case_next_cased = (_pf_case_props & 1u) != 0; + } + bool _pf_case_before = false; +""" + +_CONTEXT_SELECT_CPP = r""" + bool _pf_case_contextual = false; + if (_pf_case_before && !_pf_case_after[_pf_case_i]) { + for (const auto& _pf_case_row : _pf_tolower_final_map) { + if (_pf_case_row[0] != _pf_case_code) continue; + for (uint32_t _pf_case_k = 0; _pf_case_k < _pf_case_row[1]; ++_pf_case_k) + _pf_case_append(_pf_case_row[_pf_case_k + 2]); + _pf_case_contextual = true; + break; + } + } + if (!_pf_case_contextual) +""" + +_CONTEXT_ADVANCE_CPP = r""" + const unsigned _pf_case_props = _pf_case_flags(_pf_case_code); + if (!(_pf_case_props & 6u)) _pf_case_before = (_pf_case_props & 1u) != 0; +""" + +_BODY_CPP = r"""[](const std::string& _pf_case_text) { + // Unicode @VERSION@ casing, independent of the C locale. +@PROVENANCE@ + static constexpr uint32_t @TABLE@[][@WIDTH@] = { +@MAPPINGS@ + }; + std::vector _pf_case_codes; + _pf_case_codes.reserve(_pf_case_text.size()); + for (std::size_t _pf_case_i = 0; _pf_case_i < _pf_case_text.size();) { + const unsigned char _pf_case_lead = static_cast(_pf_case_text[_pf_case_i]); + std::size_t _pf_case_width = _pf_case_lead < 0x80u ? 1 + : _pf_case_lead >= 0xC2u && _pf_case_lead <= 0xDFu ? 2 + : _pf_case_lead >= 0xE0u && _pf_case_lead <= 0xEFu ? 3 + : _pf_case_lead >= 0xF0u && _pf_case_lead <= 0xF4u ? 4 : 0; + uint32_t _pf_case_code = _pf_case_width == 1 ? _pf_case_lead + : _pf_case_lead & (0x7Fu >> _pf_case_width); + bool _pf_case_valid = _pf_case_width && _pf_case_width <= _pf_case_text.size() - _pf_case_i; + if (_pf_case_valid) { + for (std::size_t _pf_case_k = 1; _pf_case_k < _pf_case_width; ++_pf_case_k) { + const unsigned char _pf_case_byte = + static_cast(_pf_case_text[_pf_case_i + _pf_case_k]); + if ((_pf_case_byte & 0xC0u) != 0x80u) { _pf_case_valid = false; break; } + _pf_case_code = (_pf_case_code << 6) | (_pf_case_byte & 0x3Fu); + } + const uint32_t _pf_case_min = _pf_case_width == 1 ? 0 + : _pf_case_width == 2 ? 0x80u : _pf_case_width == 3 ? 0x800u : 0x10000u; + _pf_case_valid = _pf_case_valid && _pf_case_code >= _pf_case_min && _pf_case_code <= 0x10FFFFu; + } + if (!_pf_case_valid) { + _pf_case_code = 0x110000u + _pf_case_lead; + _pf_case_width = 1; + } + if (_pf_case_code >= 0xDC00u && _pf_case_code <= 0xDFFFu + && !_pf_case_codes.empty() + && _pf_case_codes.back() >= 0xD800u && _pf_case_codes.back() <= 0xDBFFu) { + _pf_case_codes.back() = 0x10000u + + ((_pf_case_codes.back() - 0xD800u) << 10) + (_pf_case_code - 0xDC00u); + } else { + _pf_case_codes.push_back(_pf_case_code); + } + _pf_case_i += _pf_case_width; + } +@CONTEXT@ + std::string _pf_case_out; + _pf_case_out.reserve(_pf_case_text.size()); + const auto _pf_case_append = [&_pf_case_out](uint32_t _pf_case_code) { + if (_pf_case_code >= 0x110000u) { + _pf_case_out.push_back(static_cast(_pf_case_code - 0x110000u)); + } else if (_pf_case_code < 0x80u) { + _pf_case_out.push_back(static_cast(_pf_case_code)); + } else if (_pf_case_code < 0x800u) { + _pf_case_out.push_back(static_cast(0xC0u | (_pf_case_code >> 6))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } else if (_pf_case_code < 0x10000u) { + _pf_case_out.push_back(static_cast(0xE0u | (_pf_case_code >> 12))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 6) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } else { + _pf_case_out.push_back(static_cast(0xF0u | (_pf_case_code >> 18))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 12) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | ((_pf_case_code >> 6) & 0x3Fu))); + _pf_case_out.push_back(static_cast(0x80u | (_pf_case_code & 0x3Fu))); + } + }; + for (std::size_t _pf_case_i = 0; _pf_case_i < _pf_case_codes.size(); ++_pf_case_i) { + const uint32_t _pf_case_code = _pf_case_codes[_pf_case_i]; +@CONTEXT_SELECT@ + { + std::size_t _pf_case_lo = 0; + std::size_t _pf_case_hi = sizeof(@TABLE@) / sizeof(@TABLE@[0]); + while (_pf_case_lo < _pf_case_hi) { + const std::size_t _pf_case_mid = _pf_case_lo + (_pf_case_hi - _pf_case_lo) / 2; + if (@TABLE@[_pf_case_mid][0] < _pf_case_code) _pf_case_lo = _pf_case_mid + 1; + else _pf_case_hi = _pf_case_mid; + } + if (_pf_case_lo < sizeof(@TABLE@) / sizeof(@TABLE@[0]) + && @TABLE@[_pf_case_lo][0] == _pf_case_code) { + for (uint32_t _pf_case_k = 0; _pf_case_k < @TABLE@[_pf_case_lo][1]; ++_pf_case_k) + _pf_case_append(@TABLE@[_pf_case_lo][_pf_case_k + 2]); + } else { + _pf_case_append(_pf_case_code); + } + } +@CONTEXT_ADVANCE@ + } + return _pf_case_out; +}""" + + +@lru_cache(maxsize=2) +def _case_lambda(upper: bool) -> str: + data = _case_data() + rows = data.UPPER_ROWS if upper else data.LOWER_ROWS + width = max(len(row) for row in rows) + mappings = "\n".join( + " {" + ", ".join(f"0x{value:X}u" for value in row) + "}," + for row in rows + ) + context = "" + if not upper: + properties = "\n".join( + f" {{0x{start:X}u, 0x{end:X}u, {flags}u}}," + for start, end, flags in _context_ranges() + ) + final_rows = data.LOWER_FINAL_SIGMA_ROWS + final_mappings = "\n".join( + " {" + ", ".join(f"0x{value:X}u" for value in row) + "}," + for row in final_rows + ) + context = (_CONTEXT_CPP.replace("@PROPERTIES@", properties) + .replace("@FINAL_WIDTH@", str(max(len(row) for row in final_rows))) + .replace("@FINAL_MAPPINGS@", final_mappings)) + result = _BODY_CPP + for key, value in ( + ("@VERSION@", data.DATA_VERSION), + ("@PROVENANCE@", "\n".join( + f" // {name} SHA-256 {digest}" for name, digest in data.SOURCE_SHA256 + )), + ("@TABLE@", "_pf_toupper_map" if upper else "_pf_tolower_map"), + ("@WIDTH@", str(width)), + ("@MAPPINGS@", mappings), + ("@CONTEXT@", context), + ("@CONTEXT_SELECT@", "" if upper else _CONTEXT_SELECT_CPP), + ("@CONTEXT_ADVANCE@", "" if upper else _CONTEXT_ADVANCE_CPP), + ): + result = result.replace(key, value) + return result + + +def _unicode_case_expr(text: str, *, upper: bool) -> str: + return f"({_case_lambda(upper)})(({text}))" diff --git a/pineforge_codegen/codegen/unicode_regex.py b/pineforge_codegen/codegen/unicode_regex.py new file mode 100644 index 00000000..5cd098d1 --- /dev/null +++ b/pineforge_codegen/codegen/unicode_regex.py @@ -0,0 +1,438 @@ +"""Lower Unicode-character regex atoms to ordinary byte-regex expressions. + +The runtime regex accepts bytes. A Unicode character is represented by its one +to four byte encoding, retaining an isolated surrogate as three bytes. Each +regex atom is a noncapturing group over those encodings, so a quantifier +repeats characters. Class unions, ranges and complements use the entire +character domain, not an inventory of Unicode letters. + +The engine selects its first capture. An outer capture therefore selects the +whole match, with user captures retained and decimal backreferences shifted by +one. Matching and coded regex errors still use the runtime string helper. +Valid surrogate pairs form one regex character; lengths and substring offsets +continue to use the UTF-16 codec. +""" + + +def _str_match_expr(args: list[str]) -> str: + """Bind both argument expressions once in declared parameter order.""" + return ( + "[&]() { " + f"const std::string _pf_match_source = ({args[0]}); " + f"const std::string _pf_match_pattern = ({args[1]}); " + "return ::_pf_regex::_pf_match(" + "_pf_match_source, _pf_match_pattern, bar_index_); }()" + ) + + +UNICODE_REGEX_CPP = r""" +namespace _pf_regex { +using _pf_ranges = std::vector>; +struct _pf_invalid {}; + +inline std::vector _pf_characters(const std::string& _pf_text) { + const auto _pf_decoded = _pf_u16_decode(_pf_text); + std::vector _pf_result; + for (std::size_t _pf_i = 0; _pf_i < _pf_decoded.size(); ++_pf_i) { + uint32_t _pf_code = _pf_decoded[_pf_i]; + if (_pf_code >= 0xD800u && _pf_code <= 0xDBFFu + && _pf_i + 1 < _pf_decoded.size() + && _pf_decoded[_pf_i + 1] >= 0xDC00u + && _pf_decoded[_pf_i + 1] <= 0xDFFFu) { + _pf_code = 0x10000u + ((_pf_code - 0xD800u) << 10) + + (_pf_decoded[++_pf_i] - 0xDC00u); + } + _pf_result.push_back(_pf_code); + } + return _pf_result; +} + +inline _pf_ranges _pf_normalize(_pf_ranges _pf_values) { + std::sort(_pf_values.begin(), _pf_values.end()); + _pf_ranges _pf_result; + for (const auto& _pf_value : _pf_values) { + if (!_pf_result.empty() && _pf_value.first <= _pf_result.back().second + 1) + _pf_result.back().second = std::max(_pf_result.back().second, _pf_value.second); + else + _pf_result.push_back(_pf_value); + } + return _pf_result; +} + +inline _pf_ranges _pf_complement(_pf_ranges _pf_values) { + _pf_ranges _pf_result; + uint32_t _pf_next = 0; + for (const auto& _pf_value : _pf_normalize(std::move(_pf_values))) { + if (_pf_next < _pf_value.first) + _pf_result.emplace_back(_pf_next, _pf_value.first - 1); + _pf_next = _pf_value.second + 1; + } + if (_pf_next <= 0x10FFFFu) _pf_result.emplace_back(_pf_next, 0x10FFFFu); + return _pf_result; +} + +inline std::string _pf_hex(uint32_t _pf_value) { + const char* _pf_digits = "0123456789abcdef"; + std::string _pf_result = "\\x"; + _pf_result += _pf_digits[(_pf_value >> 4) & 15]; + _pf_result += _pf_digits[_pf_value & 15]; + return _pf_result; +} + +inline std::string _pf_byte_range(uint32_t _pf_low, uint32_t _pf_high) { + if (_pf_low == _pf_high) return _pf_hex(_pf_low); + return "[" + _pf_hex(_pf_low) + "-" + _pf_hex(_pf_high) + "]"; +} + +inline std::vector _pf_bytes(uint32_t _pf_unit) { + if (_pf_unit < 0x80u) return {_pf_unit}; + if (_pf_unit < 0x800u) + return {0xC0u | (_pf_unit >> 6), 0x80u | (_pf_unit & 63)}; + if (_pf_unit < 0x10000u) + return {0xE0u | (_pf_unit >> 12), 0x80u | ((_pf_unit >> 6) & 63), + 0x80u | (_pf_unit & 63)}; + return {0xF0u | (_pf_unit >> 18), 0x80u | ((_pf_unit >> 12) & 63), + 0x80u | ((_pf_unit >> 6) & 63), 0x80u | (_pf_unit & 63)}; +} + +// Split ranges at differing encoding prefixes instead of enumerating members. +inline void _pf_byte_trie(const std::vector& _pf_low, + const std::vector& _pf_high, + std::size_t _pf_at, const std::string& _pf_prefix, + std::vector& _pf_alternatives) { + if (_pf_at + 1 == _pf_low.size()) { + _pf_alternatives.push_back( + _pf_prefix + _pf_byte_range(_pf_low[_pf_at], _pf_high[_pf_at])); + return; + } + if (_pf_low[_pf_at] == _pf_high[_pf_at]) { + _pf_byte_trie(_pf_low, _pf_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_low[_pf_at]), _pf_alternatives); + return; + } + auto _pf_first_high = _pf_low; + auto _pf_last_low = _pf_high; + for (std::size_t _pf_i = _pf_at + 1; _pf_i < _pf_low.size(); ++_pf_i) { + _pf_first_high[_pf_i] = 0xBFu; + _pf_last_low[_pf_i] = 0x80u; + } + _pf_byte_trie(_pf_low, _pf_first_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_low[_pf_at]), _pf_alternatives); + if (_pf_low[_pf_at] + 1 < _pf_high[_pf_at]) { + std::string _pf_middle = _pf_prefix + + _pf_byte_range(_pf_low[_pf_at] + 1, _pf_high[_pf_at] - 1); + for (std::size_t _pf_i = _pf_at + 1; _pf_i < _pf_low.size(); ++_pf_i) + _pf_middle += "[\\x80-\\xbf]"; + _pf_alternatives.push_back(std::move(_pf_middle)); + } + _pf_byte_trie(_pf_last_low, _pf_high, _pf_at + 1, + _pf_prefix + _pf_hex(_pf_high[_pf_at]), _pf_alternatives); +} + +inline std::string _pf_atom(_pf_ranges _pf_values) { + std::vector _pf_alternatives; + for (const auto& _pf_value : _pf_normalize(std::move(_pf_values))) { + uint32_t _pf_low = _pf_value.first; + for (uint32_t _pf_limit : {0x7Fu, 0x7FFu, 0xFFFFu, 0x10FFFFu}) { + if (_pf_low > _pf_value.second || _pf_low > _pf_limit) continue; + const auto _pf_high = std::min(_pf_value.second, _pf_limit); + _pf_byte_trie(_pf_bytes(_pf_low), _pf_bytes(_pf_high), + 0, "", _pf_alternatives); + _pf_low = _pf_high + 1; + } + } + if (_pf_alternatives.empty()) return "(?!)"; + std::string _pf_result = "(?:"; + for (const auto& _pf_alternative : _pf_alternatives) { + if (_pf_result.size() > 3) _pf_result += "|"; + _pf_result += _pf_alternative; + } + return _pf_result + ")"; +} + +inline _pf_ranges _pf_class_escape(uint32_t _pf_code) { + _pf_ranges _pf_result; + switch (_pf_code) { + case 'd': case 'D': + _pf_result = {{'0', '9'}}; + break; + case 'w': case 'W': + _pf_result = {{'0', '9'}, {'A', 'Z'}, {'_', '_'}, {'a', 'z'}}; + break; + case 's': case 'S': + _pf_result = {{9, 13}, {32, 32}}; + break; + default: throw _pf_invalid{}; + } + if (_pf_code == 'D' || _pf_code == 'W' || _pf_code == 'S') + return _pf_complement(std::move(_pf_result)); + return _pf_result; +} + +inline bool _pf_is_class_escape(uint32_t _pf_code) { + return _pf_code == 'd' || _pf_code == 'D' || _pf_code == 'w' + || _pf_code == 'W' || _pf_code == 's' || _pf_code == 'S'; +} + +struct _pf_member { + _pf_ranges _pf_values; + bool _pf_single; + uint32_t _pf_unit; +}; + +class _pf_pattern { + const std::vector& _pf_units; + std::size_t _pf_at = 0; + + uint32_t _pf_take() { + if (_pf_at == _pf_units.size()) throw _pf_invalid{}; + return _pf_units[_pf_at++]; + } + + uint32_t _pf_hex_unit(unsigned _pf_count) { + uint32_t _pf_value = 0; + while (_pf_count--) { + const auto _pf_code = _pf_take(); + const int _pf_digit = _pf_code >= '0' && _pf_code <= '9' + ? static_cast(_pf_code - '0') + : _pf_code >= 'a' && _pf_code <= 'f' ? static_cast(_pf_code - 'a') + 10 + : _pf_code >= 'A' && _pf_code <= 'F' ? static_cast(_pf_code - 'A') + 10 : -1; + if (_pf_digit < 0) throw _pf_invalid{}; + _pf_value = (_pf_value << 4) | static_cast(_pf_digit); + } + return _pf_value; + } + + uint32_t _pf_character_escape(uint32_t _pf_code) { + switch (_pf_code) { + case 'u': return _pf_hex_unit(4); + case 'x': return _pf_hex_unit(2); + case 'b': return 8; + case 'f': return 12; + case 'n': return 10; + case 'r': return 13; + case 't': return 9; + case 'v': return 11; + case '0': return 0; + case 'c': { + const auto _pf_letter = _pf_take(); + if (!((_pf_letter >= 'A' && _pf_letter <= 'Z') + || (_pf_letter >= 'a' && _pf_letter <= 'z'))) + throw _pf_invalid{}; + return _pf_letter & 31; + } + default: return _pf_code; + } + } + + uint32_t _pf_pair(uint32_t _pf_high) { + if (_pf_high < 0xD800u || _pf_high > 0xDBFFu) return _pf_high; + const auto _pf_saved = _pf_at; + uint32_t _pf_low; + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= 0xDC00u + && _pf_units[_pf_at] <= 0xDFFFu) { + _pf_low = _pf_take(); + } else if (_pf_at + 1 < _pf_units.size() && _pf_units[_pf_at] == '\\' + && _pf_units[_pf_at + 1] == 'u') { + _pf_at += 2; + _pf_low = _pf_hex_unit(4); + } else { + return _pf_high; + } + if (_pf_low < 0xDC00u || _pf_low > 0xDFFFu) { + _pf_at = _pf_saved; + return _pf_high; + } + return 0x10000u + ((_pf_high - 0xD800u) << 10) + (_pf_low - 0xDC00u); + } + + // The synthetic whole-match group shifts every user capture by one. + // Add in decimal text so an invalid oversized reference cannot overflow + // into a valid group number before the engine checks it. + std::string _pf_backreference(uint32_t _pf_first) { + std::string _pf_digits(1, static_cast(_pf_first)); + while (_pf_at < _pf_units.size() && _pf_units[_pf_at] >= '0' + && _pf_units[_pf_at] <= '9') + _pf_digits += static_cast(_pf_take()); + bool _pf_carry = true; + for (std::size_t _pf_i = _pf_digits.size(); _pf_i > 0 && _pf_carry;) { + --_pf_i; + if (_pf_digits[_pf_i] == '9') _pf_digits[_pf_i] = '0'; + else { ++_pf_digits[_pf_i]; _pf_carry = false; } + } + if (_pf_carry) _pf_digits.insert(_pf_digits.begin(), '1'); + return "\\" + _pf_digits; + } + + // Classic-locale named/collating classes are sets of ASCII characters. + _pf_member _pf_named_member(uint32_t _pf_delimiter) { + std::string _pf_name; + for (;;) { + const auto _pf_code = _pf_take(); + if (_pf_code == _pf_delimiter && _pf_at < _pf_units.size() + && _pf_units[_pf_at] == ']') { + ++_pf_at; + break; + } + if (_pf_code > 127) throw _pf_invalid{}; + _pf_name += static_cast(_pf_code); + } + std::regex_traits _pf_traits; + _pf_traits.imbue(std::locale::classic()); + if (_pf_delimiter != ':') { + const auto _pf_value = _pf_traits.lookup_collatename( + _pf_name.begin(), _pf_name.end()); + if (_pf_value.size() != 1) throw _pf_invalid{}; + const auto _pf_unit = static_cast(_pf_value[0]); + if (_pf_delimiter == '=') { + const auto _pf_primary = _pf_traits.transform_primary( + _pf_value.begin(), _pf_value.end()); + _pf_ranges _pf_values; + for (uint32_t _pf_candidate = 0; _pf_candidate < 128; ++_pf_candidate) { + const std::string _pf_byte(1, static_cast(_pf_candidate)); + if (_pf_traits.transform_primary(_pf_byte.begin(), _pf_byte.end()) + == _pf_primary) + _pf_values.emplace_back(_pf_candidate, _pf_candidate); + } + return {std::move(_pf_values), false, 0}; + } + return {{{_pf_unit, _pf_unit}}, true, _pf_unit}; + } + const auto _pf_mask = _pf_traits.lookup_classname(_pf_name.begin(), _pf_name.end()); + if (_pf_mask == decltype(_pf_mask)()) throw _pf_invalid{}; + _pf_ranges _pf_values; + for (uint32_t _pf_unit = 0; _pf_unit < 128; ++_pf_unit) + if (_pf_traits.isctype(static_cast(_pf_unit), _pf_mask)) + _pf_values.emplace_back(_pf_unit, _pf_unit); + return {std::move(_pf_values), false, 0}; + } + + _pf_member _pf_class_member() { + auto _pf_code = _pf_take(); + if (_pf_code == '[' && _pf_at < _pf_units.size() + && (_pf_units[_pf_at] == ':' || _pf_units[_pf_at] == '.' + || _pf_units[_pf_at] == '=')) { + return _pf_named_member(_pf_take()); + } + uint32_t _pf_unit = _pf_code; + if (_pf_code == '\\') { + _pf_code = _pf_take(); + if (_pf_is_class_escape(_pf_code)) + return {_pf_class_escape(_pf_code), false, 0}; + _pf_unit = _pf_character_escape(_pf_code); + } + _pf_unit = _pf_pair(_pf_unit); + return {{{_pf_unit, _pf_unit}}, true, _pf_unit}; + } + + std::string _pf_bracket() { + const bool _pf_negate = _pf_at < _pf_units.size() && _pf_units[_pf_at] == '^'; + if (_pf_negate) ++_pf_at; + _pf_ranges _pf_values; + while (_pf_at < _pf_units.size() && _pf_units[_pf_at] != ']') { + auto _pf_left = _pf_class_member(); + if (_pf_at + 1 < _pf_units.size() && _pf_units[_pf_at] == '-' + && _pf_units[_pf_at + 1] != ']') { + ++_pf_at; + const auto _pf_right = _pf_class_member(); + if (!_pf_left._pf_single || !_pf_right._pf_single + || _pf_left._pf_unit > _pf_right._pf_unit) + throw _pf_invalid{}; + _pf_values.emplace_back(_pf_left._pf_unit, _pf_right._pf_unit); + } else { + _pf_values.insert(_pf_values.end(), _pf_left._pf_values.begin(), + _pf_left._pf_values.end()); + } + } + if (_pf_take() != ']') throw _pf_invalid{}; + return _pf_atom(_pf_negate ? _pf_complement(std::move(_pf_values)) + : std::move(_pf_values)); + } + +public: + explicit _pf_pattern(const std::vector& _pf_value) : _pf_units(_pf_value) {} + + std::string _pf_compile() { + std::string _pf_result = "("; + std::size_t _pf_depth = 0; + while (_pf_at < _pf_units.size()) { + auto _pf_code = _pf_take(); + if (_pf_code == '[') { + _pf_result += _pf_bracket(); + } else if (_pf_code == '.') { + _pf_result += _pf_atom(_pf_complement({{10, 10}, {13, 13}, + {0x2028, 0x2029}})); + } else if (_pf_code == '\\') { + _pf_code = _pf_take(); + if (_pf_is_class_escape(_pf_code)) { + _pf_result += _pf_atom(_pf_class_escape(_pf_code)); + } else if (_pf_code == 'u' || _pf_code == 'x' || _pf_code > 127) { + const auto _pf_unit = _pf_pair(_pf_character_escape(_pf_code)); + _pf_result += _pf_atom({{_pf_unit, _pf_unit}}); + } else if (_pf_code >= '1' && _pf_code <= '9') { + _pf_result += _pf_backreference(_pf_code); + } else { + // Assertions and ASCII character escapes keep their grammar. + _pf_result += '\\'; + _pf_result += static_cast(_pf_code); + if (_pf_code == 'c') _pf_result += static_cast(_pf_take()); + } + } else if (_pf_code == '(') { + ++_pf_depth; + _pf_result += '('; + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '?') { + _pf_result += static_cast(_pf_take()); + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] < 128) + _pf_result += static_cast(_pf_take()); + } + } else if (_pf_code == ')') { + // The wrapper must not repair an unmatched user parenthesis. + if (_pf_depth == 0) throw _pf_invalid{}; + --_pf_depth; + _pf_result += ')'; + } else if (_pf_code == '{') { + _pf_result += '{'; + while (_pf_at < _pf_units.size() + && ((_pf_units[_pf_at] >= '0' && _pf_units[_pf_at] <= '9') + || _pf_units[_pf_at] == ',')) + _pf_result += static_cast(_pf_take()); + if (_pf_at < _pf_units.size() && _pf_units[_pf_at] == '}') + _pf_result += static_cast(_pf_take()); + } else if (_pf_code == '|' || _pf_code == '^' + || _pf_code == '$' || _pf_code == '*' || _pf_code == '+' + || _pf_code == '?' || _pf_code == '}') { + _pf_result += static_cast(_pf_code); + } else { + _pf_code = _pf_pair(_pf_code); + _pf_result += _pf_atom({{_pf_code, _pf_code}}); + } + } + if (_pf_depth != 0) throw _pf_invalid{}; + return _pf_result + ")"; + } +}; + +inline std::string _pf_match(const std::string& _pf_source, + const std::string& _pf_regex, int64_t _pf_bar) { + const auto _pf_pattern_units = _pf_characters(_pf_regex); + std::string _pf_pattern_bytes; + try { + _pf_pattern_bytes = _pf_pattern(_pf_pattern_units)._pf_compile(); + } catch (const _pf_invalid&) { + // Retain the engine's exact reason/context for byte-invalid patterns. + // A range invalid only in characters needs the same coded stop. + (void)::pineforge::pine_str_match(_pf_source, _pf_regex); + const std::string _pf_message = "Error on bar " + std::to_string(_pf_bar) + + ": Invalid regex pattern in the 'str.match' function."; + _PF_STRING_STOP("invalid_regex_pattern", _pf_message.c_str()); + } + std::string _pf_source_bytes; + for (const uint32_t _pf_unit : _pf_characters(_pf_source)) + for (const uint32_t _pf_byte : _pf_bytes(_pf_unit)) + _pf_source_bytes += static_cast(_pf_byte); + const auto _pf_matched = ::pineforge::pine_str_match(_pf_source_bytes, _pf_pattern_bytes); + const auto _pf_matched_units = _pf_u16_decode(_pf_matched); + return _pf_u16_encode(_pf_matched_units, 0, _pf_matched_units.size()); +} +} // namespace _pf_regex +""" diff --git a/pineforge_codegen/codegen/visit_call.py b/pineforge_codegen/codegen/visit_call.py index edebfd5f..f94a04f7 100644 --- a/pineforge_codegen/codegen/visit_call.py +++ b/pineforge_codegen/codegen/visit_call.py @@ -872,11 +872,11 @@ def _array_method_args( ] def _array_method_arg_nodes(self, method: str, node: FuncCall) -> list: - """Merge checked-array method kwargs into Pine signature order.""" + """Merge registered array method kwargs into Pine signature order.""" return self._collection_arg_nodes("array", method, node, False) def _collection_arg_nodes(self, namespace, method, node, functional): - """The arguments of a matrix or checked-array method call in Pine's + """The arguments of a matrix or registered array method call in Pine's parameter order (the receiver first in the namespace form). A keyword argument binds to its parameter's own slot, never shifted @@ -885,7 +885,7 @@ def _collection_arg_nodes(self, namespace, method, node, functional): receiver's own extent, which the method's template reads). A keyword the lowering does not consume -- an unknown name, a parameter given twice -- is refused, as is a gap before a required parameter. A call - without keywords, and an array method outside the checked-index + without keywords, and an array method outside the keyword signature table, keep their arguments as written.""" if not node.kwargs: return list(node.args) @@ -926,7 +926,14 @@ def _collection_arg_nodes(self, namespace, method, node, functional): if slots[index] not in optional: self._collection_argument_error(node, namespace, method, "missing", slots[index]) default = optional[slots[index]] - arguments[index] = None if default is None else NumberLiteral(value=default) + if default is None: + arguments[index] = None + elif isinstance(default, str): + arguments[index] = StringLiteral(value=default) + elif isinstance(default, bool): + arguments[index] = BoolLiteral(value=default) + else: + arguments[index] = NumberLiteral(value=default) return arguments def _collection_argument_error(self, node, namespace, method, problem, name): @@ -4297,8 +4304,14 @@ def slot(index: int): def _visit_str_call(self, func_name: str, node) -> str: if func_name == "format_time": return self._visit_format_time_call(node) + visit_arg = self._visit_expr + if func_name == "split": + # Both parameters are strings; use their contextual type for na + # just as for a string declaration or assignment. + visit_arg = lambda arg: self._visit_rhs_value( + arg, target_cpp_type="std::string") args = _merge_kwargs(node.args, node.kwargs, - sigs.get_param_names("str", func_name), self._visit_expr, + sigs.get_param_names("str", func_name), visit_arg, intrinsic=sigs.get_intrinsic("str", func_name)) arg_nodes = _merge_kwargs( node.args, node.kwargs, @@ -4365,12 +4378,44 @@ def _visit_str_call(self, func_name: str, node) -> str: if func_name == "substring": if len(args) in (2, 3): end_parameter = ", auto _pf_substring_end" if len(args) == 3 else "" - count = ", _pf_substring_end - _pf_substring_begin" if len(args) == 3 else "" + # Test sentinels in their original types: arithmetic can carry + # an int64_t na even when str.pos itself returned an int na. + end_guard = ( + "if (is_na(_pf_substring_end)) return na(); " + if len(args) == 3 else "" + ) + begin = na_preserving_int_cast("_pf_substring_begin", "int64_t") + count = "" + count_guard = "" + if len(args) == 3: + # Byte length bounds UTF-16 length from above. Beyond it, + # use the existing helper's end clamp without converting + # the end. Otherwise ordered nonnegative size_t operands + # give a count without signed overflow or unsigned wrap. + count_guard = ( + "if (_pf_substring_end < _pf_substring_start) " + 'throw std::out_of_range("str.substring: end_pos is less than begin_pos"); ' + "const std::size_t _pf_substring_count = " + "_pf_substring_end > _pf_substring_text.size() ? std::string::npos : " + "static_cast(_pf_substring_end) - " + "static_cast(_pf_substring_start); " + ) + count = ", _pf_substring_count" def checked_substring(bound): return ( f"[&](const std::string& _pf_substring_text, auto _pf_substring_begin{end_parameter}){{ " - "try { return _pf_u16_substr(_pf_substring_text, _pf_substring_begin" + f"{end_guard}" + "const int64_t _pf_substring_start = " + f"is_na(_pf_substring_begin) ? int64_t{{0}} : {begin}; " + "try { " + # Preserve the helper's existing invalid-begin stop, + # including its unsigned position and UTF-16 size. + "if (_pf_substring_start < 0) return _pf_u16_substr(" + "_pf_substring_text, static_cast(_pf_substring_start)); " + f"{count_guard}" + "return _pf_u16_substr(_pf_substring_text, " + "static_cast(_pf_substring_start)" f"{count}); }} catch (const std::out_of_range& _pf_substring_error) {{ " '_PF_STRING_STOP("substring_out_of_range", _pf_substring_error.what()); ' "return std::string(); } " @@ -4412,9 +4457,11 @@ def checked_substring(bound): return 'std::string("")' if func_name in STR_FUNC_MAP and STR_FUNC_MAP[func_name] is not None: - if func_name in ("length", "pos"): - # These two count in UTF-16 units through tv_string_units. + if func_name in ("length", "pos", "contains", "startswith", "endswith", "match"): + # Measurement/searches use units; matching decodes characters. self._uses_tv_string_units = True + if func_name == "match": + self._uses_unicode_regex = True return evaluate_args_once( args, STR_ARGS_READ_REPEATEDLY.get(func_name, ()), STR_FUNC_MAP[func_name], "_pf_str_a") diff --git a/pineforge_codegen/codegen/visit_expr.py b/pineforge_codegen/codegen/visit_expr.py index 194f0cda..2b163875 100644 --- a/pineforge_codegen/codegen/visit_expr.py +++ b/pineforge_codegen/codegen/visit_expr.py @@ -91,6 +91,7 @@ from .run_stops import checked_history_offset, history_offset_guard, request_stop from .capacities import literal_history_offset +from ..analyzer.value_bindings import ValueBindings from ..errors import CodegenInvariantError, Phase from ..symbols import TypeSpec from ..external_requests import UNPINNED_ANNOTATION @@ -775,20 +776,30 @@ def _identifier_reads_series(self, node: Identifier) -> bool: if name in self._current_func_series_params: return True if (name in self._current_func_param_types - or (name in BAR_FIELDS and self._builtin_local_symbol(node) is None) or self._reads_builtin_variable(node)): return False return self._binding_is_series( name, self._call_site_var_name(node, self._safe_name(name))) def _reads_builtin_variable(self, node: Identifier) -> bool: - """Whether ``node`` reads a builtin variable (``second``, ``hour``, - ``bar_index``...) and not a user variable of the same name - (``float second = na``). A read the analyzer bound to a user - declaration in scope is the user's: it carries the annotation a - tuple-destructured name does, or ``builtin_shadow_binding``.""" + """Whether this read resolves to a builtin rather than an authored value. + + Bar fields follow the same lexical rule as other builtins. Preserve + the analyzer's exact binding, including an explicit builtin binding + before a later declaration. Copied reads retain authored-use metadata. + """ + name = node.name + if ((name not in BAR_FIELDS and name not in BAR_BUILTINS) + or name in self._current_func_param_types + or name in self._current_func_series_params): + return False + binding = ( + self.ctx.identifier_decl_bindings[id(node)] + if id(node) in self.ctx.identifier_decl_bindings + else ValueBindings.declaration_of(node) + ) annotations = node.annotations or {} - return (node.name in BAR_BUILTINS + return (binding is None and not annotations.get("tuple_builtin_binding") and not annotations.get("builtin_shadow_binding")) @@ -810,10 +821,8 @@ def _visit_ident(self, node: Identifier) -> str: if (retained and name in retained.aliases and not self._known_var_is_lexically_shadowed(name)): return f"(*{self._safe_name(name)})" - if name in BAR_FIELDS and self._builtin_local_symbol(node) is None: - return BAR_FIELDS[name] if self._reads_builtin_variable(node): - return BAR_BUILTINS[name] + return BAR_FIELDS[name] if name in BAR_FIELDS else BAR_BUILTINS[name] binding = self.ctx.identifier_decl_bindings.get(id(node)) primitive_shadow_read = ( binding is not None @@ -1969,6 +1978,24 @@ def _callable_has_ordered_effect(self, key: str) -> bool: ) return memo[key] + def _concat_type(self, operand) -> str: + """The scalar type of a concatenation operand, including series locals.""" + inferred = self._infer_type(operand) + if (inferred == "double" + and isinstance(operand, Identifier) + and operand.name in self._current_func_locals + and self._identifier_reads_series(operand) + and self._series_type_for(operand.name) == "std::string"): + # Series locals bypass the scalar local-type map. Their current + # slot has the same type as their history reads. + return "std::string" + return inferred + + def _string_concat_cpp(self, left: str, right: str) -> str: + """Join two rendered string values through their UTF-16 units.""" + self._uses_tv_string_units = True + return f"_pf_u16_concat({left}, {right})" + def _lower_binop(self, node: BinOp, left: str, right: str, widened: bool = False, na_checked: bool = False) -> str: """The C++ of ``node`` over its rendered operands; ``widened`` when @@ -1989,20 +2016,8 @@ def _lower_binop(self, node: BinOp, left: str, right: str, f"const bool _pf_v5_r = ({right}); " f"return _pf_v5_l {op} _pf_v5_r; }}())") if node.op == "+": - def _concat_type(operand): - inferred = self._infer_type(operand) - if (inferred == "double" - and isinstance(operand, Identifier) - and operand.name in self._current_func_locals - and self._identifier_reads_series(operand) - and self._series_type_for(operand.name) == "std::string"): - # Series locals bypass the scalar local-type map. Their - # current slot has the same type as their history reads. - return "std::string" - return inferred - - lt = _concat_type(node.left) - rt = _concat_type(node.right) + lt = self._concat_type(node.left) + rt = self._concat_type(node.right) if lt == "std::string" or rt == "std::string": def _as_string(rendered, inferred): if inferred == "std::string": @@ -2011,7 +2026,8 @@ def _as_string(rendered, inferred): return f'(({rendered}) ? std::string("true") : std::string("false"))' return f"std::to_string({rendered})" - return f"({_as_string(left, lt)} + {_as_string(right, rt)})" + return self._string_concat_cpp( + _as_string(left, lt), _as_string(right, rt)) if (node.op in ("+", "-", "*", "/", "%") and not widened and not na_checked and any(self._integer_na_operand(side) diff --git a/pineforge_codegen/codegen/visit_stmt.py b/pineforge_codegen/codegen/visit_stmt.py index e367346c..2b492c26 100644 --- a/pineforge_codegen/codegen/visit_stmt.py +++ b/pineforge_codegen/codegen/visit_stmt.py @@ -263,6 +263,11 @@ def _visit_stmt(self, node: ASTNode, lines: list[str], indent: int) -> None: ).get(id(node)) if metadata is not None: member_name = metadata[1] + if metadata[4] and metadata[2] == PineType.STRING: + # Persistent callable locals bypass ordinary scalar + # declarations. Activate their member's string family only + # after the initializer, for later reads and write targets. + self._current_func_local_types[node.name] = "std::string" if member_name != node.name: # The declaration RHS was emitted against the inherited # binding. Only now activate the collision-safe member for @@ -1152,6 +1157,7 @@ def remember_local_type(cpp_type: str | None) -> None: def _compound_assign_rhs(self, target_read: str, op: str, val_cpp: str, target_node=None, value_node=None, *, + target_cpp_type: str | None = None, target_is_double: bool = False, value_is_double: bool = False) -> str | None: """RHS for a compound assignment that must NOT lower to the C++ @@ -1161,10 +1167,18 @@ def _compound_assign_rhs(self, target_read: str, op: str, val_cpp: str, fmod-like — the binary-operator lowering in visit_expr casts both sides to double / uses std::fmod. ``a /= b`` and ``a %= b`` must match those semantics (C++ ``/=`` would do integer division on int - operands; ``%=`` does not even compile for doubles). Returns None - for operators where the native C++ compound form is correct - (+=, -=, *=). + operands; ``%=`` does not even compile for doubles). String ``+=`` + rejoins UTF-16 halves like string ``+``. The caller can supply its + emitted storage type when a requested binding differs from the chart + AST's type. Returns None when the native compound form is correct. """ + if (op == "+=" + and (target_cpp_type or self._concat_type(target_node)) == "std::string"): + # A bare na takes the string slot's type; other values keep the + # assignment writer's already-rendered RHS. + if self._is_na_expr(value_node): + val_cpp = "na()" + return self._string_concat_cpp(target_read, val_cpp) # Requested helper state already stores numeric values as doubles. # Its caller supplies the emitted operand families after converting # the RHS; the chart AST's int hint cannot type those series reads. diff --git a/pineforge_codegen/diagnostics_catalog.json b/pineforge_codegen/diagnostics_catalog.json index c6eef36f..9e5af129 100644 --- a/pineforge_codegen/diagnostics_catalog.json +++ b/pineforge_codegen/diagnostics_catalog.json @@ -236,6 +236,7 @@ "PF-E2091": {"severity": "error", "area": "analysis", "message": "plot() cannot have a linewidth below 1 in Pine v6: use a value of 1 or greater.", "hint": "Set the plot linewidth to an integer of at least 1.", "explanation": "A Pine v6 plot call cannot use an integer literal below one for linewidth; variable and calculated widths retain their existing handling.", "args": {}, "user_message": "plot() requires a linewidth of at least 1 in Pine v6."}, "PF-E2092": {"severity": "error", "area": "analysis", "message": "\"range\" cannot be used as a variable or function name in Pine v6: use another name.", "hint": "Rename the variable, function or method declaration.", "explanation": "Pine v6 reserves range as a variable and callable declaration name; parameters, user type fields and member access retain their existing handling.", "args": {}, "user_message": "Choose another variable, function or method name instead of range in Pine v6."}, "PF-E2095": {"severity": "error", "area": "analysis", "message": "Cannot call ''{function_name}'' with an argument of type {argument_type} for {parameter_type} parameter ''{name}''.", "hint": null, "explanation": "A non-string value cannot be used for a string parameter. Pass a string value of the declared parameter type; a permitted generic na adopts that type.", "args": {"function_name": {"kind": "identifier"}, "argument_type": {"kind": "type"}, "parameter_type": {"kind": "type"}, "name": {"kind": "identifier"}}, "user_message": "Argument ''{name}'' of ''{function_name}'' has type {argument_type}, but requires {parameter_type}."}, + "PF-E2094": {"severity": "error", "area": "analysis", "message": "str.substring() requires ''begin_pos'' to be greater than or equal to 0.", "hint": null, "explanation": "Pine v6 refuses a str.substring begin_pos proven to be a negative constant integer. Dynamic values retain runtime boundary checking, and missing positions retain their existing handling.", "args": {}, "user_message": "A constant begin_pos in str.substring() cannot be negative."}, "PF-E3001": {"severity": "error", "area": "request-security", "message": "request.security timeframe ''{name}'' cannot be resolved before the first bar: PineForge registers every requested timeframe before the script runs.", "hint": "Pass a literal, an input or a global to the helper that holds request.security.", "explanation": "The timeframe given to request.security is a name whose value PineForge cannot compute before the first bar; use a literal, an input or a global.", "args": {"name": {"kind": "identifier"}}, "user_message": "The timeframe given to request.security is a name whose value PineForge cannot compute before the first bar."}, "PF-E3002": {"severity": "error", "area": "request-security", "message": "request.security timeframe expression cannot be resolved before the first bar: PineForge registers every requested timeframe before the script runs.", "hint": "Pass a literal, an input or a global to the helper that holds request.security.", "explanation": "The timeframe expression given to request.security cannot be computed before the first bar; use a literal, an input or a global.", "args": {}, "user_message": "The timeframe expression given to request.security cannot be computed before the first bar."}, "PF-E3003": {"severity": "error", "area": "request-security", "message": "request.security timeframe switch arms must contain one expression", "hint": "Compute multi-statement timeframe logic before the request.security call or rewrite it as pure switch arms.", "explanation": "A switch that picks a request.security timeframe must have exactly one expression in each arm; compute multi-statement logic beforehand.", "args": {}, "user_message": "A switch that picks a request.security timeframe must have exactly one expression in each arm."}, diff --git a/pineforge_codegen/intrinsic_binding.py b/pineforge_codegen/intrinsic_binding.py index 780aaa92..ed03ba13 100644 --- a/pineforge_codegen/intrinsic_binding.py +++ b/pineforge_codegen/intrinsic_binding.py @@ -1,4 +1,4 @@ -"""Keyword validation and positional binding from intrinsic records.""" +"""Intrinsic binding and proven builtin argument-domain diagnostics.""" from __future__ import annotations @@ -7,6 +7,7 @@ from .errors import CompileError, Diagnostic, Level, Phase, SourceLocation from .pine_spelling import expr_start from .signatures import IntrinsicFunc, Param +from .symbols import TypeSpec _STRATEGY_ORDER_FUNCTIONS = frozenset({ @@ -15,6 +16,28 @@ }) +def array_method_domain_error( + method: str, receiver: TypeSpec | None, +) -> tuple[str, str | None] | None: + """Return a proven builtin receiver mismatch, leaving other domains open. + + Join independently admits string, int and float arrays and refuses bool + arrays. The bool refusal does not prove any other array domain invalid. + Callers resolve authored methods before applying this builtin contract. + """ + if (method != "join" or receiver is None or receiver.kind != "array" + or receiver.element is None or receiver.element.kind != "primitive" + or receiver.element.name != "bool"): + return None + return render_diagnostic( + "PF-E1088", { + "name": "array.join", + "message": "cannot accept an array argument for parameter 'id'", + "spelled": "CE10123", + }, + ) + + def intrinsic_keyword_diagnostics( intrinsic: IntrinsicFunc | None, kwargs: dict[str, ASTNode], diff --git a/pineforge_codegen/loop_types.py b/pineforge_codegen/loop_types.py index 85e5ddc5..65b01d9d 100644 --- a/pineforge_codegen/loop_types.py +++ b/pineforge_codegen/loop_types.py @@ -387,23 +387,30 @@ def array_result(name, spec): return None -class _NumericValueQuery(_CounterQuery): - """One value query for storage and counters, including integral proofs.""" +class IntegerConstants: + """Memoized immutable integer arithmetic, shared by value and argument checks. - _use_reassignment_values = True + Missing values and runtime expressions supply no proof. In particular, na + is never converted to the native integer sentinel while checking a value. + """ - def __init__(self, check_query): - super().__init__(check_query.sources, check_query.bindings, check_query.scopes) - self.check_query = check_query - self.integer_constants = {} + def __init__(self, sources, bindings, budget=None): + self.sources = sources + self.bindings = bindings + self.budget = budget + self.values = {} - def _integer_constant(self, node): + def value(self, node): """Prove an integer using only immutable literal arithmetic.""" + if node is None: + return None + if self.budget is not None: + self.budget.check(node.loc, Phase.ANALYZER) key = id(node) - if key in self.integer_constants: - return self.integer_constants[key] + if key in self.values: + return self.values[key] # Memoize failures too; a provisional failure breaks alias cycles. - self.integer_constants[key] = None + self.values[key] = None value = None if isinstance(node, NumberLiteral) and type(node.value) is int: value = node.value @@ -415,14 +422,14 @@ def _integer_constant(self, node): and declaration.name not in self.sources.reassigned_names and declaration.type_hint in (None, "int") and (declaration.annotations or {}).get("qualifier") in (None, "const")): - value = self._integer_constant(declaration.value) + value = self.value(declaration.value) elif isinstance(node, UnaryOp) and node.op in {"+", "-"}: - operand = self._integer_constant(node.operand) + operand = self.value(node.operand) if operand is not None: value = -operand if node.op == "-" else operand elif isinstance(node, BinOp) and node.op in {"+", "-", "*", "/", "%"}: - left = self._integer_constant(node.left) - right = self._integer_constant(node.right) + left = self.value(node.left) + right = self.value(node.right) if left is not None and right is not None: if node.op == "+": value = left + right @@ -436,9 +443,23 @@ def _integer_constant(self, node): elif node.op == "%": remainder = abs(left) % abs(right) value = -remainder if left < 0 else remainder - self.integer_constants[key] = value + self.values[key] = value return value + +class _NumericValueQuery(_CounterQuery): + """One value query for storage and counters, including integral proofs.""" + + _use_reassignment_values = True + + def __init__(self, check_query): + super().__init__(check_query.sources, check_query.bindings, check_query.scopes) + self.check_query = check_query + self._integer_constants = IntegerConstants(self.sources, self.bindings) + + def _integer_constant(self, node): + return self._integer_constants.value(node) + def infer(self, node): if isinstance(node, BinOp) and node.op == "/": return _INT if self._integer_constant(node) is not None else _FLOAT diff --git a/pineforge_codegen/signatures.py b/pineforge_codegen/signatures.py index 0f4483d5..7e48fb78 100644 --- a/pineforge_codegen/signatures.py +++ b/pineforge_codegen/signatures.py @@ -445,7 +445,7 @@ def _str(short_name: str, *sigs: FuncSig, _str("format", _sig([("formatString", S)], ret=S)) # variadic _str("format_time", _sig([("time", I), ("format", S), ("timezone", S, "")], ret=S), keyword_names_verified=True) -_str("length", _sig([("string", S)], ret=I)) +_str("length", _sig([("string", S)], ret=I), keyword_names_verified=True) _str("contains", _sig([("source", S), ("str", S)], ret=B)) _str("startswith", _sig([("source", S), ("str", S)], ret=B)) _str("endswith", _sig([("source", S), ("str", S)], ret=B)) diff --git a/scripts/gen_unicode_case_data.py b/scripts/gen_unicode_case_data.py new file mode 100644 index 00000000..4812a1c1 --- /dev/null +++ b/scripts/gen_unicode_case_data.py @@ -0,0 +1,319 @@ +"""Generate pinned casing tables offline from the three Unicode 14.0.0 inputs. + +Only explicitly named files are read. Input hashes and sizes are mandatory. +No host Unicode database, locale, network, environment or package import +supplies data. The output is an ASCII Python module included in the package. +Run this generator on an approved build/test venue, not the authoring host. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import re + + +DATA_VERSION = "14.0.0" +DATA_FORMAT = 1 +INPUT_PINS = ( + ("UnicodeData.txt", 1897793, + "36018e68657fdcb3485f636630ffe8c8532e01c977703d2803f5b89d6c5feafb"), + ("SpecialCasing.txt", 16830, + "c667b45908fd269af25fd55d2fc5bbc157fb1b77675936e25c513ce32e080334"), + ("DerivedCoreProperties.txt", 1040794, + "e3eddd7d469cd1b0feed7528defad1a1cc7c6a9ceb0ae4446a6d10921ed2e7bc"), +) +_CODE = re.compile(r"[0-9A-F]{4,6}", re.ASCII) +_LANGUAGE = re.compile(r"[a-z]{2,3}(?:[-_][A-Za-z0-9]+)*", re.ASCII) +_DOMAIN = 0x110000 + + +class DataError(ValueError): + """An input fails its pinned identity or the supported UCD format.""" + + +@dataclass(frozen=True) +class CaseData: + lower: tuple[tuple[int, ...], ...] + upper: tuple[tuple[int, ...], ...] + lower_final_sigma: tuple[tuple[int, ...], ...] + context: tuple[tuple[int, int, int], ...] + excluded_locale_rules: tuple[str, ...] + + +def _code(value: str) -> int: + if not _CODE.fullmatch(value): + raise DataError(f"invalid UCD code point: {value!r}") + result = int(value, 16) + if result >= _DOMAIN: + raise DataError(f"UCD code point outside Unicode: {value}") + return result + + +def _sequence(value: str) -> tuple[int, ...]: + return tuple(_code(part) for part in value.split()) + + +def _records(text: str): + for line_number, line in enumerate(text.split("\n"), 1): + body = line.split("#", 1)[0].strip(" \t\r") + if body: + if not body.isascii(): + raise DataError(f"non-ASCII UCD data field at line {line_number}") + yield line_number, [field.strip(" \t\r") for field in body.split(";")] + + +def _unicode_data(text: str, flags: bytearray): + lower: dict[int, tuple[int, ...]] = {} + upper: dict[int, tuple[int, ...]] = {} + previous = -1 + pending = None + + def apply(first: int, last: int, fields: list[str]) -> None: + if first != last and (fields[12] or fields[13]): + raise DataError("a UnicodeData range carries a simple case mapping") + if fields[2] == "Nd": + for point in range(first, last + 1): + flags[point] |= 4 + if fields[12]: + upper[first] = (_code(fields[12]),) + if fields[13]: + lower[first] = (_code(fields[13]),) + + for line, fields in _records(text): + if len(fields) != 15: + raise DataError(f"UnicodeData line {line}: expected 15 fields") + point = _code(fields[0]) + if point <= previous: + raise DataError(f"UnicodeData line {line}: non-increasing code points") + previous = point + name = fields[1] + if name.endswith(", First>"): + if pending is not None: + raise DataError(f"UnicodeData line {line}: nested range") + pending = (point, fields) + elif name.endswith(", Last>"): + if pending is None: + raise DataError(f"UnicodeData line {line}: range has no first row") + first, first_fields = pending + if (first_fields[1].removesuffix(", First>") + != name.removesuffix(", Last>") + or first_fields[2:] != fields[2:]): + raise DataError(f"UnicodeData line {line}: inconsistent range") + apply(first, point, first_fields) + pending = None + else: + if pending is not None: + raise DataError(f"UnicodeData line {line}: unterminated range") + apply(point, point, fields) + if pending is not None: + raise DataError("UnicodeData: unterminated range at end of input") + return lower, upper + + +def _special_casing(text: str, lower: dict, upper: dict): + final_lower: dict[int, tuple[int, ...]] = {} + unconditional_seen: set[int] = set() + excluded: list[str] = [] + for line, fields in _records(text): + if len(fields) == 6 and fields[-1] == "": + fields.pop() + if len(fields) != 5: + raise DataError(f"SpecialCasing line {line}: expected five fields") + point = _code(fields[0]) + lo, title, up = (_sequence(field) for field in fields[1:4]) + conditions = tuple(fields[4].split()) + if not conditions: + if point in unconditional_seen: + raise DataError(f"SpecialCasing line {line}: duplicate unconditional row") + unconditional_seen.add(point) + lower[point], upper[point] = lo, up + elif any(_LANGUAGE.fullmatch(token) for token in conditions): + # Root casing deliberately excludes every language-tailored row, + # including deletions. Keep an inventory in the generated module. + excluded.append(f"{point:04X}: {' '.join(conditions)}") + elif conditions == ("Final_Sigma",): + if point in final_lower: + raise DataError(f"SpecialCasing line {line}: duplicate contextual row") + if up != (point,) or title != (point,): + raise DataError("Final_Sigma has a nonidentity upper/title mapping") + final_lower[point] = lo + else: + raise DataError(f"SpecialCasing line {line}: unsupported root condition") + return final_lower, tuple(sorted(excluded)) + + +def _properties(text: str, flags: bytearray) -> None: + bits = {"Cased": 1, "Case_Ignorable": 2} + previous: dict[str, int] = {} + for line, fields in _records(text): + if len(fields) != 2: + raise DataError(f"DerivedCoreProperties line {line}: expected two fields") + prop = fields[1] + if prop not in bits: + continue + interval = fields[0].split("..") + if len(interval) not in (1, 2): + raise DataError(f"DerivedCoreProperties line {line}: invalid interval") + first, last = _code(interval[0]), _code(interval[-1]) + if first > last or first <= previous.get(prop, -1): + raise DataError(f"DerivedCoreProperties line {line}: unordered interval") + previous[prop] = last + for point in range(first, last + 1): + flags[point] |= bits[prop] + if set(previous) != set(bits): + raise DataError("DerivedCoreProperties: a required casing property is missing") + + +def _mapping_rows(mapping: dict[int, tuple[int, ...]]): + return tuple( + (point, len(converted), *converted) + for point, converted in sorted(mapping.items()) + if converted != (point,) + ) + + +def _context_ranges(flags: bytearray): + result = [] + first = 0 + while first < len(flags): + value = flags[first] + last = first + while last + 1 < len(flags) and flags[last + 1] == value: + last += 1 + if value: + result.append((first, last, value)) + first = last + 1 + return tuple(result) + + +def parse_ucd(unicode_data: str, special_casing: str, derived_properties: str) -> CaseData: + """Pure parser; production CLI calls this only after all three hash checks.""" + flags = bytearray(_DOMAIN) + lower, upper = _unicode_data(unicode_data, flags) + final_lower, excluded = _special_casing(special_casing, lower, upper) + _properties(derived_properties, flags) + data = CaseData( + _mapping_rows(lower), _mapping_rows(upper), + _mapping_rows(final_lower), _context_ranges(flags), excluded, + ) + if not all((data.lower, data.upper, data.lower_final_sigma, data.context)): + raise DataError("a required casing table is empty") + return data + + +def _verified_text(name: str, payload: bytes) -> str: + pin = next((item for item in INPUT_PINS if item[0] == name), None) + if pin is None: + raise DataError(f"input is not in the pinned UCD set: {name}") + _, size, digest = pin + if len(payload) != size or hashlib.sha256(payload).hexdigest() != digest: + raise DataError(f"pinned size/SHA-256 mismatch: {name}") + text = payload.decode("utf-8") + if name != "UnicodeData.txt": + stem = name.removesuffix(".txt") + if not text.startswith(f"# {stem}-{DATA_VERSION}.txt\n"): + raise DataError(f"pinned version header mismatch: {name}") + return text + + +def render_module(data: CaseData) -> bytes: + """Deterministic ASCII source: no dates, paths, host versions or locale.""" + lines = [ + '"""Generated Unicode 14.0.0 casing data; do not edit table entries.', + "", + "Source: https://www.unicode.org/Public/14.0.0/ucd/", + "Copyright 2021 Unicode, Inc.", + "Unicode data terms: https://www.unicode.org/terms_of_use.html", + 'Regenerate with scripts/gen_unicode_case_data.py using pinned inputs."""', + "", + f"DATA_FORMAT = {DATA_FORMAT}", + f'DATA_VERSION = "{DATA_VERSION}"', + "SOURCE_SHA256 = (", + ] + lines.extend(f' ("{name}", "{digest}"),' for name, _, digest in INPUT_PINS) + lines.extend((")", "SOURCE_BYTE_COUNTS = (")) + lines.extend(f' ("{name}", {size}),' for name, size, _ in INPUT_PINS) + lines.extend((")", "EXCLUDED_LOCALE_RULES = (")) + lines.extend(f' "{rule}",' for rule in data.excluded_locale_rules) + lines.append(")") + for name, rows in ( + ("LOWER_ROWS", data.lower), + ("UPPER_ROWS", data.upper), + ("LOWER_FINAL_SIGMA_ROWS", data.lower_final_sigma), + ("CONTEXT_RANGES", data.context), + ): + lines.extend(("", f"{name} = (")) + lines.extend(" (" + ", ".join(f"0x{value:X}" for value in row) + ")," + for row in rows) + lines.append(")") + return ("\n".join(lines) + "\n").encode("ascii") + + +def _checked_path(value: str, filename: str) -> Path: + """Reject sensitive names before any filesystem query; reject symlinks.""" + path = Path(value) + parts = path.parts + if any((part.startswith(".") and part not in {".", ".."}) + or part.endswith(".env") for part in parts): + raise DataError("refused protected file path by name") + if not path.is_absolute() or ".." in parts or path.name != filename: + raise DataError(f"expected an absolute path ending in {filename}") + for component in (*reversed(path.parents), path): + if component.is_symlink(): + raise DataError(f"refused symlink path: {component}") + return path + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--unicode-data") + parser.add_argument("--special-casing") + parser.add_argument("--derived-properties") + parser.add_argument("--output") + parser.add_argument("--check", action="store_true", help="compare an existing output") + parser.add_argument("--crash-control", action="store_true") + args = parser.parse_args(argv) + if args.crash_control: + print('{"read_files": [], "write_files": []}', flush=True) + raise RuntimeError("deliberate Unicode generator crash control") + values = (args.unicode_data, args.special_casing, args.derived_properties) + if not all(values) or not args.output: + parser.error("all three explicit UCD inputs and --output are required") + # Announce the exact follow set before checking paths or opening a file. + print(json.dumps({ + "read_files": list(values) + ([args.output] if args.check else []), + "write_files": [] if args.check else [args.output], + }, sort_keys=True), flush=True) + paths = [_checked_path(value, pin[0]) for value, pin in zip(values, INPUT_PINS)] + output = _checked_path(args.output, "_unicode_case_data.py") + texts = [] + for path, (name, _, _) in zip(paths, INPUT_PINS): + texts.append(_verified_text(name, path.read_bytes())) + data = parse_ucd(*texts) + encoded = render_module(data) + if args.check: + if output.read_bytes() != encoded: + raise DataError("generated casing module differs from pinned inputs") + else: + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("xb") as handle: + if handle.write(encoded) != len(encoded): + raise OSError("short casing data write") + print(json.dumps({ + "output": str(output), + "sha256": hashlib.sha256(encoded).hexdigest(), + "bytes": len(encoded), + "lower_rows": len(data.lower), + "upper_rows": len(data.upper), + "context_ranges": len(data.context), + "final_sigma_rows": len(data.lower_final_sigma), + }, sort_keys=True), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/_older_codegen.py b/tests/_older_codegen.py index e123ba47..37ba7323 100644 --- a/tests/_older_codegen.py +++ b/tests/_older_codegen.py @@ -73,9 +73,13 @@ def with_utf16_units(cpp: str) -> str: The older emission counted ``str.length`` through ``(int)text.length()`` and carried no unit helpers. The calls become calls of the helper and the helper block is added before the syminfo helpers, where the current emitter puts - it. A text without such a call is returned as it is.""" + it. A repaired string concatenation also needs the block. A text that + already carries the current block is not given a second copy.""" spelled, sites = _spell_lengths(cpp) - if not sites: + if not sites and "_pf_u16_concat(" not in spelled: return cpp + if "inline int _pf_u16_length(" in spelled: + assert TV_STRING_UNITS_CPP in spelled, spelled + return spelled assert spelled.count(_SYMINFO_HELPERS) == 1, spelled return spelled.replace(_SYMINFO_HELPERS, TV_STRING_UNITS_CPP + "\n\n" + _SYMINFO_HELPERS) diff --git a/tests/fixtures/diagnostic_codes_pin.json b/tests/fixtures/diagnostic_codes_pin.json index a59e136b..4b7a21f4 100644 --- a/tests/fixtures/diagnostic_codes_pin.json +++ b/tests/fixtures/diagnostic_codes_pin.json @@ -237,6 +237,7 @@ "PF-E2091": "2d1b04fd25a4198619628b94bf8f90b4002f33e4081c2b780fea510c49e0b44a", "PF-E2092": "74b483fe343d239ec254f03e839b257611db5295b0433eabd15072f919950f3d", "PF-E2095": "a2bf71e6ab48d39b72365b5a69960a8622c7b38deb0aace009215b46dc3cb89d", + "PF-E2094": "b693cc3415fcd4b4738b511967e6cfb2b88fd4aa8bba6f498f0c850bfc32485d", "PF-E3001": "585dad07e797bbe2cfffa9f45e14dee040be5570cc13f8e46fa4c31e953fc92a", "PF-E3002": "db089893825afe5ad1d91934c21107fc58451cadf3d71de2539b9c5805549433", "PF-E3003": "02571beee5163d1596d8a11fe49336dfcf65f86d4fed13254bac6b36023c3c79", diff --git a/tests/fixtures/string_pos_na/README.md b/tests/fixtures/string_pos_na/README.md new file mode 100644 index 00000000..d96a76c1 --- /dev/null +++ b/tests/fixtures/string_pos_na/README.md @@ -0,0 +1,143 @@ +# String position and boundary fixture + +`strategy.pine` checks `str.pos` when a needle is absent, found, empty, or +longer than its text. It checks missing results through `na()` and `nz()`, +using both literal arguments and arguments that change on alternate bars. + +The public source is the reporter's observations in +[issue #382](https://github.com/pineforge-4pass/pineforge-codegen-oss/issues/382): +item 5 for positions, item 9 for substring boundaries, and item 8 for trimming. +**`strategy.pine` has no captured TradingView trade tape.** Its expected report +is derived from those public observations, not a new TradingView capture. +Related substring and trim cases live in `tests/test_string_units.py`; the +strategy itself checks positions only. + +## Expected position values + +| Expression | Expected result | +| --- | --- | +| `str.pos("abc", "x")` | integer `na` | +| `na(str.pos("abc", "x"))` | `true` | +| `nz(str.pos("abc", "x"), 0)` | `0` | +| `str.tostring(str.pos("abc", "x"))` | `"NaN"`: generated-string contract, without a separate reported TradingView measurement | + +The related native tests include these runtime inputs. The Unicode positions +are derived from UTF-16 units, where an emoji occupies two units; they are not +separate TradingView measurements of these exact inputs. The `"NaN"` entries +all use the generated-string contract above. + +| Text | Needle | `na(p)` | `nz(p, 0)` | `str.tostring(p)` | +| --- | --- | --- | ---: | --- | +| `"abc"` | `"x"` | `true` | 0 | `"NaN"` | +| `"seed42"` | `"?"` | `true` | 0 | `"NaN"` | +| `"abc"` | `"b"` | `false` | 1 | `"1"` | +| `"abc"` | `"a"` | `false` | 0 | `"0"` | +| `"abc"` | `""` | `false` | 0 | `"0"` | +| `""` | `"x"` | `true` | 0 | `"NaN"` | +| `""` | `""` | `false` | 0 | `"0"` | +| `"ab"` | `"abc"` | `true` | 0 | `"NaN"` | +| `"a😀b"` | `"b"` | `false` | 3 | `"3"` | +| `"中😀tail"` | `"l"` | `false` | 6 | `"6"` | + +The strategy opens a position on its first bar and closes it on its second. +The exact expected exit-comment report is: + +```text +literal_na=true|literal_nz=0|variable_na=true|variable_nz=0|long_na=true|long_nz=0|start=0|found=1|empty=0 +``` + +For comparison, the report derived from affected-version behavior is: + +```text +literal_na=false|literal_nz=-1|variable_na=false|variable_nz=-1|long_na=false|long_nz=-1|start=0|found=1|empty=0 +``` + +## Substring observations and controls + +| Form | Expected result | +| --- | --- | +| `str.substring("abc", str.pos("abc", ":") + 1)` | `"abc"`, completed run | +| Substring with `na` start and omitted end | Whole text, completed run | +| Substring with `na` end | String `na`, completed run | +| `str.substring("abc", 2, j)` with finite runtime `j = 1` | Run-time stop; detail `end_pos is less than begin_pos` | + +Item 9 reports undefined affected-version behavior that depends on platform +and compiled context. With runtime positions, releases 1.10.0 and 1.11.0 +produced: + +| Form on `"abc"` | Affected-version x86-64 output | Affected-version ARM64 output | TradingView observation | +| --- | --- | --- | --- | +| Start 2, end 1 | `"c"` | Empty text | Run-time error | +| Start -1, omitted end | Run-time error | `"abc"` | Run-time error | +| Start 0, end `na` | `"abc"` | Empty text, reported as `na()` | String `na` | +| Start `na`, omitted end | `"abc"` | `"abc"` | `"abc"` | + +The affected-version results involving invalid or missing positions are not +guaranteed behavior. In particular, matching a TradingView value through an +undefined conversion does not make the result reliable. + +Finite valid slices must also complete successfully: + +| Expression | Expected result | +| --- | --- | +| `str.substring("abc", 1, 2)` | `"b"`, UTF-8 hex `62` | +| `str.substring("a😀b", 1, 3)` | Whole emoji, UTF-8 hex `f09f9880` | +| `str.substring("abc", 1, 9)` | Clamped `"bc"`, length 2 | +| `str.substring("abc", 3, 3)` | Empty text, length 0 | + +The native representation stores string `na` as an empty string, so it cannot +distinguish a missing string from an authored empty string. + +## Trim observations + +Item 8 reports these exact TradingView results: + +| Text passed to `str.trim` | Expected result | +| --- | --- | +| U+00A0, `a`, U+00A0 | `"a"` | +| U+2003, `a`, U+2003 | `"a"` | +| U+FEFF, `a` | `"a"` | +| Vertical tab U+000B, `a`, form feed U+000C | `"a"` | + +Reported as removed at the ends: no-break space U+00A0, em space U+2003, +ideographic space U+3000, Ogham space mark U+1680, line separator U+2028, +zero-width no-break space U+FEFF, vertical tab U+000B, and form feed U+000C, +in addition to space, tab, line feed, and carriage return. + +Reported as kept: next-line U+0085, unit separator U+001F, zero-width space +U+200B, and Mongolian vowel separator U+180E. Interior characters are kept. +The tests compose these characters into literal and runtime-input cases, +including mixed boundaries and empty text; those compositions are derived +contracts, not separate TradingView measurements or a full Unicode whitespace +inventory. + +## How to run + +Open `strategy.pine` in TradingView's Pine Editor and add it to a chart with +at least two bars. In the strategy's trade list, inspect the closing order's +comment and compare it with the expected report above. + +To run the related native checks, configure the engine headers, generated +headers, built runtime, Eigen, and compiler as described in the repository +setup, then run: + +```sh +PINEFORGE_REQUIRE_NATIVE=1 python -m pytest -q -ra tests/test_string_units.py +``` + +`PINEFORGE_REQUIRE_NATIVE=1` makes missing native inputs fail instead of skip. + +## Unmeasured limits + +- A substring with an `na` start and an explicit finite end has no reported + TradingView observation or native case here. +- The variable idiom `p = str.pos(s, ":")` followed by + `str.substring(s, p + 1)` has no native case here. +- Recorded native runs used ARM64. Finite slicing under UBSan, a sanitizer + control demonstrating the old overflow, a sanitized runtime archive, and + native behavior on x86-64 have not been established by these checks. The + existing four UBSan cases return at the `na`-end guard before finite arithmetic. +- Boundary and consumer cases use `-O0` and `-O2`; the original position cases + use `-O0`. Broader `-O2` coverage, `request.security`, array-index and plot + compositions, surrogate-half search, and other issue #382 string functions + are outside this fixture's coverage. diff --git a/tests/fixtures/string_pos_na/strategy.pine b/tests/fixtures/string_pos_na/strategy.pine new file mode 100644 index 00000000..21718cf6 --- /dev/null +++ b/tests/fixtures/string_pos_na/strategy.pine @@ -0,0 +1,35 @@ +//@version=6 +// Issue #382 item 5. Candidate public parity probe; no captured TV tape yet. +strategy("382 str.pos missing match", overlay=true, process_orders_on_close=true) + +// The literal forms are the public expressions reported in the issue packet. +literalNa = na(str.pos("abc", "x")) +literalNz = nz(str.pos("abc", "x"), 0) + +// Both arguments vary at run time; neither possible needle occurs in its source. +source = bar_index % 2 == 0 ? "abc" : "ab" +needle = bar_index % 2 == 0 ? "x" : "z" +variablePos = str.pos(source, needle) +variableNa = na(variablePos) +variableNz = nz(variablePos, 0) + +longPos = str.pos("abc", "abcdef") +longNa = na(longPos) +longNz = nz(longPos, 0) + +// ASCII controls: a match at zero, another found position and an empty needle. +startPos = str.pos("abc", "a") +foundPos = str.pos("abc", "b") +emptyPos = str.pos("abc", "") + +report = "literal_na=" + str.tostring(literalNa) + "|literal_nz=" + str.tostring(literalNz) +report += "|variable_na=" + str.tostring(variableNa) + "|variable_nz=" + str.tostring(variableNz) +report += "|long_na=" + str.tostring(longNa) + "|long_nz=" + str.tostring(longNz) +report += "|start=" + str.tostring(startPos) + "|found=" + str.tostring(foundPos) + "|empty=" + str.tostring(emptyPos) + +if bar_index == 0 + strategy.entry("probe", strategy.long, qty=1) +if bar_index == 1 + strategy.close("probe", comment=report) + +plot(close) diff --git a/tests/test_array_join_domains.py b/tests/test_array_join_domains.py new file mode 100644 index 00000000..879cae05 --- /dev/null +++ b/tests/test_array_join_domains.py @@ -0,0 +1,143 @@ +"""Builtin join's proven receiver mismatch, independent of C++ storage. + +The retained compiler response refuses array. String, int and float +have independent admissions; other domains stay open rather than inheriting +an exclusive array whitelist from that one overload diagnostic. +""" + +import pytest + +from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.ast_nodes import Identifier +from pineforge_codegen.codegen import CodeGen +from pineforge_codegen.errors import CompileError, Level, Phase, SourceLocation +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser +from pineforge_codegen.symbols import PineType, TypeSpec + + +HEADER = '//@version=6\nstrategy("join domains")\n' +FILENAME = "join_domains.pine" + + +def _analyze(body): + source = HEADER + body + program = Parser( + Lexer(source, filename=FILENAME).tokenize(), + source=source, filename=FILENAME, + ).parse() + return Analyzer(program, filename=FILENAME).analyze() + + +def _refusal(body, *, analyze_only=False): + with pytest.raises(CompileError) as caught: + if analyze_only: + _analyze(body) + else: + transpile(HEADER + body, filename=FILENAME) + diagnostic = next(d for d in caught.value.diagnostics if d.level == Level.ERROR) + assert diagnostic.code == "PF-E1088" + assert diagnostic.args["name"] == "array.join" + assert "array" in diagnostic.message + assert diagnostic.args["spelled"] == "CE10123" + assert diagnostic.hint == "Use the parameters of TradingView's signature." + return diagnostic + + +@pytest.mark.parametrize("initializer", [ + "array.from(true, false)", + "array.new_bool(0)", +]) +@pytest.mark.parametrize("call", [ + 'array.join(items, "|")', + 'array.join(id=items, separator="|")', + 'array.join(separator="|", id=items)', + 'array.join(items, separator="|")', + "array.join(items)", + 'items.join("|")', + 'items.join(separator="|")', + "items.join()", +]) +def test_bool_domain_refusal_is_located_on_the_receiver(initializer, call): + statement = f"observed = {call}" + diagnostic = _refusal(f"items = {initializer}\n{statement}\n") + column = statement.index("items") + 1 + assert diagnostic.phase == Phase.ANALYZER + assert diagnostic.location == SourceLocation( + FILENAME, 4, column, column + len("items"), + ) + + +@pytest.mark.parametrize("body", [ + 'observed = array.join(array.from(true, false), "|")\n', + "render_flags(array flags) => flags.join()\n", + "items = array.from(true, false)\nplot(str.length(items.join()))\n", + "items = array.from(true, false)\nplot(str.length(string=items.join()))\n", + "items = array.from(true, false)\nif false\n observed = items.join()\n", +]) +def test_analyzer_checks_join_domain_before_ignored_calls_can_disappear(body): + assert _refusal(body, analyze_only=True).phase == Phase.ANALYZER + + +@pytest.mark.parametrize("initializer", [ + 'array.from("alpha", "beta")', + "array.from(1, -2)", + "array.from(1.25, -2.5)", + "array.new(0)", + "array.new(0)", +]) +def test_domain_guard_does_not_refuse_other_element_types(initializer): + # Analyzer-only scope for color/UDT: this does not assert native compile + # support or a TradingView value for either unmeasured domain. + context = _analyze( + "type JoinItem\n int value = 0\n" + f'items = {initializer}\nobserved = array.join(items, "|")\n' + ) + assert context.symbols.resolve("observed").pine_type == PineType.STRING + + +def test_namespace_receiver_type_query_does_not_register_ta_again(): + source = HEADER + 'observed = array.join(array.from(ta.change(close)), "|")\n' + program = Parser(Lexer(source).tokenize(), source=source).parse() + analyzer = Analyzer(program) + analyzer.analyze() + before = len(analyzer._ta_call_sites) + assert before > 0 + join_call = program.body[-1].value + assert analyzer._fixed_array_call_result(join_call, None) == TypeSpec.primitive("string") + assert len(analyzer._ta_call_sites) == before + + +AUTHORED_JOIN = ( + 'method join(array flags, string separator = ";") =>\n' + " str.length(str.repeat(separator, array.size(flags)))\n" +) + + +def test_authored_bool_join_keeps_its_declared_default(): + body = AUTHORED_JOIN + "items = array.from(true, false)\nobserved = items.join()\n" + assert _analyze(body).symbols.resolve("observed").pine_type == PineType.INT + cpp = transpile(HEADER + body, filename=FILENAME) + assert 'std::string(";")' in cpp + assert "_pf_join_parts" not in cpp + + +def test_authored_method_does_not_override_namespace_join_domain(): + body = AUTHORED_JOIN + 'items = array.from(true, false)\nobserved = array.join(items, "|")\n' + assert _refusal(body).phase == Phase.ANALYZER + + +def test_codegen_checks_join_domain_when_analysis_is_bypassed(): + generator = CodeGen(_analyze("plot(close)\n")) + location = SourceLocation(FILENAME, 7, 9, 14) + receiver = Identifier(name="items", loc=location) + with pytest.raises(CompileError) as caught: + generator._array_method_expr( + "items", "join", ['std::string("|")'], + TypeSpec.array(TypeSpec.primitive("bool")), receiver, + ) + diagnostic, = caught.value.diagnostics + assert diagnostic.code == "PF-E1088" + assert diagnostic.phase == Phase.CODEGEN + assert diagnostic.location == location diff --git a/tests/test_array_terminal_returns.py b/tests/test_array_terminal_returns.py index df220933..eb492d6d 100644 --- a/tests/test_array_terminal_returns.py +++ b/tests/test_array_terminal_returns.py @@ -1,8 +1,9 @@ -"""Exact primitive UDF return typing for direct terminal array ``get`` calls.""" +"""Primitive UDF return typing for terminal array calls and forwarding.""" from __future__ import annotations from itertools import product +import re import pytest @@ -15,6 +16,147 @@ from tests._legacy_cpp import without_source_rename +@pytest.mark.parametrize( + "expression, separator_cpp", + [ + pytest.param("values.join(separator)", "separator", id="method-positional"), + pytest.param( + "values.join(separator=separator)", "separator", id="method-named", + ), + pytest.param("values.join()", 'std::string("")', id="method-default"), + pytest.param( + "array.join(values, separator)", "separator", id="namespace-positional", + ), + pytest.param( + "array.join(values, separator=separator)", "separator", + id="namespace-mixed", + ), + pytest.param( + "array.join(id=values, separator=separator)", "separator", + id="namespace-named", + ), + pytest.param( + "array.join(separator=separator, id=values)", "separator", + id="namespace-reordered", + ), + pytest.param( + "values.copy().join(separator)", "separator", id="temporary-receiver", + ), + pytest.param( + "array.join(values)", 'std::string("")', id="namespace-default", + ), + pytest.param( + "array.join(id=values)", 'std::string("")', id="namespace-named-default", + ), + ], +) +@pytest.mark.parametrize( + "element_type, elements", + [("string", '"alpha", "beta"'), ("int", "2, 7")], +) +def test_join_helpers_and_forwarded_results_keep_string_signatures( + expression, separator_cpp, element_type, elements, +): + source = f'''//@version=6 +strategy("array join return types") +combine(array<{element_type}> values, string separator) => + {expression} +relay(array<{element_type}> values, string separator) => + combine(values=values, separator=separator) +parts = array.from({elements}) +first_result = relay(parts, "|") +second_result = relay(separator=":", values=parts) +plot(close) +''' + cpp = transpile(source) + for name in ("combine", "relay"): + assert f"std::string {name}(" in cpp, cpp + for name in ("first_result", "second_result"): + assert re.search(rf"\bstd::string {name}\s*[=;]", cpp), cpp + # A string signature alone still compiles when a named join emits + # ``return 0`` (a null C string), or silently takes the default separator. + assert f"const std::string _pf_join_separator = ({separator_cpp});" in cpp, cpp + if element_type == "string": + assert "_pf_u16_join(_pf_join_parts, _pf_join_separator)" in cpp, cpp + else: + assert "if (_pf_join_i != 0) _pf_join_result += _pf_join_separator;" in cpp, cpp + assert "_pf_join_result += _pf_join_number(_pf_join_parts[_pf_join_i]);" in cpp, cpp + assert "return _pf_u16_normalize(_pf_join_result);" in cpp, cpp + compile_cpp(cpp, label="array-join-return-types") + + +@pytest.mark.parametrize( + "expression", + [ + pytest.param( + "array.join(make_parts(), make_separator())", id="namespace-positional", + ), + pytest.param( + "array.join(make_parts(), separator=make_separator())", + id="namespace-mixed", + ), + pytest.param( + "array.join(id=make_parts(), separator=make_separator())", + id="namespace-named", + ), + pytest.param( + "array.join(separator=make_separator(), id=make_parts())", + id="namespace-reordered", + ), + pytest.param( + "make_parts().join(make_separator())", id="method-positional", + ), + pytest.param( + "make_parts().join(separator=make_separator())", id="method-named", + ), + ], +) +def test_join_argument_emission_binds_receiver_then_separator_once(expression): + source = f'''//@version=6 +strategy("array join argument bindings") +events = array.new_int() +make_parts() => + events.push(1) + array.from("alpha", "beta") +make_separator() => + events.push(2) + "|" +result = {expression} +plot(close) +''' + cpp = transpile(source) + assignment, = ( + line for line in cpp.splitlines() + if line.strip().startswith("result =") + ) + assert assignment.count("make_parts()") == 1, assignment + assert assignment.count("make_separator()") == 1, assignment + assert assignment.index("make_parts()") < assignment.index("make_separator()") + assert "const auto& _pf_join_parts = (" in assignment, assignment + assert "const std::string _pf_join_separator = (make_separator());" in assignment, assignment + assert "_pf_u16_join(_pf_join_parts, _pf_join_separator)" in assignment, assignment + compile_cpp(cpp, label="array-join-argument-bindings") + + +def test_join_result_keeps_string_type_through_local_and_method_returns(): + source = '''//@version=6 +strategy("array join local and method returns") +method combine(array values, string separator) => + joined_value = values.join(separator) + joined_value +relay(array values, string separator) => + values.combine(separator) +parts = array.from("alpha", "beta") +result = relay(parts, "|") +plot(close) +''' + cpp = transpile(source) + assert re.search(r"\bstd::string _udt_array_string_+combine\(", cpp), cpp + assert "std::string relay(" in cpp + assert re.search(r"\bstd::string result\s*[=;]", cpp), cpp + compile_cpp(cpp, label="array-join-local-method-return") + + _GET_SHAPES_SOURCE = r'''//@version=6 strategy("Terminal array get shapes") diff --git a/tests/test_binop_operand_order.py b/tests/test_binop_operand_order.py index 68327fee..8d9393bf 100644 --- a/tests/test_binop_operand_order.py +++ b/tests/test_binop_operand_order.py @@ -47,20 +47,22 @@ def test_mutating_operand_is_evaluated_after_the_left_operand_is_bound(): s6 = _assignment(cpp, "s6") bound = _BINDING.findall(s6) # The + that adds str.tostring(a.remove(-1)) and the + that adds - # array.join(a, ","); the + of a literal "," and the effect-free + of the - # two reads before them keep their spelling. + # array.join(a, ","); literal separators and the effect-free join of the + # two reads before them need no operand-order binding. assert len(bound) == 2, s6 # The inner binding's right operand comes first in the text. before_remove, before_join = sorted( - s6.index(f"return ({token} + ") for token in bound + s6.index(f"return _pf_u16_concat({token}, ") for token in bound ) # a.get(-1) is read into the inner binding, then a.remove(-1) erases. assert s6.index("return (__pf_array[(size_t)__pf_array_index]);") < before_remove assert s6.index(".erase(") > before_remove # array.join(a, ",") reads the array after the outer binding. - assert s6.index("std::to_string(a[") > before_join - # Operands no other operand changes keep the plain C++ operator: s4's - # two reads, and every entry id's concatenation. + join_receiver = s6.index("const auto& _pf_join_parts = (a);") + assert join_receiver > before_join + assert s6.index("_pf_join_number(_pf_join_parts[_pf_join_i])") > join_receiver + # Operands no other operand changes need no order binding: s4's two + # reads, and every entry id's concatenation. assert "__pf_binop_lhs_" not in _assignment(cpp, "s4") assert cpp.count("__pf_binop_lhs_") == s6.count("__pf_binop_lhs_") @@ -111,7 +113,9 @@ def test_user_calls_methods_and_payloads_bind_their_left_operand(): ): line = _assignment(cpp, target) (token,) = _BINDING.findall(line) - assert (line.index(bound) < line.index(f"return ({token} ") + returned = (f"return ({token} " if target == "through_call" + else f"return _pf_u16_concat({token}, ") + assert (line.index(bound) < line.index(returned) < line.index(after)), line # A bool operand is bound as a bool: a std::vector element is a # proxy of the bit, which the remove moves (a bound proxy stays right only @@ -123,7 +127,7 @@ def test_user_calls_methods_and_payloads_bind_their_left_operand(): # A request.security payload binds its left operand the same way. payload = _assignment(cpp, "_req_sec_0") (token,) = _BINDING.findall(payload) - assert payload.index(".erase(") > payload.index(f"return ({token} + ") + assert payload.index(".erase(") > payload.index(f"return _pf_u16_concat({token}, ") def test_operands_are_evaluated_left_to_right_at_run_time(): diff --git a/tests/test_builtin_name_shadow.py b/tests/test_builtin_name_shadow.py index 2835f73c..276754a7 100644 --- a/tests/test_builtin_name_shadow.py +++ b/tests/test_builtin_name_shadow.py @@ -6,7 +6,12 @@ of the builtin itself keeps its emitted text. """ +import re + +import pytest + from pineforge_codegen import transpile +from pineforge_codegen.codegen.tables import BAR_BUILTINS, BAR_FIELDS HEADER = '//@version=6\nstrategy("builtin name shadow", initial_capital = 100000)\n' @@ -36,3 +41,109 @@ def test_read_of_the_builtin_variable_keeps_its_emitted_text(): cpp = transpile(HEADER + "float s = second + minute\nplot(s)\n") assert "pine_second(current_bar_.timestamp, syminfo_.timezone)" in cpp assert "pine_minute(current_bar_.timestamp, syminfo_.timezone)" in cpp + + +@pytest.mark.parametrize("name", sorted(BAR_FIELDS.keys() | BAR_BUILTINS.keys())) +def test_string_binding_types_repeated_constructor_and_scalar_operands(name): + cpp = transpile( + HEADER + + 'parts = str.split("first|second", "|")\n' + + f"{name} = array.get(parts, bar_index % 2)\n" + + f"values = array.from({name}, {name})\n" + + f'joined = {name} + ":"\n' + + "plot(1)\n" + ) + # Both elements must read the inferred string declaration, including + # builtin names whose C++ spelling needs escaping. + initializer = re.search( + r"values = std::vector\{([A-Za-z_]\w*), \1\};", + cpp, + ) + assert initializer is not None, cpp + binding = initializer.group(1) + assert f"std::string {binding}" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("joined = ") + ] + assert len(assignments) == 1 + assert binding in assignments[0] + # Correct spelling with a numeric inferred type would still emit an + # invalid std::to_string call around the string operand. + assert "std::to_string(" not in assignments[0] + + +@pytest.mark.parametrize("name, builtin", sorted(BAR_FIELDS.items())) +def test_bar_read_before_global_declaration_keeps_its_source_binding(name, builtin): + cpp = transpile( + HEADER + + f"before = {name}\n" + + f"{name} = str.tostring(bar_index)\n" + + f'after = {name} + ":"\n' + + "plot(before)\n" + ) + assert f"before = {builtin};" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("after = ") + ] + assert len(assignments) == 1 + assert builtin not in assignments[0] + assert "std::to_string(" not in assignments[0] + + +@pytest.mark.parametrize("name, builtin", sorted(BAR_FIELDS.items())) +@pytest.mark.parametrize("block", ["if", "while", "switch"]) +def test_bar_builtin_returns_after_a_string_block_binding_expires(name, builtin, block): + body = ( + f" {name} = str.tostring(bar_index)\n" + f' seen := {name} + ":"\n' + ) + if block == "switch": + branch = "switch\n bar_index >= 0 =>\n" + "".join( + " " + line for line in body.splitlines(keepends=True) + ) + elif block == "while": + branch = "while bar_index >= 0\n" + body + " break\n" + else: + branch = "if bar_index >= 0\n" + body + cpp = transpile( + HEADER + + f"before = {name}\n" + + 'seen = ""\n' + + branch + + f"after = {name}\n" + + "plot(before + after)\n" + ) + assert "double before" in cpp + assert "double after" in cpp + assert f"before = {builtin};" in cpp + assert f"after = {builtin};" in cpp + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("seen = ") and 'std::string("")' not in line + ] + assert len(assignments) == 1 + assert builtin not in assignments[0] + assert "std::to_string(" not in assignments[0] + + +def test_qualified_member_and_other_bar_builtin_keep_their_bindings(): + cpp = transpile( + HEADER + + "type Holder\n string high\n" + + 'holder = Holder.new("member")\n' + + 'parts = str.split("first|second", "|")\n' + + "high = array.get(parts, bar_index % 2)\n" + + "seen = holder.high + high\n" + + "builtin = low\nplot(builtin)\n" + ) + assignments = [ + line.strip() for line in cpp.splitlines() + if line.strip().startswith("seen = ") + ] + assert len(assignments) == 1 + assert ".high" in assignments[0] + assert "current_bar_." not in assignments[0] + assert "std::to_string(" not in assignments[0] + assert "builtin = current_bar_.low;" in cpp diff --git a/tests/test_codegen_audit_fixes.py b/tests/test_codegen_audit_fixes.py index c9498803..5adc6a94 100644 --- a/tests/test_codegen_audit_fixes.py +++ b/tests/test_codegen_audit_fixes.py @@ -297,21 +297,73 @@ def test_array_variance_biased_arg(): # B16/B17 — array.join / copy / slice honor the element type # --------------------------------------------------------------------------- -def test_array_join_string_elements(): +JOIN_CALLS = [ + pytest.param('array.join(items, "|")', 'std::string("|")', id="namespace-positional"), + pytest.param('array.join(id=items, separator="|")', 'std::string("|")', + id="namespace-named"), + pytest.param('array.join(separator="|", id=items)', 'std::string("|")', + id="namespace-reordered"), + pytest.param('array.join(items, separator="|")', 'std::string("|")', + id="namespace-mixed"), + pytest.param('array.join(items)', 'std::string("")', id="namespace-default"), + pytest.param('items.join("|")', 'std::string("|")', id="method-positional"), + pytest.param('items.join(separator="|")', 'std::string("|")', id="method-named"), + pytest.param('items.join()', 'std::string("")', id="method-default"), +] + + +@pytest.mark.parametrize("call, separator", JOIN_CALLS) +def test_array_join_string_elements(call, separator): + # Independent TV string sources admit all eight forms. In particular, + # omitted namespace/method separators produce "1.25-2.5", not a comma. cpp = _gen( - 'sa = array.new(0)\narray.push(sa, "a")\n' - 's = array.join(sa, ",")\nplot(close)\n' + 'items = array.from("1.25", "-2.5")\n' + f's = {call}\nplot(close)\n' ) - assert "__pf_r+=sa[__pf_i];" in cpp - assert "std::to_string(sa[__pf_i])" not in cpp + assert "const auto& _pf_join_parts = (items);" in cpp + assert f"const std::string _pf_join_separator = ({separator});" in cpp + assert "return _pf_u16_join(_pf_join_parts, _pf_join_separator);" in cpp + assert "std::to_string(_pf_join_parts[" not in cpp -def test_array_join_numeric_elements(): +@pytest.mark.parametrize("element, values", [("int", "1, -2"), ("float", "1.25, -2.5")]) +@pytest.mark.parametrize("call, separator", JOIN_CALLS) +def test_array_join_numeric_elements(element, values, call, separator): + # Direct int and float outputs are measured independently. Other call + # forms here test the shared lowering, not separate TV value admissions. cpp = _gen( - "fa = array.new(0)\narray.push(fa, close)\n" - 's = array.join(fa, ",")\nplot(close)\n' + f"array<{element}> items = array.from({values})\n" + f's = {call}\nplot(close)\n' ) - assert "std::to_string(fa[__pf_i])" in cpp + assert "static std::string pine_str_tostring_tv(" in cpp + assert "const auto& _pf_join_parts = (items);" in cpp + assert f"const std::string _pf_join_separator = ({separator});" in cpp + assert "_pf_join_number(_pf_join_parts[_pf_join_i])" in cpp + assert "pine_str_tostring_tv(_pf_join_value)" in cpp + assert "if (!is_na(_pf_join_value)) return std::to_string(_pf_join_value);" in cpp + assert "std::to_string(_pf_join_parts[" not in cpp + + +@pytest.mark.parametrize("call", [ + "array.join(read_items(items, calls), read_separator(calls))", + "array.join(separator=read_separator(calls), id=read_items(items, calls))", + "read_items(items, calls).join(separator=read_separator(calls))", +]) +def test_array_join_binds_receiver_before_separator_outside_the_loop(call): + cpp = _gen( + "read_items(array values, array calls) =>\n" + " array.push(calls, 1)\n" + " values\n" + "read_separator(array calls) =>\n" + " array.push(calls, 2)\n" + ' "|"\n' + "items = array.new_float(0)\ncalls = array.new_int(0)\n" + f"observed = {call}\nplot(close)\n" + ) + lowered = next(line for line in cpp.splitlines() + if "observed = " in line and "_pf_join_parts" in line) + assert lowered.count("read_items") == lowered.count("read_separator") == 1 + assert lowered.index("read_items") < lowered.index("read_separator") < lowered.index("for (") def test_array_copy_int_elements(): diff --git a/tests/test_codegen_new.py b/tests/test_codegen_new.py index f93b26c2..beddec7e 100644 --- a/tests/test_codegen_new.py +++ b/tests/test_codegen_new.py @@ -782,20 +782,27 @@ def test_map_codegen(): def test_str_contains(): src = 'x = str.contains("hello world", "world")\n' cpp = _generate(src) - assert "find" in cpp - assert "npos" in cpp + assert "_pf_u16_contains(_pf_search_source, _pf_search_needle)" in cpp def test_str_lower(): src = 'x = str.lower("HELLO")\n' cpp = _generate(src) - assert "tolower" in cpp + # The lowering carries a complete Unicode table even for an ASCII source. + assert "_pf_tolower_map" in cpp + assert "{0xC9u, 0x1u, 0xE9u}" in cpp + assert "{0x10400u, 0x1u, 0x10428u}" in cpp + assert "::tolower" not in cpp def test_str_upper(): src = 'x = str.upper("hello")\n' cpp = _generate(src) - assert "toupper" in cpp + assert "_pf_toupper_map" in cpp + assert "{0xE9u, 0x1u, 0xC9u}" in cpp + assert "{0xDFu, 0x2u, 0x53u, 0x53u}" in cpp + assert "{0x10428u, 0x1u, 0x10400u}" in cpp + assert "::toupper" not in cpp def test_str_length(): @@ -808,20 +815,19 @@ def test_str_length(): def test_str_startswith(): src = 'x = str.startswith("hello", "he")\n' cpp = _generate(src) - assert "substr" in cpp + assert "_pf_u16_startswith(_pf_search_source, _pf_search_needle)" in cpp def test_str_endswith(): src = 'x = str.endswith("hello", "lo")\n' cpp = _generate(src) - assert "compare" in cpp + assert "_pf_u16_endswith(_pf_search_source, _pf_search_needle)" in cpp def test_str_pos(): src = 'x = str.pos("hello", "ll")\n' cpp = _generate(src) - assert "find" in cpp - assert "npos" in cpp + assert "_pf_u16_pos(_pf_search_source, _pf_search_needle)" in cpp def test_str_substring(): @@ -845,9 +851,40 @@ def test_str_replace(): def test_str_trim(): - src = 'x = str.trim(" hello ")\n' - cpp = _generate(src) - assert "erase" in cpp + from tests._compile import run_emitted_tu + + src = ( + '//@version=6\n' + 'strategy("trim runtime input")\n' + 'source_value = input.string("unused default", "Text")\n' + 'x = str.trim(source_value)\n' + 'plot(close)\n' + ) + cpp = transpile(src) + driver = r''' +#include +#include + +int main() { + GeneratedStrategy strategy; + const char source_bytes[] = "\t \302\240hello world\343\200\200\n"; + strategy.set_input("Text", std::string(source_bytes, sizeof(source_bytes) - 1)); + pineforge::Bar bar{}; + bar.open = bar.high = bar.low = bar.close = 100.0; + bar.volume = 1.0; + bar.timestamp = 1704067200000LL; + strategy.run(&bar, 1, "1", "1"); + if (!strategy.last_error().empty()) { + std::cerr << strategy.last_error() << "\n"; + return 14; + } + std::cout << strategy.x << "\n"; + return 0; +} +''' + assert run_emitted_tu( + cpp, driver, opt="-O0", label="str.trim runtime input", + ) == "hello world\n" def test_str_repeat(): @@ -1291,13 +1328,28 @@ def test_str_format_uses_runtime(): assert "pine_str_format" in cpp -def test_str_match_uses_runtime(): - cpp = _generate(""" -//@version=6 -strategy("T") -x = str.match("hello123", "\\\\d+") -""") - assert "pine_str_match" in cpp +@pytest.mark.parametrize("call", [ + 'str.match("hello123", "\\\\d+")', + 'str.match("aéb", "a[é]b")', + 'str.match(regex="a[à-ÿ]b", source="aéb")', + 'str.match("😀", ".")', + 'str.match(regex="a[😀-😇]b", source="a😀b")', +]) +def test_str_match_uses_runtime(call): + cpp = _generate(f'x = {call}\ny = {call}\n') + assert "return ::_pf_regex::_pf_match(" in cpp + assert cpp.count("inline std::string _pf_match(") == 1 + assert "::pineforge::pine_str_match(_pf_source_bytes, _pf_pattern_bytes)" in cpp + + +def test_str_match_evaluates_both_arguments_once_in_parameter_order(): + from pineforge_codegen.codegen.tables import STR_FUNC_MAP + + expression = STR_FUNC_MAP["match"](["next_source()", "next_pattern()"]) + assert expression.count("next_source()") == 1 + assert expression.count("next_pattern()") == 1 + assert expression.index("next_source()") < expression.index("next_pattern()") + assert expression.index("next_pattern()") < expression.index("return ::_pf_regex::_pf_match(") def test_str_split_uses_runtime(): diff --git a/tests/test_cpp_allowlist.py b/tests/test_cpp_allowlist.py index ac8c82ba..9196a3fd 100644 --- a/tests/test_cpp_allowlist.py +++ b/tests/test_cpp_allowlist.py @@ -28,12 +28,14 @@ # predates the unconditional settings and run-stop blocks, so it cannot show # those paths. These fixtures are transpiled here, against the current # emitter, for the positive corpus: settings/inputs (which emit ``<::``), -# timezone use (the sealed timestamp calendar run), drawing and request. +# timezone use (the sealed timestamp calendar run), drawing, request and the +# late-emitted ordinary-byte regex helpers. CURRENT_CODEGEN_FIXTURES = ( "tests/fixtures/matrix_eigen_pca.pine", "tests/fixtures/input_metadata_numeric.pine", "tests/fixtures/drawing_lifetime/w9dg-label-collect/strategy.pine", "tests/fixtures/request_discovery/case_strategy.pine", + "tests/gate-corpus/ok/validation__ta-str-match-regex-filter-01.pine", ) diff --git a/tests/test_foreign_footprint.py b/tests/test_foreign_footprint.py index 4f1ad221..33df4988 100644 --- a/tests/test_foreign_footprint.py +++ b/tests/test_foreign_footprint.py @@ -25,7 +25,8 @@ from pineforge_codegen import transpile, transpile_full from pineforge_codegen.errors import CompileError -from tests._compile import compile_cpp +from tests._compile import compile_cpp, run_emitted_tu +from tests._cpp_tokens import string_values from tests._e2e import chart_feed_head, skip_unless_e2e_env from tests._requests import build, hourly, run, same, write_root @@ -135,3 +136,116 @@ def test_footprint_of_the_charts_own_symbol_reads_its_feed(tmp_path_factory): assert same(result.trace[chart_open]["pf_d"], deltas[visible[-1]]), chart_open result = run(engine, work, feed_path, None, tag="none") assert not result.ok and PINNED in result.error + + +@pytest.mark.parametrize("names", [ + ("fp\0left", "fp\0right", "fp"), + ("fp\x01left", "fp\x01right", "fp"), + ("fpleft", "fpright", "fp"), + ("é\0中😀left", "é\0中😀right", "é"), + ("", "suffix", "missing"), +], ids=["nul", "other-control", "ascii", "unicode-nul", "empty-name"]) +def test_generated_column_helpers_compare_whole_names(names): + """Internal serializer/lookup contract; public Pine produces ASCII names.""" + from pineforge_codegen import CodeGen, _generate + from pineforge_codegen.external_requests import FOOTPRINT_COLUMN_ANNOTATION + + # Supply an internal annotation as serializer input, after normal checking. + # No Pine source or public footprint-name producer is changed or bypassed. + _, context, _, _, _ = _generate(HEAD + REQUEST + "d = fp.delta()\n", True, "") + (site,) = context.security_calls + assert site.expression.annotations[FOOTPRINT_COLUMN_ANNOTATION] == "fp_delta_100_70" + site.expression.annotations[FOOTPRINT_COLUMN_ANNOTATION] = names[0] + cpp = CodeGen(context).generate() + registration = next(line for line in cpp.splitlines() + if "if (_pf_symbol_data_installed(" in line) + payload = next(line for line in cpp.splitlines() if "_req_sec_0 = _pf_symbol_column(" in line) + for line in (registration, payload): + assert names[0] in string_values(line) + if "\0" in names[0]: + counted = re.findall(r'std::string\(("(?:[^"\\]|\\.)*"), (\d+)\)', line) + assert any( + string_values(literal) == [names[0]] + and int(count) == len(names[0].encode("utf-8")) + for literal, count in counted + ), line + + def native_bytes(value): + return "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + + body = "\n".join( + f'check("name{index}", probe._pf_symbol_data_installed("PF:A", "15", ' + f'{native_bytes(name)}));' + for index, name in enumerate(names) + ) + driver = r''' +#include +#include +#include +#include +#include + +class ColumnProbe : public GeneratedStrategy { +public: + using GeneratedStrategy::_pf_symbol_data_installed; + using GeneratedStrategy::_pf_symbol_column; + + void install(const std::string& left, const std::string& right, + const std::vector& bars, + const std::vector& closes) { + if (!set_symbol_feed("PF:A", "15", bars.data(), closes.data(), bars.size())) + throw std::runtime_error("could not install column test feed"); + auto& feed = symbol_feeds_.back(); + feed.columns.emplace_back(); + feed.columns.back().name = left; + feed.columns.back().values = {1.25, -4.5, 19.75}; + feed.columns.emplace_back(); + feed.columns.back().name = right; + feed.columns.back().values = {7.0, 11.0, -23.5}; + } +}; + +static_assert(std::is_invocable_r_v); + +void check(const char* name, bool found) { + std::cout << name << "=" << found << "\n"; +} + +int main() { + std::vector bars(3); + std::vector closes; + for (std::size_t i = 0; i < bars.size(); ++i) { + bars[i].open = bars[i].low = 90.0 + double(i); + bars[i].high = 110.0 + double(i); + bars[i].close = 100.0 + double(i); + bars[i].volume = 1.0; + bars[i].timestamp = 1704067200000LL + static_cast(i) * 900000LL; + closes.push_back(bars[i].timestamp + 900000LL); + } + ColumnProbe probe; + probe.install(LEFT, RIGHT, bars, closes); + // BODY + check("no-column-filter", probe._pf_symbol_data_installed("PF:A", "15")); + check("empty-name", probe._pf_symbol_data_installed("PF:A", "15", std::string{})); + check("other-symbol", probe._pf_symbol_data_installed("PF:B", "15", LEFT)); + check("other-timeframe", probe._pf_symbol_data_installed("PF:A", "60", LEFT)); + check("unknown-context-na", std::isnan(probe._pf_symbol_column(-1, LEFT))); + probe.run(bars.data(), bars.size(), "15", "15"); + if (!probe.last_error().empty()) throw std::runtime_error(probe.last_error()); + std::cout << "payload=" << probe.d << "\n"; + std::cout << "left=" << probe._pf_symbol_column(0, LEFT) << "\n"; + std::cout << "right=" << probe._pf_symbol_column(0, RIGHT) << "\n"; + check("unknown-column-na", std::isnan(probe._pf_symbol_column(0, MISSING))); +} +'''.replace("LEFT", native_bytes(names[0])).replace("RIGHT", native_bytes(names[1])) + driver = driver.replace("MISSING", native_bytes(names[2])).replace("// BODY", body) + output = run_emitted_tu(cpp, driver, opt="-O0", label="complete column names") + assert output.splitlines() == [ + "name0=1", "name1=1", "name2=0", "no-column-filter=1", + f"empty-name={int(names[0] == '')}", "other-symbol=0", "other-timeframe=0", + "unknown-context-na=1", "payload=19.75", "left=19.75", "right=-23.5", + "unknown-column-na=1", + ] diff --git a/tests/test_input_metadata_receipt.py b/tests/test_input_metadata_receipt.py index deb8816c..d59f81b9 100644 --- a/tests/test_input_metadata_receipt.py +++ b/tests/test_input_metadata_receipt.py @@ -206,20 +206,27 @@ def test_unrepresentable_strings_match_receipt_in_both_envelopes(): id="literal-escape-text"), pytest.param("x\\\0" + "789", "x\\", id="backslash-before-nul"), pytest.param("x\0y\0z", "x", id="multiple-nuls"), + pytest.param("é\0中😀", "é", id="unicode-nul"), ] @pytest.mark.parametrize("value,expected", STRING_VALUES) -@pytest.mark.parametrize("wrapped", [False, True], ids=["literal", "std-string"]) +@pytest.mark.parametrize("wrapped", [False, True, "counted"], + ids=["literal", "std-string", "counted"]) def test_emitted_string_setting_decoder(value, expected, wrapped): literal = '"' + NamingHelper._cpp_string_escape(value) + '"' - lowered = f"std::string({literal})" if wrapped else literal + if wrapped == "counted": + lowered = f"std::string({literal}, {len(value.encode('utf-8'))})" + else: + lowered = f"std::string({literal})" if wrapped else literal assert _string_setting_arg(StringLiteral(value=value), lowered) == lowered - assert _string_setting_value(lowered) == expected + assert _string_setting_value(lowered) == (value if wrapped == "counted" else expected) @pytest.mark.parametrize("lowered", [ - "0", "nullptr", 'std::string("x", 1)', '"x" + other', '"x" "y"', + "0", "nullptr", 'std::string("x", 0)', 'std::string("x", 2)', + 'std::string("é\\000z", 3)', 'std::string("é\\000z", 5)', + '"x" + other', '"x" "y"', r'"\u2603"', r'"\x41"', r'"\012"', r'"\0"', r'"\a"', '"x\\"', '"x\ny"', '"x\ry"', '"x\0y"', ]) @@ -229,9 +236,69 @@ def test_string_setting_decoder_refuses_non_emitted_forms(lowered): _string_setting_value(lowered) +INPUT_DRIVER = r''' +#include +#include +#include + +class InputProbe : public GeneratedStrategy { +public: + using GeneratedStrategy::_pf_set_input_checked; + using GeneratedStrategy::_pf_settings_receipt; +}; + +void observe(const char* name, const std::string& value) { + static const char digits[] = "0123456789abcdef"; + std::cout << name << "="; + for (unsigned char byte : value) + std::cout << digits[byte >> 4] << digits[byte & 15]; + std::cout << "\n"; +} + +void execute(InputProbe& strategy) { + std::vector bars(3); + for (std::size_t i = 0; i < bars.size(); ++i) { + bars[i].open = bars[i].low = 90.0 + double(i); + bars[i].high = 110.0 + double(i); + bars[i].close = 100.0 + double(i); + bars[i].volume = 1.0; + bars[i].timestamp = 1704067200000LL + static_cast(i) * 60000LL; + } + strategy.run(bars.data(), bars.size(), "1", "1"); + if (!strategy.last_error().empty()) + throw std::runtime_error(strategy.last_error()); +} + +int main() { + // BODY +} +''' + + +def _bytes_cpp(value): + """Supply a native setter value independently of the product serializer.""" + return "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + + +def _run_input_observer(full, body, opt): + output = run_emitted_tu( + full["cpp"], INPUT_DRIVER.replace("// BODY", body), + opt=opt, label="string input bytes and receipt", + ) + records = {} + for line in output.splitlines(): + key, value = line.split("=", 1) + assert key not in records, output + records[key] = value + return records + + @pytest.mark.parametrize("value,expected", STRING_VALUES) @pytest.mark.parametrize("opt", ["-O0", "-O2"]) -def test_all_string_input_forms_match_native_receipt(value, expected, opt): +@pytest.mark.parametrize("call_style", ["positional", "named"]) +def test_all_string_input_forms_match_native_receipt(value, expected, opt, call_style): literal = pine_string_literal(value) forms = [ ("string", "String default", ""), @@ -242,14 +309,24 @@ def test_all_string_input_forms_match_native_receipt(value, expected, opt): ("string", "String option", f', options=[{literal}, "safe"]'), ] source = '//@version=6\nstrategy("literal settings")\n' - source += "".join(f'choice_{index} = input.{kind}({literal}, "{title}"{options})\n' - for index, (kind, title, options) in enumerate(forms)) + for index, (kind, title, options) in enumerate(forms): + arguments = (f'defval={literal}, title="{title}"' + if call_style == "named" else f'{literal}, "{title}"') + source += f'choice_{index} = input.{kind}({arguments}{options})\n' full = transpile_full(source) envelope = json.loads(GLUE(source)) assert envelope["ok"] is True assert envelope["inputs"] == full["inputs"] - receipt = json.loads(run_emitted_tu(full["cpp"], RECEIPT_DRIVER, opt=opt, - label="literal settings receipt")) + body = "InputProbe strategy;\nexecute(strategy);\n" + body += "".join( + f'observe("choice_{index}", strategy.choice_{index});\n' + for index in range(len(forms)) + ) + body += 'std::cout << "receipt=" << strategy._pf_settings_receipt() << "\\n";\n' + records = _run_input_observer(full, body, opt) + assert set(records) == {*(f"choice_{index}" for index in range(len(forms))), "receipt"} + receipt = json.loads(records.pop("receipt")) + assert list(records.values()) == [value.encode("utf-8").hex()] * len(forms) assert len(full["inputs"]) == len(receipt["inputs"]) == len(forms) for metadata, checked, (kind, title, options) in zip(full["inputs"], receipt["inputs"], forms): assert metadata["title"] == checked["name"] == title @@ -257,6 +334,88 @@ def test_all_string_input_forms_match_native_receipt(value, expected, opt): if kind == "symbol": assert metadata["kind"] == "symbol" assert metadata["supported"] is True - assert metadata["default"] == checked["default"] == checked["effective_value"] == expected - assert metadata.get("options", []) == ([expected, "safe"] if options else []) + # STRING_VALUES' second field describes uncounted C strings only. + # Authored defaults and options retain the complete Pine value. + assert metadata["default"] == checked["default"] == checked["effective_value"] == value + assert metadata.get("options", []) == ([value, "safe"] if options else []) assert _manifest_view(metadata) == _receipt_view(checked) + + +@pytest.mark.parametrize("separator", ["\0", "\x01", ""], + ids=["nul", "other-control", "ascii"]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_string_options_keep_distinct_suffixes_in_native_values(separator, opt): + """The C++ checked setter validates whole option values, including NUL.""" + options = [f"a{separator}b", f"a{separator}c"] + source = '//@version=6\nstrategy("option identity")\n' + source += f'var string FIRST = {pine_string_literal(options[0])}\n' + source += f'const string SECOND = {pine_string_literal(options[1])}\n' + source += 'choice = input.string(FIRST, "Choice", options=[FIRST, SECOND])\n' + full = transpile_full(source) + attempts = [*options, "a"] + body = "" + for index, attempt in enumerate(attempts): + body += f''' +{{ + InputProbe strategy; + try {{ + strategy._pf_set_input_checked("Choice", {_bytes_cpp(attempt)}); + std::cout << "accepted{index}=1\\n"; + }} catch (const ::pineforge::checked_settings::Error&) {{ + std::cout << "accepted{index}=0\\n"; + }} + execute(strategy); + observe("value{index}", strategy.choice); + std::cout << "receipt{index}=" << strategy._pf_settings_receipt() << "\\n"; +}} +''' + records = _run_input_observer(full, body, opt) + assert set(records) == { + f"{field}{index}" for field in ("accepted", "value", "receipt") for index in range(3) + } + assert full["inputs"][0]["supported"] is True + assert full["inputs"][0]["options"] == options + for index, attempt in enumerate(attempts): + accepted = attempt in options + effective = attempt if accepted else options[0] + assert records[f"accepted{index}"] == str(int(accepted)) + assert records[f"value{index}"] == effective.encode("utf-8").hex() + (checked,) = json.loads(records[f"receipt{index}"])["inputs"] + assert checked["supported"] is True + assert checked["options"] == options + assert checked["effective_value"] == effective + + +@pytest.mark.parametrize("separator", ["\0", "\x01", ""], + ids=["nul", "other-control", "ascii"]) +@pytest.mark.parametrize("opt", ["-O0", "-O2"]) +def test_input_keys_keep_distinct_suffixes_for_scalar_and_source_getters(separator, opt): + """Full C++ keys address scalar/source inputs independently of prefixes.""" + keys = [f"Key{separator}left", f"Key{separator}right", "Key", f"Source{separator}value"] + source = '//@version=6\nstrategy("key identity")\n' + source += f'const string KEY = {pine_string_literal("Key" + separator)}\n' + source += 'first = input.int(11, KEY + "left")\n' + source += f'second = input.int(defval=22, title={pine_string_literal(keys[1])})\n' + source += 'prefix = input.int(33, "Key")\n' + source += f'selected = input.source(title={pine_string_literal(keys[3])}, defval=close)\n' + full = transpile_full(source) + overrides = ["41", "42", "43", "high"] + body = "InputProbe strategy;\n" + for key, value in zip(keys, overrides, strict=True): + body += f"strategy._pf_set_input_checked({_bytes_cpp(key)}, {_bytes_cpp(value)});\n" + body += ''' +execute(strategy); +std::cout << "first=" << strategy.first << "\\n"; +std::cout << "second=" << strategy.second << "\\n"; +std::cout << "prefix=" << strategy.prefix << "\\n"; +std::cout << "selected=" << strategy.selected << "\\n"; +std::cout << "receipt=" << strategy._pf_settings_receipt() << "\\n"; +''' + records = _run_input_observer(full, body, opt) + assert set(records) == {"first", "second", "prefix", "selected", "receipt"} + assert [records[name] for name in ("first", "second", "prefix")] == overrides[:3] + assert float(records["selected"]) == 112.0 + assert [entry["title"] for entry in full["inputs"]] == keys + inputs = json.loads(records["receipt"])["inputs"] + assert [entry["name"] for entry in inputs] == keys + assert [entry["effective_value"] for entry in inputs] == overrides diff --git a/tests/test_method_receiver_types.py b/tests/test_method_receiver_types.py index 21a2c3c9..7e5d9e0c 100644 --- a/tests/test_method_receiver_types.py +++ b/tests/test_method_receiver_types.py @@ -100,6 +100,32 @@ def test_array_user_method_precedes_same_named_builtin() -> None: compile_cpp(cpp, label="array-method-builtin-collision") +@pytest.mark.parametrize( + "receiver_type, receiver", + [ + pytest.param("array", 'array.from("alpha", "beta")', id="array"), + pytest.param("array", "array.from(true, false)", id="bool-array"), + pytest.param("int", "7", id="namespace-shadow"), + ], +) +def test_authored_join_return_precedes_fixed_builtin_result(receiver_type, receiver): + source = f'''//@version=6 +strategy("authored join return") +method join({receiver_type} self, string separator) => + self + str.length(separator) +relay({receiver_type} array, string separator) => + array.join(separator) +parts = {receiver} +result = relay(parts, "|") +plot(close) +''' + cpp = transpile(source) + assert "int relay(" in cpp + assert re.search(r"\bint result\s*[=;]", cpp), cpp + compile_cpp(cpp, label="authored-array-join-return") + + def test_map_receiver_keeps_shared_id_value_handle() -> None: source = '''//@version=6 strategy("map builtin collision") diff --git a/tests/test_na_and_compare_refusals.py b/tests/test_na_and_compare_refusals.py index 74b3134a..23f13610 100644 --- a/tests/test_na_and_compare_refusals.py +++ b/tests/test_na_and_compare_refusals.py @@ -330,16 +330,17 @@ def test_grouping_around_consecutive_not_keeps_the_refusal( ) -def test_string_ordering_reports_the_first_string_operand() -> None: +@pytest.mark.parametrize("operator", ["<", "<=", ">", ">="]) +def test_string_ordering_reports_the_first_string_operand(operator: str) -> None: _assert_diagnostic( _script( 'string a = "a"\n' 'string b = "b"\n' - "value = a < b\n" + f"value = a {operator} b\n" ), "PF-E2052", - "Cannot order string values with <.", - {"op": "<"}, + f"Cannot order string values with {operator}.", + {"op": operator}, 5, 9, # a is the first operand of the ordered comparison. ) diff --git a/tests/test_security_helper_local_na.py b/tests/test_security_helper_local_na.py index 15724ee0..1a2e0fff 100644 --- a/tests/test_security_helper_local_na.py +++ b/tests/test_security_helper_local_na.py @@ -15,6 +15,8 @@ import re +import pytest + from pineforge_codegen import transpile from tests import _compile as compile_env @@ -98,3 +100,80 @@ def test_int_locals_read_from_helper_state_narrow_without_losing_na(): body, ), body compile_env.compile_cpp(cpp, label="security-helper-state-narrowing") + + +@pytest.mark.parametrize( + "prefix, arguments, forwarded", + [ + pytest.param("string absent = na\n", '"é", absent', False, id="global-na"), + pytest.param( + "string absent = na\nstring alias = absent\n", + '"中", alias', False, id="aliased-global-na", + ), + pytest.param("", 'na, "suffix"', False, id="parameter-na"), + pytest.param( + "string absent = na\n", 'absent, "尾"', True, id="forwarded-first-na", + ), + pytest.param( + "first = 17\nstring absent = na\n", '"lead", absent', True, + id="parameter-shadows-global", + ), + ], +) +def test_requested_string_arguments_type_inferred_locals(prefix, arguments, forwarded): + # The declaration and missing-value representations are the unit target; + # Unicode/NA value observations belong to the external parity probes. + helper = ( + "combine(string first, string last) =>\n" + " accumulated = first\n" + " accumulated += last\n" + " accumulated\n" + ) + if forwarded: + helper += ( + "forward(string first, string last) =>\n" + " copied = first\n" + " combine(copied, last)\n" + ) + callee = "forward" if forwarded else "combine" + source = ( + '//@version=6\nstrategy("requested string argument storage")\n' + + helper + prefix + + f'joined = request.security(syminfo.tickerid, "1", {callee}({arguments}))\n' + + "plot(str.length(joined))\n" + ) + cpp = transpile(source) + body = _eval_bodies(cpp) + assert re.search(r"std::string _sec0_combine_\d+_accumulated = ", body), body + if forwarded: + assert re.search(r"std::string _sec0_forward_\d+_copied = ", body), body + assert "na()" in body, body + assert "na()" not in body, body + assert "_pf_u16_concat(" in body, body + compile_env.compile_cpp(cpp, label="requested-string-argument-storage") + + +@pytest.mark.parametrize( + "storage", + [pytest.param("var ", id="persistent"), pytest.param("", id="history")], +) +def test_requested_inferred_string_state_declares_string_history(storage): + history = " previous = accumulated[1]\n" if not storage else "" + result = "previous" if history else "accumulated" + source = ( + '//@version=6\nstrategy("requested inferred string history")\n' + "read_value(string first) =>\n" + f" {storage}accumulated = first\n" + + history + f" {result}\n" + + 'joined = request.security(syminfo.tickerid, "1", read_value(na))\n' + + "plot(str.length(joined))\n" + ) + cpp = transpile(source) + body = _eval_bodies(cpp) + assert "Series> _security_helper_series_str_;" in cpp + assert "_security_helper_series_str_" in body, body + assert "na()" in body, body + assert "na()" not in body, body + if history: + assert re.search(r"std::string _sec0_read_value_\d+_previous = ", body), body + compile_env.compile_cpp(cpp, label="requested-inferred-string-history") diff --git a/tests/test_string_local_history_concat.py b/tests/test_string_local_history_concat.py index 28e657e3..4e099955 100644 --- a/tests/test_string_local_history_concat.py +++ b/tests/test_string_local_history_concat.py @@ -7,9 +7,14 @@ from pineforge_codegen import transpile from pineforge_codegen.errors import CompileError, Phase from tests._compile import compile_cpp, run_emitted_tu +from tests._older_codegen import with_utf16_units -_REFERENCE_JOIN = re.compile(r"std::to_string\((\w+)\[0\]\) \+ (\w+)\)") +_REFERENCE_JOIN = re.compile(r"\(std::to_string\((\w+)\[0\]\) \+ (\w+)\)") +_REFERENCE_REQUESTED_JOIN = re.compile( + r'\((\(_security_helper_series_str_\.[A-Za-z_][A-Za-z_0-9]*\(' + r'"(?:\\.|[^"\\])*", [0-9]+\)[^()\n;]*\)\[0\]) \+ (\w+)\)' +) def with_string_join_fixed(cpp): @@ -17,11 +22,15 @@ def with_string_join_fixed(cpp): The older emission wrapped the current value of a string local read with history in std::to_string when joining it with its previous value (issue - #354), which does not compile. A test that compares a new emission with an - older one passes only when that single site is the whole difference. + #354), which does not compile. Both that join and the corresponding + requested join now use UTF-16 concatenation. The comparison still checks + the complete C++, allowing only these joins and the shared helper block. """ assert len(_REFERENCE_JOIN.findall(cpp)) == 1, cpp - return _REFERENCE_JOIN.sub(r"\1[0] + \2)", cpp) + cpp = _REFERENCE_JOIN.sub(r"_pf_u16_concat(\1[0], \2)", cpp) + cpp, requested = _REFERENCE_REQUESTED_JOIN.subn(r"_pf_u16_concat(\1, \2)", cpp) + assert requested <= 1, cpp + return with_utf16_units(cpp) _REPRODUCTION = """//@version=6 @@ -141,7 +150,8 @@ def test_history_string_operands_compile_and_return_the_join( for member in members: assert f"std::to_string({member}" not in cpp expression = ( - f"(bb + {member}[0])" if swapped else f"({member}[0] + bb)" + f"_pf_u16_concat(bb, {member}[0])" if swapped + else f"_pf_u16_concat({member}[0], bb)" ) assert expression in cpp # Bar 0 executes to seed history. Only bars with an existing previous diff --git a/tests/test_string_nul_values.py b/tests/test_string_nul_values.py index 4991c646..f9cf2da4 100644 --- a/tests/test_string_nul_values.py +++ b/tests/test_string_nul_values.py @@ -84,7 +84,7 @@ def _run(script: str, observe: str) -> str: ), pytest.param( f'enum Mode\n fast = "f{NUL}t"\n slow\n' - 'e = Mode.fast\ntext = str.tostring(e)\nn = str.length(text)\nplot(n)\n', + 'e = Mode.fast\ntitle_value = str.tostring(e)\nn = str.length(title_value)\nplot(n)\n', 'std::cout << strategy.n << "\\n";', "3\n", id="length-of-an-enum-title", ), pytest.param( @@ -98,13 +98,11 @@ def _run(script: str, observe: str) -> str: 'std::cout << strategy.n << "\\n";', "3\n", id="length-of-a-history-read-var-set-from-a-literal", ), - # A known limit, pinned so that a change of it is seen: the default of a - # string input is still cut at its first NUL, so the input no longer equals - # the literal it was given (the two were equal only while both were cut). + # The input's getter must preserve the same bytes as its authored default. pytest.param( f'm = input.string("a{NUL}b", "Text")\nu = m == "a{NUL}b" ? 1 : 0\n' 'k = str.length(m)\nplot(u + k)\n', - 'std::cout << strategy.u << " " << strategy.k << "\\n";', "0 1\n", + 'std::cout << strategy.u << " " << strategy.k << "\\n";', "1 3\n", id="known-limit-an-input-default-is-cut-and-differs-from-its-literal", ), ] @@ -180,3 +178,58 @@ def test_the_forms_above_keep_their_lengths_at_run_time(): '<< strategy.viaheld << " " << strategy.joined << " " << strategy.chosen << "\\n";' ) assert _run(script, observe) == "3 3 3 4 3\n" + + +@pytest.mark.parametrize("value", [ + "\0", "x\0" + "789", "x\0y\0z", 'é"\\\0中😀', "a\x01b", "60", "", +], ids=["nul", "digits", "multiple", "utf8-escapes", "other-control", "ascii", "empty"]) +def test_length_aware_literal_consumers_keep_the_authored_bytes(value): + """Serializer units: a timeframe's grammar is a separate validation rule.""" + from pineforge_codegen import Analyzer, CodeGen, Lexer, Parser + from pineforge_codegen.codegen.helpers import NamingHelper + from pineforge_codegen.codegen.input import InputHelper + from pineforge_codegen.codegen.security import SecurityEmitter + + class Consumers(NamingHelper, InputHelper, SecurityEmitter): + pass + + consumer = Consumers() + ast = Parser(Lexer(HEADER).tokenize(), source=HEADER).parse() + generator = CodeGen(Analyzer(ast).analyze()) + generator._known_vars["sample"] = value + expressions = [ + consumer._cpp_string_literal(value), + consumer._input_key_literal(value), + consumer._security_tf_literal(value), + generator._resolve_known("sample"), + ] + expected = "std::string{" + ", ".join( + f"static_cast({byte})" for byte in value.encode("utf-8") + ) + "}" + body = "\n".join( + f'check("consumer{index}", {expression}, {expected});' + for index, expression in enumerate(expressions) + ) + driver = r''' +#include +#include + +void check(const char* name, const std::string& value, const std::string& expected) { + if (value != expected) throw std::runtime_error("literal consumer changed bytes"); + static const char digits[] = "0123456789abcdef"; + std::cout << name << "="; + for (unsigned char byte : value) + std::cout << digits[byte >> 4] << digits[byte & 15]; + std::cout << "\n"; +} + +int main() { + // BODY +} +'''.replace("// BODY", body) + output = run_emitted_tu( + transpile(HEADER), driver, opt="-O0", label="length-aware literal consumers", + ) + assert output.splitlines() == [ + f"consumer{index}={value.encode('utf-8').hex()}" for index in range(len(expressions)) + ] diff --git a/tests/test_string_split_empty_separator.py b/tests/test_string_split_empty_separator.py index 45b90272..389c6132 100644 --- a/tests/test_string_split_empty_separator.py +++ b/tests/test_string_split_empty_separator.py @@ -12,7 +12,7 @@ import pytest from pineforge_codegen import transpile -from tests._compile import run_emitted_tu +from tests._compile import compile_cpp, run_emitted_tu HEADER = '//@version=6\nstrategy("string split")\n' @@ -133,3 +133,45 @@ def test_the_empty_text_gives_one_part(script, observe, expected): @pytest.mark.parametrize("script, observe, expected", CONTROL_ROWS) def test_a_separator_that_is_not_empty_splits_as_before(script, observe, expected): assert _run(script, observe) == expected + + +MISSING_SEPARATOR_ROWS = [ + pytest.param( + 'n = array.size(str.split("abc", na))\n' + 'parts = str.split("abc", na)\nplot(n)\n', + id="literal-separator-and-nested-consumer", + ), + pytest.param( + 'parts = str.split("", separator = na)\n' + 'n = array.size(parts)\nplot(n)\n', + id="empty-source-with-named-separator", + ), + pytest.param( + 'parts = str.split(separator = na, string = "é中")\n' + 'n = array.size(parts)\nplot(n)\n', + id="bmp-source-with-reversed-keywords", + ), + pytest.param( + 'parts = str.split("a\U0001f600b", na)\n' + 'n = array.size(parts)\nplot(n)\n', + id="astral-source-with-positional-separator", + ), + pytest.param( + 'src = close > open ? "longer" : "xy"\n' + 'parts = str.split(src, na)\nn = array.size(parts)\nplot(n)\n', + id="computed-source", + ), + pytest.param( + 'f(string src) =>\n str.split(src, na)\n' + 'parts = f("aéb")\nn = array.size(parts)\nplot(n)\n', + id="helper-body", + ), +] + + +@pytest.mark.parametrize("script", MISSING_SEPARATOR_ROWS) +def test_a_literal_na_separator_has_the_string_parameter_type(script): + # This target is contextual argument typing: compile the complete emitted + # TU so a numeric na cannot reach a string parameter. Runtime/TV values + # belong to the split probes, including the separate named/Unicode tapes. + compile_cpp(transpile(HEADER + script), label="split string argument type") diff --git a/tests/test_string_units.py b/tests/test_string_units.py index fcdf24e2..d23fc5c9 100644 --- a/tests/test_string_units.py +++ b/tests/test_string_units.py @@ -13,20 +13,33 @@ * ``measured-*``: the script text and the value TradingView returned for it (str.length, str.pos and str.substring on the strings "a" + one character + - "b"; and str.substring at its edges on "abc" and on a, an emoji, b: - TradingView refuses the two begin positions outside the text when it compiles - a script whose arguments are constants, where the engine stops the run); + "b"; and str.substring at its edges on "abc" and on a, an emoji, b). + A constant negative begin has a separate compiler-refusal assertion below; + the positive out-of-range constant still exercises the native runtime guard + and does not establish compiler agreement; * ``derived-*``: arithmetic on the same unit (an emoji counts two), not a separate reading; * ``design-*``: a choice the measurement does not decide. A cut between the two units of an emoji keeps each half as its own three-byte sequence, the form str.split gives its parts, so the length of the half is one unit; * ``control-*``: a case in which bytes, code points and UTF-16 units agree. + +For issue #382 item 8, ``measured-*`` trim rows repeat the issue's four exact +examples. ``contract-derived-*`` rows compose only its supplied removed/kept +characters with other text, interior characters, empty text and runtime inputs; +they are not separate TradingView measurements or a full Unicode whitespace set. """ +import json + import pytest from pineforge_codegen import transpile +from pineforge_codegen.analyzer import Analyzer +from pineforge_codegen.diagnostic_codes import UNCATALOGUED +from pineforge_codegen.errors import CompileError, Level, Phase, SourceLocation +from pineforge_codegen.lexer import Lexer +from pineforge_codegen.parser import Parser from tests._compile import run_emitted_tu HEADER = '//@version=6\nstrategy("string units")\n' @@ -50,6 +63,7 @@ bars[i].timestamp = 1704067200000LL + static_cast(i) * 60000LL; } GeneratedStrategy strategy; + // INPUTS strategy.run(bars.data(), bars.size(), "1", "1"); if (!strategy.last_error().empty()) { std::cerr << strategy.last_error() << "\n"; @@ -66,10 +80,18 @@ FEFF = "\U0000feff" -def _run(script: str, observe: str) -> str: +def _run(script: str, observe: str, *, opt: str = "-O0", + inputs: dict[str, str] | None = None, setup: str = "") -> str: cpp = transpile(HEADER + script) + input_setup = "\n".join( + f"strategy.set_input({json.dumps(name)}, {json.dumps(value, ensure_ascii=False)});" + for name, value in (inputs or {}).items() + ) + driver = DRIVER.replace( + "// INPUTS", input_setup + ("\n" + setup if setup else ""), + ).replace("// OBSERVE", observe) return run_emitted_tu( - cpp, DRIVER.replace("// OBSERVE", observe), opt="-O0", label="string units", + cpp, driver, opt=opt, label="string units", ) @@ -103,15 +125,159 @@ def test_length_counts_utf16_units(text, expected): pytest.param("aab", "b", 2, id="measured-after-an-ascii-letter"), pytest.param(f"{EMOJI}{EMOJI}b", "b", 4, id="derived-after-two-emoji"), pytest.param(f"a{EMOJI}b", EMOJI, 1, id="derived-the-emoji-itself"), - pytest.param(f"a{EMOJI}b", "x", -1, id="control-not-found"), + # Issue #382 item 5: an absent match is na, including after non-ASCII text. + pytest.param(f"a{EMOJI}b", "x", "na", id="control-not-found"), pytest.param(f"a{CJK}b", CJK, 1, id="control-the-bmp-character-itself"), ] @pytest.mark.parametrize("text, needle, expected", POS_ROWS) def test_pos_counts_utf16_units(text, needle, expected): - script = f'p = str.pos("{text}", "{needle}")\nplot(p)\n' - assert _run(script, 'std::cout << strategy.p << "\\n";') == f"{expected}\n" + script = (f'p = str.pos("{text}", "{needle}")\n' + 'missing = na(p)\nplot(p)\n') + observe = ( + 'if (strategy.missing) std::cout << "na\\n"; ' + 'else std::cout << strategy.p << "\\n";' + ) + assert _run(script, observe) == f"{expected}\n" + + +# Retain the original module selection and run these +# boundary/consumer rows at both optimisation levels. These are native lowering +# checks of the behavior described by the public string fixture. +NATIVE_OPTS = [ + pytest.param("-O0", id="O0"), + pytest.param("-O2", id="O2"), +] + +MISSING_POS_CONSUMERS = [ + pytest.param('nz(str.pos("abc", "x"), 0)', "0\n", id="nz"), + pytest.param('str.tostring(str.pos("abc", "x"))', "NaN\n", id="tostring"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("expression, expected", MISSING_POS_CONSUMERS) +def test_missing_pos_literal_consumers(expression, expected, opt): + # Keep the reported public expressions intact, including direct nesting. + script = f"observed = {expression}\nplot(close)\n" + assert _run(script, 'std::cout << strategy.observed << "\\n";', opt=opt) == expected + + +RUNTIME_POS_ROWS = [ + pytest.param("abc", "x", True, 0, "NaN", id="missing-ascii"), + pytest.param("seed42", "?", True, 0, "NaN", id="missing-distinct"), + pytest.param("abc", "b", False, 1, "1", id="found-middle"), + pytest.param("abc", "a", False, 0, "0", id="found-zero"), + pytest.param("abc", "", False, 0, "0", id="empty-needle"), + pytest.param("", "x", True, 0, "NaN", id="empty-haystack"), + pytest.param("", "", False, 0, "0", id="both-empty"), + pytest.param("ab", "abc", True, 0, "NaN", id="longer-needle"), + pytest.param(f"a{EMOJI}b", "b", False, 3, "3", id="found-utf16"), + pytest.param(f"{CJK}{EMOJI}tail", "l", False, 6, "6", id="found-distinct"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("text, needle, missing, replaced, rendered", RUNTIME_POS_ROWS) +def test_pos_consumers_use_runtime_inputs(text, needle, missing, replaced, rendered, opt): + # Every row emits the same Pine source. Only the native input overrides + # change; expected positions/strings are never injected into that source. + script = ( + 'source_value = input.string("unused text", "Text")\n' + 'needle = input.string("unused needle", "Needle")\n' + 'p = str.pos(source_value, needle)\n' + 'missing = na(p)\n' + 'replaced = nz(p, 0)\n' + 'rendered = str.tostring(p)\n' + 'plot(p)\n' + ) + observe = ( + 'std::cout << strategy.missing << "|" << strategy.replaced ' + '<< "|" << strategy.rendered << "\\n";' + ) + assert _run(script, observe, opt=opt, inputs={"Text": text, "Needle": needle}) == ( + f"{int(missing)}|{replaced}|{rendered}\n" + ) + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +def test_substring_after_absent_delimiter_completes(opt): + # An unexpected run stop remains a failing native execution in DRIVER, + # never an expected-stop assertion. + script = ( + 't = str.substring("abc", str.pos("abc", ":") + 1)\n' + 'missing = na(t)\n' + 'completed = true\n' + 'plot(close)\n' + ) + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << "\\n"; ' + 'hex(strategy.t);' + ) + assert _run(script, observe, opt=opt) == "1 0\n616263\n" + + +MISSING_START_ROWS = [ + pytest.param("abcdef", "int(na)", id="ascii-int-na"), + pytest.param(f"a{EMOJI}b", "int(na) + 1", id="unicode-wide-na"), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("text, begin", MISSING_START_ROWS) +def test_substring_missing_start_returns_whole_text(text, begin, opt): + script = ( + 'source_value = input.string("unused text", "Text")\n' + f't = str.substring(source_value, {begin})\n' + 'missing = na(t)\n' + 'completed = true\n' + 'plot(close)\n' + ) + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << "\\n"; ' + 'hex(strategy.t);' + ) + assert _run(script, observe, opt=opt, inputs={"Text": text}) == ( + f"1 0\n{text.encode('utf-8').hex()}\n" + ) + + +MISSING_END_ROWS = [ + pytest.param( + 't = str.substring("abc", 0, str.pos("abc", ":"))\n', {}, + id="literal-zero-begin", + ), + pytest.param( + 't = str.substring("abc", 1, str.pos("abc", ":"))\n', {}, + id="literal-nonzero-begin", + ), + pytest.param( + 'source_value = input.string("unused text", "Text")\n' + 'needle = input.string("unused needle", "Needle")\n' + 'begin = input.int(1, "Begin")\n' + 't = str.substring(source_value, begin, str.pos(source_value, needle))\n', + {"Text": f"r{EMOJI}tail", "Needle": "!", "Begin": "2"}, + id="runtime-utf16-nonzero-begin", + ), +] + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +@pytest.mark.parametrize("script, inputs", MISSING_END_ROWS) +def test_substring_missing_end_returns_string_na(script, inputs, opt): + # The zero-begin row isolates the wrong value without signed overflow. + # Check a missing end before subtracting begin. A stop or undefined + # conversion cannot satisfy the value assertion. + script += 'missing = na(t)\ncompleted = true\nplot(close)\n' + # visit_call.py lowers Pine na(string) as .empty(); the engine has no + # is_na(std::string) overload. Compare the actual result with its canonical + # na() too, without hard-coding empty bytes or a numeric NA. + observe = ( + 'std::cout << strategy.completed << " " << strategy.missing << " " ' + '<< (strategy.t == na()) << "\\n";' + ) + assert _run(script, observe, opt=opt, inputs=inputs) == "1 1 1\n" TAIL_ROWS = [ @@ -241,19 +407,114 @@ def test_the_arguments_may_be_any_string_expression(script, observe, expected): id="str-substring-alone"), pytest.param('p = str.pos("abc", "b")\nplot(close)\n', True, id="str-pos-alone"), pytest.param('n = str.length("abc")\nplot(close)\n', True, id="str-length-alone"), + pytest.param('t = str.match("aéb", "a.b")\nplot(close)\n', True, + id="str-match-alone"), + pytest.param('n = str.contains("abc", "b")\nplot(close)\n', True, + id="str-contains-alone"), + pytest.param('n = str.startswith("abc", "a")\nplot(close)\n', True, + id="str-startswith-alone"), + pytest.param('n = str.endswith("abc", "c")\nplot(close)\n', True, + id="str-endswith-alone"), + pytest.param('s = "a" + "b"\nplot(close)\n', True, id="string-plus-alone"), + pytest.param('s = "a"\ns += "b"\nplot(close)\n', True, id="string-append-alone"), + pytest.param("n = 1\nn += 2\nplot(close)\n", False, + id="control-numeric-append"), + pytest.param( + 'parts = array.from("a", "b")\n' + 'joined = array.join(parts, "")\nplot(close)\n', + True, id="string-array-namespace-join-alone", + ), + pytest.param( + 'parts = array.from("a", "b")\njoined = parts.join()\nplot(close)\n', + True, id="string-array-method-join-default-separator", + ), + pytest.param( + 'joined = array.from("a", "b").join("")\nplot(close)\n', + True, id="string-array-temporary-join-alone", + ), + pytest.param( + 'collect_join(array parts, array results) =>\n' + ' array.push(results, parts.join(""))\n' + ' array.size(parts)\n' + 'results = array.new_string(0)\n' + 'counted = collect_join(array.from("a", "b"), results)\nplot(close)\n', + True, id="string-array-forwarded-join-alone", + ), pytest.param('t = str.upper("abc")\nplot(close)\n', False, id="control-another-string-function"), pytest.param("plot(close)\n", False, id="control-no-string-function"), # A plot argument is not emitted, so the call emits nothing and needs no helper. pytest.param('plot(str.length("abc"))\n', False, id="control-a-call-that-is-only-a-plot-argument"), + pytest.param('plot(str.match("aéb", "a.b"))\n', False, + id="control-a-match-that-is-only-a-plot-argument"), ] @pytest.mark.parametrize("script, emitted", EMISSION_ROWS) def test_the_helpers_are_emitted_only_for_a_call_that_uses_them(script, emitted): cpp = transpile(HEADER + script) - assert (HELPER_DEFINITION in cpp) is emitted + assert cpp.count(HELPER_DEFINITION) == int(emitted) + + +@pytest.mark.parametrize("operation", ("pos", "contains", "startswith", "endswith")) +@pytest.mark.parametrize("named", (False, True), ids=("positional", "named")) +def test_search_arguments_bind_once_in_parameter_order(operation, named): + arguments = 'read_value("abc", calls, 1), read_value("b", calls, 2)' + if named: + arguments = 'str=read_value("b", calls, 2), source=read_value("abc", calls, 1)' + cpp = transpile( + HEADER + + "read_value(string payload, array calls, int tag) =>\n" + + " array.push(calls, tag)\n payload\n" + + "calls = array.new_int(0)\n" + + f"observed = str.{operation}({arguments})\nplot(close)\n" + ) + line = next(line for line in cpp.splitlines() if line.strip().startswith("observed =")) + source = 'const std::string _pf_search_source = (read_value(std::string("abc"), calls, 1));' + needle = 'const std::string _pf_search_needle = (read_value(std::string("b"), calls, 2));' + searched = f"return _pf_u16_{operation}(_pf_search_source, _pf_search_needle);" + assert line.count(source) == line.count(needle) == line.count(searched) == 1 + assert line.index(source) < line.index(needle) < line.index(searched) + assert line.count("read_value(") == 2 + + +@pytest.mark.parametrize("persistent, history", ((False, False), (True, False), (False, True))) +@pytest.mark.parametrize("string", (False, True), ids=("numeric", "string")) +@pytest.mark.parametrize("parameters", (False, True), ids=("literals", "parameters")) +def test_requested_compound_assignment_uses_its_storage_family( + persistent, history, string, parameters, +): + from tests.test_security_series_timeframe import requested_body + + kind, first, last = ("string", '"a"', '"b"') if string else ("float", "2.5", "1.25") + signature = f"{kind} first, {kind} last" if parameters else "" + initial, appended = ("first", "last") if parameters else (first, last) + arguments = f"{first}, {last}" if parameters else "" + cpp = transpile( + HEADER + f"combine({signature}) =>\n" + + f" {'var ' if persistent else ''}{kind} accumulated = {initial}\n" + + f" accumulated += {appended}\n" + + (" previous = accumulated[1]\n" if history else "") + + " accumulated\n" + + f'joined = request.security(syminfo.tickerid, "60", combine({arguments}))\n' + + "plot(close)\n" + ) + body = requested_body(cpp, 0) + assert ("_pf_u16_concat(" in body) is string + assert (HELPER_DEFINITION in cpp) is string + if string: + assert 'std::string("b")' in body + # The chart helper is emitted too, even when only a request calls it. + # Both bodies must keep the persistent/local string out of arithmetic. + assert "_pf_u16_concat(std::to_string(" not in cpp + for line in body.splitlines(): + if "_pf_u16_concat(" in line: + assert "std::to_string(" not in line + assert "is_na(" not in line + assert "(double)" not in line + else: + assert "1.25" in body STOP_DRIVER = r''' @@ -277,20 +538,323 @@ def test_a_substring_past_the_end_stops_the_run_and_names_the_units(): assert "(which is 5) > this->size() (which is 4)" in stopped -REFUSED_ROWS = [ +RUNTIME_BEGIN_ROWS = [ pytest.param(4, "(which is 4) > this->size() (which is 3)", id="measured-ascii-begin-beyond-length-stops"), - pytest.param(-1, "(which is 18446744073709551615) > this->size() (which is 3)", - id="measured-ascii-begin-negative-stops"), + pytest.param('input.int(-1, "Begin")', + "(which is 18446744073709551615) > this->size() (which is 3)", + id="dynamic-ascii-begin-negative-stops"), ] -@pytest.mark.parametrize("begin, detail", REFUSED_ROWS) +@pytest.mark.parametrize("begin, detail", RUNTIME_BEGIN_ROWS) def test_a_begin_outside_the_text_stops_the_run(begin, detail): - """TradingView refuses these two scripts when it compiles them (their - arguments are constants); the engine stops the run, with the text - std::string::substr gives. Both refuse; the moment is not compared.""" + """Pin the existing native boundary guard and its diagnostic detail. + + A negative input remains a runtime value. The separate constant-argument + test below checks the compiler refusal; a runtime stop cannot satisfy it. + The positive out-of-range constant retains its existing runtime behavior. + """ cpp = transpile(HEADER + f't = str.substring("abc", {begin})\nplot(close)\n') stopped = run_emitted_tu(cpp, STOP_DRIVER, opt="-O2", label="substring begin outside") assert "basic_string::substr" in stopped assert detail in stopped + + +SUBSTRING_CALLS = [ + pytest.param('result_value = str.substring("abc", {begin})', id="positional"), + pytest.param('result_value = str.substring("abc", {begin}, 3)', id="positional-end"), + pytest.param('result_value = str.substring(source="abc", begin_pos={begin})', + id="named"), + pytest.param('result_value = str.substring(end_pos=3, begin_pos={begin}, source="abc")', + id="named-reordered"), + pytest.param('result_value = str.substring("abc", end_pos=3, begin_pos={begin})', + id="mixed-reordered"), +] + + +@pytest.mark.parametrize("check_support", [True, False]) +@pytest.mark.parametrize("call_template", SUBSTRING_CALLS) +@pytest.mark.parametrize("declarations, begin, first_token", [ + pytest.param("", "-1", "-", id="literal"), + pytest.param("", "-(2 + 1)", "-", id="negated-expression"), + pytest.param("", "1 - 4", "1", id="folded-subtraction"), + pytest.param("", "-8 / 2", "-", id="folded-exact-division"), + pytest.param("", "-8 % 3", "-", id="folded-remainder"), + pytest.param("begin = -2\n", "begin", "begin", id="inferred-constant"), + pytest.param("int start = -3\nbegin = start\n", "begin", "begin", + id="typed-alias"), + pytest.param("const int origin = 2\nconst int begin = origin - 5\n", + "begin", "begin", id="const-alias-chain"), +]) +def test_constant_negative_substring_begin_is_refused( + declarations, begin, first_token, call_template, check_support, +): + # TradingView CE10276 refuses the preserved constant-negative source at + # its begin argument. These are unit cases for the general constant rule, + # not additional TradingView measurements or a trade-value oracle. + source = HEADER + declarations + call_template.format(begin=begin) + "\nplot(close)\n" + with pytest.raises(CompileError) as caught: + transpile(source, filename="substring.pine", check_support=check_support) + diagnostic, = caught.value.diagnostics + assert diagnostic.level is Level.ERROR + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = call_template.index("{begin}") + 1 + assert diagnostic.location == SourceLocation( + "substring.pine", 3 + declarations.count("\n"), + column, column + len(first_token), + ) + # Allocation belongs to the catalog generator. Require registration without + # assigning an unreserved code in this test or in the diagnostic emitter. + assert diagnostic.code not in UNCATALOGUED.values() + assert diagnostic.args == {} + + +@pytest.mark.parametrize("body", [ + pytest.param('unused() => str.substring("abc", -1)\nplot(close)\n', + id="unused-body"), + pytest.param('plot(close, title=str.substring("abc", -1))\n', + id="visual-keyword"), +]) +def test_constant_negative_substring_is_checked_before_ignored_calls_are_lowered(body): + source = HEADER + body + with pytest.raises(CompileError) as caught: + transpile(source, filename="substring.pine") + diagnostic, = caught.value.diagnostics + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = body.splitlines()[0].index("-1") + 1 + assert diagnostic.location == SourceLocation("substring.pine", 3, column, column + 1) + + +def test_direct_analyzer_checks_constant_negative_substring_arguments(): + source = HEADER + 'result_value = str.substring("abc", -1)\n' + program = Parser( + Lexer(source, filename="substring.pine").tokenize(), + source=source, filename="substring.pine", + ).parse() + with pytest.raises(CompileError) as caught: + Analyzer(program, filename="substring.pine").analyze() + diagnostic, = caught.value.diagnostics + assert diagnostic.phase is Phase.ANALYZER + assert diagnostic.message == ( + "str.substring() requires 'begin_pos' to be greater than or equal to 0." + ) + column = source.splitlines()[2].index("-1") + 1 + assert diagnostic.location == SourceLocation("substring.pine", 3, column, column + 1) + + +@pytest.mark.parametrize("declarations, begin", [ + pytest.param('begin = input.int(-1, "Begin")\n', "begin", id="input"), + pytest.param("begin = bar_index - 1\n", "begin", id="series"), + pytest.param("begin = -1\nbegin := 0\n", "begin", id="reassigned"), + pytest.param("var int begin = -1\n", "begin", id="persistent"), + pytest.param("int begin = na\n", "begin", id="missing-alias"), + pytest.param("", "int(na) + 1", id="missing-expression"), + pytest.param("", "0", id="zero"), + pytest.param("", "2", id="positive"), +]) +def test_substring_begin_without_negative_constant_proof_keeps_its_lowering( + declarations, begin, +): + # Acceptance and emitted runtime checking only; native value/stop assertions + # remain in the existing missing, edge and runtime-boundary tests. + cpp = transpile( + HEADER + declarations + + f'result_value = str.substring(end_pos=3, source="abc", begin_pos={begin})\n' + + "plot(close)\n" + ) + assert '_PF_STRING_STOP("substring_out_of_range"' in cpp + + +def test_substring_constant_proof_respects_parameter_bindings_and_authored_overloads(): + source = HEADER + ( + "const int begin = -2\n" + 'cut(int begin) => str.substring("abc", begin)\n' + "substring(int begin) => begin + 3\n" + "method substring(int self, int begin) => self + begin\n" + "base = 4\n" + "result_value = cut(1)\n" + "n = substring(-2) + base.substring(-3)\n" + "plot(n)\n" + ) + cpp = transpile(source) + assert '_PF_STRING_STOP("substring_out_of_range"' in cpp + + +def test_substring_negative_end_retains_runtime_validation(): + # This change has no compiler evidence for an end-domain refusal. + cpp = transpile(HEADER + 'result_value = str.substring("abc", 0, -1)\nplot(close)\n') + assert "str.substring: end_pos is less than begin_pos" in cpp + + +@pytest.mark.parametrize("opt", NATIVE_OPTS) +def test_substring_finite_end_before_start_stops_the_run(opt): + # Issue #382 item 9: a finite end before begin stops the run. + # Retain actual native values and errors in stdout for diagnosis. + script = ( + 'j = input.int(1, "End")\n' + 'completed = false\n' + 't = str.substring("abc", 2, j)\n' + 'completed := true\n' + 'plot(close)\n' + ) + cpp = transpile(HEADER + script) + driver = r''' +#include +int main() { + GeneratedStrategy strategy; + strategy.set_input("End", "1"); + Bar bars[3]{ + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067200000LL}, + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067260000LL}, + {1.0, 1.0, 1.0, 1.0, 1.0, 1704067320000LL}, + }; + strategy.run(bars, 3, "1", "1"); + std::cout << "j=" << strategy.j << "\n" + << "completed=" << strategy.completed << "\n" + << "value=" << strategy.t << "\n" + << "error=" << strategy.last_error() << "\n"; + return 0; +} +''' + # Compile/link failures and unexpected process exits propagate unchanged. + # Only the native run's own last_error() can satisfy the stop assertion. + observed = run_emitted_tu( + cpp, driver, opt=opt, label="substring finite end before start", + ) + print(f"native stdout at {opt}:\n{observed}", end="") + position, completed, value, error = observed.split("\n", 3) + assert position == "j=1", observed + assert value.startswith("value="), observed + assert error.startswith("error="), observed + assert "end_pos is less than begin_pos" in error, observed + assert completed == "completed=0", observed + + +def _assert_trim_cases(cases: list[tuple[str, str]], input_shape: str) -> None: + """Observe both the actual Pine source value and its trim result as bytes.""" + script = [] + setup = [] + observe = [] + expected = [] + for index, (text, trimmed) in enumerate(cases): + source = f"trim_source_{index}" + result = f"trim_result_{index}" + if input_shape == "runtime-input": + title = f"Trim text {index}" + # A distinct default makes an ignored override observable. Counted + # UTF-8 bytes keep NUL intact without exercising the known input- + # default truncation defect from item 6. + script.append(f'{source} = input.string("unused default", "{title}")\n') + encoded = text.encode("utf-8") + octets = "".join(f"\\{byte:03o}" for byte in encoded) + setup.append( + f'strategy.set_input("{title}", ' + f'std::string("{octets}", {len(encoded)}));' + ) + else: + assert input_shape == "literal" + # Pine supports \n and \t; \r spells the letter r. Keep a raw CR + # inside the quotes, where the lexer preserves it as string data. + literal = (text.replace("\\", "\\\\").replace('"', '\\"') + .replace("\n", "\\n").replace("\t", "\\t")) + script.append(f'{source} = "{literal}"\n') + script.append(f"{result} = str.trim({source})\n") + observe.append(f"hex(strategy.{source}); hex(strategy.{result});") + expected.extend((text.encode("utf-8").hex() + "\n", + trimmed.encode("utf-8").hex() + "\n")) + script.append("plot(close)\n") + # hex() iterates std::string's bytes, including NUL. Only ASCII reaches + # stdout, so invalid UTF-8 cannot be hidden by decoding or replacement. + # Compare every line, including empty ones and the final newline. + assert _run("".join(script), "\n".join(observe), + setup="\n".join(setup)) == "".join(expected) + + +TRIM_MEASURED_ROWS = [ + pytest.param("\u00a0a\u00a0", "a", id="measured-nbsp-both-ends"), + pytest.param("\u2003a\u2003", "a", id="measured-em-space-both-ends"), + pytest.param("\ufeffa", "a", id="measured-feff-left-end"), + pytest.param("\va\f", "a", id="measured-vertical-tab-and-form-feed"), +] + + +@pytest.mark.parametrize("text, expected", TRIM_MEASURED_ROWS) +def test_trim_measured_end_whitespace(text, expected): + _assert_trim_cases([(text, expected)], "literal") + + +# The issue's explicit classification, not Python's isspace()/strip() or a +# guessed Unicode category. The test shapes below are contract compositions. +TRIM_CHARACTER_ROWS = [ + pytest.param("\u00a0", True, id="removes-u00a0-nbsp"), + pytest.param("\u2003", True, id="removes-u2003-em-space"), + pytest.param("\u3000", True, id="removes-u3000-ideographic-space"), + pytest.param("\u1680", True, id="removes-u1680-ogham-space-mark"), + pytest.param("\u2028", True, id="removes-u2028-line-separator"), + pytest.param("\ufeff", True, id="removes-ufeff-zero-width-no-break-space"), + pytest.param("\v", True, id="removes-u000b-vertical-tab"), + pytest.param("\f", True, id="removes-u000c-form-feed"), + pytest.param(" ", True, id="removes-u0020-space"), + pytest.param("\t", True, id="removes-u0009-tab"), + pytest.param("\n", True, id="removes-u000a-line-feed"), + pytest.param("\r", True, id="removes-u000d-carriage-return"), + pytest.param("\u0085", False, id="keeps-u0085-next-line"), + pytest.param("\u001f", False, id="keeps-u001f-unit-separator"), + pytest.param("\u200b", False, id="keeps-u200b-zero-width-space"), + pytest.param("\u180e", False, id="keeps-u180e-mongolian-vowel-separator"), +] + +TRIM_INPUT_SHAPES = [ + pytest.param("literal", id="contract-derived-literal"), + pytest.param("runtime-input", id="contract-derived-runtime-input"), +] + + +@pytest.mark.parametrize("input_shape", TRIM_INPUT_SHAPES) +@pytest.mark.parametrize("character, remove_at_ends", TRIM_CHARACTER_ROWS) +def test_trim_contract_characters_at_ends_and_inside(character, remove_at_ends, input_shape): + cases = [] + # Two different interiors, one with a NUL followed by an emoji and more + # text: neither a constant "a" nor a C-string prefix can satisfy all rows. + for left, right in (("left", "right"), (f"{CJK}\0", f"{EMOJI}tail")): + body = left + right + interior = left + character + right + for prefix, suffix in ((character * 2, ""), ("", character * 3), + (character * 2, character * 3)): + text = prefix + body + suffix + cases.append((text, body if remove_at_ends else text)) + cases.append((interior, interior)) + text = character + interior + character + cases.append((text, interior if remove_at_ends else text)) + text = " \t" + character + body + character + "\n\r" + cases.append((text, body if remove_at_ends else character + body + character)) + blank_only = character * 3 + cases.append((blank_only, "" if remove_at_ends else blank_only)) + _assert_trim_cases(cases, input_shape) + + +@pytest.mark.parametrize("input_shape", TRIM_INPUT_SHAPES) +def test_trim_contract_mixed_boundaries_and_empty(input_shape): + removed = "\u00a0\u2003\u3000\u1680\u2028\ufeff\v\f \t\n\r" + kept = "\u0085\u001f\u200b\u180e" + body = f"head{removed}tail{CJK}\0{EMOJI}" + cases = [ + ("", ""), + ("plain words", "plain words"), + (removed, ""), + (removed + body + removed, body), + (removed + kept + removed, kept), + ] + for boundary in kept: + # A kept character stops trimming: removable characters behind it are + # interior text and must survive along with every byte after the NUL. + interior = boundary + removed + body + removed + boundary + cases.append((removed + interior + removed, interior)) + _assert_trim_cases(cases, input_shape) diff --git a/tests/test_type_keyword_arguments.py b/tests/test_type_keyword_arguments.py index 7e88ad3e..0b11e01e 100644 --- a/tests/test_type_keyword_arguments.py +++ b/tests/test_type_keyword_arguments.py @@ -23,6 +23,7 @@ def _error(body): @pytest.mark.parametrize("value", [ '"abc"', '"abcdefg"', + '"a😀b"', 'bar_index % 2 == 0 ? "abcd" : "abcdefg"', 'str.tostring(math.sqrt(number = 4.0))', ]) @@ -31,6 +32,20 @@ def test_type_keyword_label_lowers_like_positional(value): assert _transpile(f"pf_value = str.length(string = {value})") == expected +@pytest.mark.parametrize("name", ["source", "value", "pf_unknown"]) +def test_length_rejects_undeclared_parameter_names(name): + # TV accepts string= and independently refuses source= (unknown argument + # and missing string). Every undeclared label uses the same registry rule. + body = f'pf_value = str.length({name} = "a😀b")' + diagnostic = _error(body) + assert diagnostic.code == "PF-E2027" + assert diagnostic.phase == Phase.ANALYZER + assert diagnostic.args == {"key": "str.length", "name": name} + assert diagnostic.location.line == 3 + argument_start = body.index("(") + 1 + assert diagnostic.location.col == body.index(f"{name} =", argument_start) + 1 + + @pytest.mark.parametrize("name", TYPE_NAMES) def test_type_keyword_label_still_requires_a_matching_parameter(name): ordinary = _error("pf_value = ta.sma(pf_unknown = close, length = 5)") diff --git a/tests/test_unicode_case_context.py b/tests/test_unicode_case_context.py new file mode 100644 index 00000000..0eb83160 --- /dev/null +++ b/tests/test_unicode_case_context.py @@ -0,0 +1,39 @@ +"""Unit classification of casing context; Pine values belong to TV probes.""" + +import pytest + +from pineforge_codegen.codegen.unicode_case import _context_flags +from pineforge_codegen.codegen import unicode_case + + +@pytest.mark.parametrize("char, flags", [ + ("A", 1), # Cased + ("\u0301", 2), # Case_Ignorable + ("\u0345", 3), # Cased and Case_Ignorable; skipping takes precedence + (".", 2), + (",", 0), + ("1", 4), # Decimal_Number, not Case_Ignorable or Cased + ("\u0664", 4), + ("\uff16", 4), + ("\U0001d7d8", 4), + ("中", 0), +]) +def test_context_flags_keep_decimal_runs_separate_from_case_properties(char, flags): + assert _context_flags(char) == flags + + +@pytest.mark.parametrize("field, invalid", [ + ("DATA_FORMAT", 0), + ("DATA_VERSION", "0.0.0"), + ("SOURCE_SHA256", ()), + ("SOURCE_BYTE_COUNTS", ()), +]) +def test_casing_data_requires_the_pinned_identity_before_use(monkeypatch, field, invalid): + data = unicode_case._case_data() + unicode_case._case_data.cache_clear() + try: + monkeypatch.setattr(data, field, invalid) + with pytest.raises(RuntimeError, match="pinned UCD inputs"): + unicode_case._case_data() + finally: + unicode_case._case_data.cache_clear() diff --git a/tests/test_unicode_case_data_generation.py b/tests/test_unicode_case_data_generation.py new file mode 100644 index 00000000..e795f6db --- /dev/null +++ b/tests/test_unicode_case_data_generation.py @@ -0,0 +1,128 @@ +"""Unit targets for the offline UCD parser and its pinned serialization.""" + +import ast +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from scripts import gen_unicode_case_data as gen + + +def _row(point, name, category, *, upper="", lower=""): + fields = [f"{point:04X}", name, category, "0", "L", + "", "", "", "", "N", "", "", upper, lower, ""] + return ";".join(fields) + "\n" + + +def _inputs(): + # Synthetic UCD records test parsing, not the casing of a Pine script. + ordinary = ( + _row(0x31, "TEST DECIMAL", "Nd") + + _row(0x41, "TEST CAPITAL", "Lu", lower="0061") + + _row(0x61, "TEST SMALL", "Ll", upper="0041") + ) + special = ( + "0061; 0061; 0042 0043; 0042 0043;\n" + "0041; 0062; 0041; 0041; Final_Sigma;\n" + "0061; ; 0049; 0049; tr After_I;\n" + ) + properties = ( + "0041..0042 ; Cased\n" + "002E ; Case_Ignorable\n" + "0042 ; Case_Ignorable\n" + ) + return ordinary, special, properties + + +def test_ucd_parser_combines_simple_full_and_contextual_data_without_locale_tailoring(): + data = gen.parse_ucd(*_inputs()) + assert data.lower == ((0x41, 1, 0x61),) + assert data.upper == ((0x61, 2, 0x42, 0x43),) + assert data.lower_final_sigma == ((0x41, 1, 0x62),) + assert data.context == ( + (0x2E, 0x2E, 2), (0x31, 0x31, 4), + (0x41, 0x41, 1), (0x42, 0x42, 3), + ) + assert data.excluded_locale_rules == ("0061: tr After_I",) + + +def test_unicode_data_first_last_ranges_apply_properties_to_the_whole_interval(): + flags = bytearray(0x110000) + records = (_row(0x100, "", "Nd") + + _row(0x105, "", "Nd")) + gen._unicode_data(records, flags) + assert gen._context_ranges(flags) == ((0x100, 0x105, 4),) + with pytest.raises(gen.DataError, match="unterminated range"): + gen._unicode_data(_row(0x100, "", "Nd"), flags) + with pytest.raises(gen.DataError, match="inconsistent range"): + gen._unicode_data(records.replace("", ""), flags) + + +@pytest.mark.parametrize("name,size,digest", gen.INPUT_PINS) +def test_input_identity_rejects_truncated_and_same_size_wrong_content(name, size, digest): + with pytest.raises(gen.DataError, match="size/SHA-256"): + gen._verified_text(name, b"") + with pytest.raises(gen.DataError, match="size/SHA-256"): + gen._verified_text(name, b"\n" * size) + + +def test_unknown_root_condition_is_an_error_instead_of_a_dropped_override(): + ordinary, special, properties = _inputs() + with pytest.raises(gen.DataError, match="unsupported root condition"): + gen.parse_ucd(ordinary, special.replace("Final_Sigma", "Future_Context"), properties) + + +def test_generated_module_round_trips_tables_and_ignores_irrelevant_record_order(): + ordinary, special, properties = _inputs() + data = gen.parse_ucd(ordinary, special, properties) + encoded = gen.render_module(data) + # These two source sections are sets, unlike ordered UnicodeData records. + reordered = gen.parse_ucd( + ordinary, "\n".join(reversed(special.splitlines())) + "\n", + "002E ; Case_Ignorable\n0042 ; Case_Ignorable\n0041..0042 ; Cased\n", + ) + assert gen.render_module(reordered) == encoded + tree = ast.parse(encoded.decode("ascii")) + assignments = { + node.targets[0].id: ast.literal_eval(node.value) + for node in tree.body if isinstance(node, ast.Assign) + } + assert assignments["LOWER_ROWS"] == data.lower + assert assignments["UPPER_ROWS"] == data.upper + assert assignments["LOWER_FINAL_SIGMA_ROWS"] == data.lower_final_sigma + assert assignments["CONTEXT_RANGES"] == data.context + assert assignments["SOURCE_SHA256"] == tuple( + (name, digest) for name, _, digest in gen.INPUT_PINS + ) + + +@pytest.mark.parametrize("filename", [ + "/no-access/.ssh/UnicodeData.txt", + "/no-access/.aws/UnicodeData.txt", + "/no-access/.config/pineforge/UnicodeData.txt", + "/no-access/.hidden-tool/UnicodeData.txt", + "/no-access/private.env/UnicodeData.txt", +]) +def test_sensitive_paths_are_refused_by_name_before_any_filesystem_query(monkeypatch, filename): + def no_filesystem_query(*args, **kwargs): + pytest.fail("protected name reached a filesystem query") + + monkeypatch.setattr(Path, "is_symlink", no_filesystem_query) + with pytest.raises(gen.DataError, match="protected file path"): + gen._checked_path(filename, "UnicodeData.txt") + + +def test_symlink_input_is_refused_without_rejecting_a_regular_input(): + # A neutral temporary root reaches the symlink check even when the + # checkout and pytest's own temporary directory have hidden components. + with TemporaryDirectory(prefix="unicode-case-path-", dir=Path("/tmp").resolve()) as directory: + root = Path(directory) + source = root / "source" / "UnicodeData.txt" + source.parent.mkdir() + source.touch() + assert gen._checked_path(str(source), "UnicodeData.txt") == source + link = root / "UnicodeData.txt" + link.symlink_to(source) + with pytest.raises(gen.DataError, match="symlink path"): + gen._checked_path(str(link), "UnicodeData.txt") diff --git a/tests/test_unicode_regex_lowering.py b/tests/test_unicode_regex_lowering.py new file mode 100644 index 00000000..c46908b9 --- /dev/null +++ b/tests/test_unicode_regex_lowering.py @@ -0,0 +1,71 @@ +"""Unit checks of the byte-pattern compiler, not Pine replay expectations.""" + +from pineforge_codegen import transpile +from tests._compile import run_emitted_tu + + +def _compiler_tu(): + return transpile('//@version=6\nstrategy("regex compiler")\n' + 'result = str.match("abc", "b")\n') + + +def test_pattern_compiler_preserves_capture_targets_under_whole_match_group(): + driver = r''' +#include +static void check(const std::string& pattern, const std::string& subject, + const std::vector& captures) { + const auto units = _pf_regex::_pf_characters(pattern); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + const std::regex re(compiled); + std::smatch match; + assert(std::regex_search(subject, match, re)); + assert(match.size() == captures.size() + 2); + assert(match[1].str() == match[0].str()); + for (std::size_t i = 0; i < captures.size(); ++i) + assert(match[i + 2].str() == captures[i]); +} +int main() { + check(R"((x)(y)\2\1)", "xyyx", {"x", "y"}); + check(R"((?:q)(x)(?=y)(y)\2\1)", "qxyyx", {"x", "y"}); + check(R"((a)(b)(c)(d)(e)(f)(g)(h)(i)\9)", "abcdefghii", + {"a", "b", "c", "d", "e", "f", "g", "h", "i"}); + check(R"((a)(b)(c)(d)(e)(f)(g)(h)(i)(j)\10)", "abcdefghijj", + {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}); + check(R"((é+)\1)", "éééé", {"éé"}); + check(R"(\(x\))", "(x)", {}); + check(R"([(]x[)])", "(x)", {}); + // Two backslashes and the digit are a literal, not a reference to shift. + check(R"(\\1)", R"(\1)", {}); + const auto units = _pf_regex::_pf_characters(R"(\99999999999999999999)"); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + assert(compiled == R"((\100000000000000000000))"); +} +''' + assert run_emitted_tu(_compiler_tu(), driver, opt="-O0", + label="regex capture compiler") == "" + + +def test_whole_match_wrapper_cannot_balance_invalid_user_parentheses(): + driver = r''' +#include +int main() { + for (const std::string pattern : {")(", ")a(", "(", "a)", "(?:x"}) { + const auto units = _pf_regex::_pf_characters(pattern); + bool refused = false; + try { (void)_pf_regex::_pf_pattern(units)._pf_compile(); } + catch (const _pf_regex::_pf_invalid&) { refused = true; } + assert(refused); + } + // A synthetic group cannot make a nonexistent user reference valid. + const auto units = _pf_regex::_pf_characters(R"(\1)"); + const std::string compiled = _pf_regex::_pf_pattern(units)._pf_compile(); + bool refused = false; + try { (void)std::regex(compiled); } + catch (const std::regex_error& error) { + refused = error.code() == std::regex_constants::error_backref; + } + assert(refused); +} +''' + assert run_emitted_tu(_compiler_tu(), driver, opt="-O0", + label="regex wrapper grammar") == ""