From e979a619af1e338ad4b283093d10ca3a4bbe668c Mon Sep 17 00:00:00 2001 From: gstarr-cigna Date: Tue, 18 Aug 2026 23:20:52 -0400 Subject: [PATCH] fix(decoder): normalize delimiter names in decode options The CLI passes delimiter names ('comma'/'tab'/'pipe') as default_delimiter, but DecoderOptions used the value directly as the split character, so tabular rows were never split on the literal string 'comma'. Normalize names to their character form, matching what EncoderOptions already does, so CLI decode round-trips correctly. --- tests/test_decoder.py | 17 +++++++++++++++++ toon/decoder.py | 11 ++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_decoder.py b/tests/test_decoder.py index ec10f91..cf28683 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -88,6 +88,23 @@ def test_decode_array_delimiters(): assert decode('numbers: [1|2|3]') == {'numbers': [1, 2, 3]} +def test_decode_delimiter_name_normalization(): + """Test that delimiter names ('comma', 'tab', 'pipe') are normalized like the encoder does.""" + toon = """products[2]{id,name,price}: + 101,Laptop Pro,1299 + 102,Magic Mouse,79""" + expected = { + 'products': [ + {'id': 101, 'name': 'Laptop Pro', 'price': 1299}, + {'id': 102, 'name': 'Magic Mouse', 'price': 79}, + ] + } + + # Named delimiter (as passed by the CLI) must behave like the character form + assert decode(toon, {'default_delimiter': 'comma'}) == expected + assert decode(toon, {'default_delimiter': ','}) == expected + + def test_decode_tabular_array(): """Test decoding of tabular arrays.""" toon = """users[2]{id,name,role}: diff --git a/toon/decoder.py b/toon/decoder.py index 28f4e22..798e480 100644 --- a/toon/decoder.py +++ b/toon/decoder.py @@ -5,6 +5,7 @@ COMMA, TAB, PIPE, COLON, QUOTE, NEWLINE, SPACE, DEFAULT_DELIMITER, DEFAULT_STRICT, EXPAND_PATHS_OFF, EXPAND_PATHS_SAFE, + DELIMITER_TAB, DELIMITER_PIPE, DELIMITER_COMMA, LEFT_BRACKET, RIGHT_BRACKET, LEFT_BRACE, RIGHT_BRACE ) from .utils import unescape_string, parse_literal @@ -60,8 +61,16 @@ def __init__( Args: strict: Validate structure strictly expand_paths: Path expansion mode ('off' | 'safe') - default_delimiter: Default delimiter for arrays + default_delimiter: Default delimiter for arrays (',' | 'tab' | 'pipe') """ + # Normalize delimiter names to their character form + if default_delimiter == DELIMITER_TAB: + default_delimiter = TAB + elif default_delimiter == DELIMITER_PIPE: + default_delimiter = PIPE + elif default_delimiter == DELIMITER_COMMA: + default_delimiter = COMMA + self.strict = strict self.expand_paths = expand_paths self.default_delimiter = default_delimiter