From cca57170c131540444a7cf20bdf1768d13e09ebe Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 1 Sep 2026 22:51:37 -0700 Subject: [PATCH 1/3] fix: two rules that ignored they were writing expression source (#340, #341) `inside_dollar_string` says the text being produced is part of an expression rather than a value for the caller. `StringRule` checks it and keeps its quotes, because `upper("x")` becoming `upper(x)` asks for a variable nobody declared. Two rules did not. A heredoc argument was spliced in bare: `upper(< Date: Mon, 7 Sep 2026 14:31:55 +0200 Subject: [PATCH 2/3] fix: keep the value-form fixes, drop the default-path change (#340, #341) The branch also stopped quoting a heredoc that sits inside an expression on the default path, which neither #340 nor #341 asks for -- both are scoped to `strip_string_quotes`. It regressed the round trip: `dumps(loads(...))` of a heredoc argument raised `UnexpectedToken`, because the emitted `trimspace(< --- CHANGELOG.md | 4 +- hcl2/rules/strings.py | 5 - test/unit/rules/test_expression_source.py | 132 ++++++++++++++++++---- test/unit/test_api.py | 10 +- 4 files changed, 118 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fe62afc..4eae4695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed -- A heredoc inside an expression is written as a string rather than spliced in bare. `SerializationContext.inside_dollar_string` says the text being produced is expression source, and `StringRule` checks it for exactly this reason; the heredoc rules did not, so `upper(< "X\n", so the argument is a string + "%{ if local.x == "y" }t%{ endif }" -> "t", with plain quotes + +The escaped spelling this grammar also accepts, `\"y\"` inside a directive, +Terraform rejects outright with "Invalid character". That divergence is filed +as #341's sibling, #353; the tests below only pin that the value form stops +mangling it into a reference, which is what #341 asks for. """ from unittest import TestCase -from hcl2.api import loads +from hcl2.api import dumps, loads from hcl2.utils import SerializationOptions VALUE = SerializationOptions(preserve_heredocs=False, strip_string_quotes=True) QUOTED = SerializationOptions(strip_string_quotes=True) SOURCE = SerializationOptions(preserve_heredocs=False) +DEFAULT = SerializationOptions() class TestAHeredocInsideAnExpression(TestCase): """#340: the body was spliced in bare, so it read as a reference.""" + def value(self, source: str) -> str: + return loads(source, serialization_options=VALUE)["a"] + def test_it_stays_a_string(self): - self.assertEqual(loads("a = upper(< str: + return loads(source, serialization_options=VALUE)["a"] + + def test_a_nested_call(self): + self.assertEqual(self.value("a = upper(lower(< Date: Wed, 23 Sep 2026 11:43:09 -0700 Subject: [PATCH 3/3] fix: a parenthesised term writes what it wraps as expression source `(...)` is written as `${(...)}`, but the term serialized its inside as a value, so literals came back in Python's spelling: `(true)` as `${(True)}` and `(null)` as `${(None)}` on the default options, which dumps wrote back as references OpenTofu rejects, and a tuple or object inside as a repr that did not parse. With strip_string_quotes, `("s")` lost its quotes. Every changed output was one no reader could evaluate; each round trip now evaluates, in OpenTofu v1.12.6, to the value the source does. --- CHANGELOG.md | 1 + hcl2/rules/expressions.py | 11 ++++- test/unit/rules/test_expression_source.py | 60 ++++++++++++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e93bf13b..ec5be326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `strip_string_quotes` now writes a heredoc inside an expression as a string instead of splicing its body in bare. `StringRule` checks `inside_dollar_string` to keep its quotes for exactly this reason; the heredoc rules did not, so `upper(< ExpressionRule: def serialize(self, options=SerializationOptions(), context=SerializationContext()) -> Any: """Serialize, handling parenthesized expression wrapping.""" - with context.modify(inside_parentheses=self.parentheses or context.inside_parentheses): + # A parenthesised term is written as `${(...)}`, so what it wraps is + # expression source, exactly as a function's arguments are. Serialized + # as a value, `(true)` came back as `${(True)}` and `(null)` as + # `${(None)}` -- Python's spelling, which OpenTofu reads as references + # to undeclared variables -- and a tuple or object inside came back as + # a Python repr that did not parse. + with context.modify( + inside_parentheses=self.parentheses or context.inside_parentheses, + inside_dollar_string=self.parentheses or context.inside_dollar_string, + ): result = self.expression.serialize(options, context) if self.parentheses: diff --git a/test/unit/rules/test_expression_source.py b/test/unit/rules/test_expression_source.py index d6a31122..d40ef27e 100644 --- a/test/unit/rules/test_expression_source.py +++ b/test/unit/rules/test_expression_source.py @@ -9,7 +9,10 @@ Both defects are in the value form only -- `strip_string_quotes=True`. What the other modes emit is unchanged; `TestTheOtherModesAreUntouched` states so, including the round trip, because that is the half a change here can break -without any of the assertions above noticing. +without any of the assertions above noticing. The one exception is a +parenthesised term, `TestAParenthesisedTermIsExpressionSource`, whose default +output was itself not HCL -- `(true)` came back as `${(True)}` -- so fixing it +changes only output that no reader could evaluate. Checked against Terraform v1.11.4: @@ -180,3 +183,58 @@ def test_the_value_dict_still_round_trips(self): with self.subTest(source=source): written = dumps(loads(source, serialization_options=VALUE)) self.assertEqual(dumps(loads(written)), written) + + +class TestAParenthesisedTermIsExpressionSource(TestCase): + r"""`(...)` is written as `${(...)}`, so what it wraps is expression source. + + The term wrapped its result in `${(` and `)}` but serialized the inside as + a value, so every literal in it came back in Python's spelling rather than + HCL's. This was not confined to the value form: on the default options + `(true)` came back as `${(True)}` and `(null)` as `${(None)}`, and `dumps` + wrote those back as `(True)` and `(None)` -- references to variables + nobody declared, which OpenTofu v1.12.6 rejects with "Invalid reference" + where the source evaluates to `true` and `null`. A tuple or an object + inside the parentheses came back as a Python repr, which `dumps` could not + parse at all. With `strip_string_quotes`, `("s")` lost its quotes and became + `${(s)}`, another reference. + + Checked against OpenTofu v1.12.6 (`jsonencode` of each local): + + (true) -> "true" (null) -> "null" ("s") -> "\"s\"" + ([1, "a"]) -> "[1,\"a\"]" ({a = 1}) -> "{\"a\":1}" + """ + + CASES = { + "(true)": "${(true)}", + "(null)": "${(null)}", + '("s")': '${("s")}', + "(1)": "${(1)}", + '([1, "a"])': '${([1, "a"])}', + "(1 + 2)": "${(1 + 2)}", + "((true))": "${((true))}", + } + + def test_default_options(self): + for source, expected in self.CASES.items(): + with self.subTest(source=source): + self.assertEqual(loads(f"x = {source}\n")["x"], expected) + + def test_the_value_form(self): + for source, expected in self.CASES.items(): + with self.subTest(source=source): + self.assertEqual(loads(f"x = {source}\n", serialization_options=QUOTED)["x"], expected) + + def test_an_object_inside_parentheses(self): + self.assertEqual(loads("x = ({a = 1})\n")["x"], "${({a = 1})}") + + def test_inside_a_tuple(self): + self.assertEqual(loads("x = [(true)]\n")["x"], ["${(true)}"]) + + def test_the_round_trip_reads_back_the_same_value(self): + for source in (*self.CASES, "({a = 1})", "[(true)]"): + for options in (DEFAULT, QUOTED, VALUE): + with self.subTest(source=source, options=options): + original = loads(f"x = {source}\n", serialization_options=options) + written = dumps(original) + self.assertEqual(loads(written, serialization_options=options), original)