diff --git a/CHANGELOG.md b/CHANGELOG.md index b8fda71d..c1fb7c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Python 3.14 is now tested and declared as supported. No source changes were needed; the full suite passes on 3.14 as-is. +### Fixed + +- Parse heredocs whose closing marker carries trailing whitespace, such as `EOF `. Both heredoc terminals required the newline to follow the delimiter immediately, so the marker went unrecognised, the heredoc ran on to a later one, and the parse failed pointing at an unrelated line. Terraform ends a heredoc at any line holding the delimiter and nothing else that matters, so such a file parses everywhere else. One input changes meaning: a body line consisting of the delimiter plus trailing whitespace now closes the heredoc rather than being content, as it does in Terraform. Whitespace here means what it does to Terraform's scanner -- Go's `unicode.IsSpace`, so a trailing non-breaking space or form feed closes the heredoc too, and a line led or trailed by U+001C-U+001F, which Python's `\s` counts and Go does not, stays body text as it does in OpenTofu. Thanks, @livingstaccato ([#349](https://github.com/amplify-education/python-hcl2/pull/349)) + ## \[8.1.4\] - 2026-09-08 ### Fixed diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark index 13ddb006..e857af54 100644 --- a/hcl2/hcl2.lark +++ b/hcl2/hcl2.lark @@ -89,8 +89,15 @@ COLONS: "::" // to a later marker. The delimiter itself is `[a-zA-Z][a-zA-Z0-9._-]*` — the // trailing `*` rather than `+` because the spec's Identifier permits a single // character, so `<[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/ -HEREDOC_TEMPLATE_TRIM : /<<-(?P[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/ +// The whitespace around the closing delimiter is what Terraform's scanner +// trims: Go's `unicode.IsSpace`, which is Python's `\s` less U+001C-U+001F. +// So the trailing whitespace an editor may leave on that line still closes +// the heredoc -- a non-breaking space or a form feed as much as a space -- +// while `\x1cEOF` and `EOF x` stay body text, as they do in OpenTofu. Because +// the body group is lazy, the earliest such line closes it, which is also +// where Terraform closes. +HEREDOC_TEMPLATE : /<<(?P[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??[^\S\n\x1c-\x1f]*(?P=heredoc)[^\S\n\x1c-\x1f]*\r?\n/ +HEREDOC_TEMPLATE_TRIM : /<<-(?P[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??[^\S\n\x1c-\x1f]*(?P=heredoc_trim)[^\S\n\x1c-\x1f]*\r?\n/ // Ignore whitespace (but not newlines, as they're significant in HCL). // \r is ignored too so CRLF line endings (\r\n) parse the same as LF: the diff --git a/test/unit/test_heredoc_marker_whitespace.py b/test/unit/test_heredoc_marker_whitespace.py new file mode 100644 index 00000000..d09ed827 --- /dev/null +++ b/test/unit/test_heredoc_marker_whitespace.py @@ -0,0 +1,198 @@ +# pylint: disable=C0103,C0114,C0115,C0116 +r"""A closing marker may carry trailing whitespace (GH #343). + +The spec puts the delimiter "alone on its own line", and Terraform's scanner +ends the heredoc at a line holding the word and nothing else that matters -- +trailing spaces and tabs included. `HEREDOC_TEMPLATE` required the newline to +follow the word immediately, so `EOF ` was not a marker: the heredoc ran on, +swallowed the rest of the file, and the parse failed with an error pointing +somewhere else entirely. + +Trailing whitespace is invisible, survives copy-paste, and is left behind by +editors that do not trim it, so a file someone has been running through +Terraform for months could fail here. + +Every expectation below was checked against Terraform v1.11.4: + + < "body\n", and the next attribute reads + < "", and b reads as 1 + < "EOFX\nEOF x\nbody\n" + +The value assertions compare a marker carrying trailing whitespace against the +same heredoc without it, rather than pinning an absolute string. The two must +agree whatever the body-value rules are, so stating it as an equality keeps +these tests honest across the separate fixes to those rules (GH #326). +""" + +from unittest import TestCase + +from hcl2.api import loads, parses_to_tree, reconstruct, transform +from hcl2.utils import SerializationOptions + +VALUE = SerializationOptions(preserve_heredocs=False, strip_string_quotes=True) + + +def value_of(src: str) -> str: + """Return the body that the heredoc assigned to `a` evaluates to.""" + return loads(src, serialization_options=VALUE)["a"] + + +class TestATrailingSpaceClosesTheHeredoc(TestCase): + """The attribute after the heredoc is reached, so the marker closed it.""" + + def test_spaces(self): + self.assertEqual(loads("a = <