From 2ef3aefb0aaa2ee79157a70a9e2c08063e0c048e Mon Sep 17 00:00:00 2001 From: simpleqt <2015900818@qq.com> Date: Fri, 16 Oct 2026 22:01:33 +0800 Subject: [PATCH] jslexer: stop regexp literals from spanning lines The tokenizer matched a JS regexp literal with re.DOTALL, so an unterminated match could run past line terminators and swallow the rest of the file. A literal like /'/ inside a ternary made extract_javascript drop every gettext message that followed it (issue #640). A JS regexp literal cannot contain an ECMAScript line terminator - CR, LF, U+2028 or U+2029 - and a backslash does not continue one onto the next line, so drop DOTALL and exclude all four terminators from the ordinary and the escaped character classes. This restores extraction of the messages after such a literal; test_jsx_extraction now expects the same messages with and without the jsx option. Signed-off-by: simpleqt <2015900818@qq.com> --- babel/messages/jslexer.py | 8 +++++- tests/messages/test_js_extract.py | 22 ++++++++++++++++- tests/messages/test_jslexer.py | 41 +++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index d751b58f7..3d9fbca94 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -27,7 +27,13 @@ name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE) dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE) division_re = re.compile(r'/=?') -regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL) +# A regexp literal cannot contain an ECMAScript line terminator, and a backslash +# cannot escape one either: `\` followed by a line terminator is a syntax error +# inside a regexp literal, so it must not continue the literal on the next line. +# https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators +regex_re = re.compile( + r'/(?:[^/\\\r\n\u2028\u2029]*(?:\\[^\r\n\u2028\u2029][^/\\\r\n\u2028\u2029]*)*)/[a-zA-Z]*', +) line_re = re.compile(r'(\r\n|\n|\r)') line_join_re = re.compile(r'\\' + line_re.pattern) uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index af6dfb144..b9eee0457 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -122,7 +122,10 @@ def test_jsx_extraction(jsx_enabled): if jsx_enabled: assert messages == EXPECTED_JSX_MESSAGES else: - assert messages != EXPECTED_JSX_MESSAGES + # Without the jsx option the regex literals in the source used to + # swallow following messages (issue #640); with that lexer bug + # fixed, plain JavaScript extraction finds them all too. + assert messages == EXPECTED_JSX_MESSAGES def test_dotted_keyword_extract(): @@ -189,3 +192,20 @@ def test_inside_nested_template_string(): ) assert messages == [(1, 'Greetings!', [], None), (1, 'This is a lovely evening.', [], None), (1, 'The day is really nice!', [], None)] + + +@pytest.mark.parametrize("terminator", ["\n", "\r", "\r\n", "\u2028", "\u2029"]) +def test_regexp_literals_do_not_swallow_messages(terminator): + # https://github.com/python-babel/babel/issues/640: a bogus fallback + # regex token used to span line terminators and silently drop every + # gettext() call inside the swallowed region. Applies to every ECMAScript + # line terminator, not just U+000A. + source = terminator.join(( + 'alert(gettext("First message"));', + 'var expr = />/g;', + 'alert(gettext("Second message"));', + "var expr2 = /'/g;", + 'alert(gettext("Third message"));', + )).encode() + messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})] + assert messages == ["First message", "Second message", "Third message"] diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 00afc78ae..3f4ae5d6c 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -1,3 +1,5 @@ +import pytest + from babel.messages import jslexer @@ -123,3 +125,42 @@ def test_jsx(): ('jsx_tag', '', 8), ] + + +LINE_TERMINATORS = ["\n", "\r", "\r\n", "\u2028", "\u2029"] + + +@pytest.mark.parametrize("terminator", LINE_TERMINATORS) +def test_regexp_literal_cannot_span_lines(terminator): + # A JS regex literal may not contain any of the ECMAScript line terminators; + # the fallback regex match must stop at one instead of swallowing the rest + # of the file (issue #640). + source = terminator.join(( + 'alert(gettext("x"));', + 'var e = />/g;', + 'alert(gettext("y"));', + )) + tokens = list(jslexer.tokenize(source)) + assert any(kind == 'string' and value == '"y"' for kind, value, _ in tokens) + for kind, value, _lineno in tokens: + assert not any(t in value for t in LINE_TERMINATORS), \ + f"{kind} token spans lines: {value!r}" + + +@pytest.mark.parametrize("terminator", LINE_TERMINATORS) +def test_regexp_literal_cannot_continue_after_a_backslash(terminator): + # A backslash is not a line continuation inside a regexp literal, so the + # literal must not be continued past the terminator either. + source = f'var e = /a\\{terminator}b/;\nalert(gettext("still here"));' + tokens = list(jslexer.tokenize(source)) + assert any(kind == 'string' and value == '"still here"' for kind, value, _ in tokens) + for kind, value, _lineno in tokens: + if kind != 'regexp': + continue + assert not any(t in value for t in LINE_TERMINATORS), f"regexp spans lines: {value!r}" + + +def test_ordinary_regexp_literals_still_match(): + # Escaped slashes and flags are unaffected by the line terminator guard. + tokens = list(jslexer.tokenize(r'var e = /a\/b+c/gi;')) + assert ('regexp', r'/a\/b+c/gi', 1) in tokens