diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index d751b58f7..3d9fbca94 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -27,7 +27,13 @@ name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE) dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE) division_re = re.compile(r'/=?') -regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL) +# A regexp literal cannot contain an ECMAScript line terminator, and a backslash +# cannot escape one either: `\` followed by a line terminator is a syntax error +# inside a regexp literal, so it must not continue the literal on the next line. +# https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators +regex_re = re.compile( + r'/(?:[^/\\\r\n\u2028\u2029]*(?:\\[^\r\n\u2028\u2029][^/\\\r\n\u2028\u2029]*)*)/[a-zA-Z]*', +) line_re = re.compile(r'(\r\n|\n|\r)') line_join_re = re.compile(r'\\' + line_re.pattern) uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index af6dfb144..b9eee0457 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -122,7 +122,10 @@ def test_jsx_extraction(jsx_enabled): if jsx_enabled: assert messages == EXPECTED_JSX_MESSAGES else: - assert messages != EXPECTED_JSX_MESSAGES + # Without the jsx option the regex literals in the source used to + # swallow following messages (issue #640); with that lexer bug + # fixed, plain JavaScript extraction finds them all too. + assert messages == EXPECTED_JSX_MESSAGES def test_dotted_keyword_extract(): @@ -189,3 +192,20 @@ def test_inside_nested_template_string(): ) assert messages == [(1, 'Greetings!', [], None), (1, 'This is a lovely evening.', [], None), (1, 'The day is really nice!', [], None)] + + +@pytest.mark.parametrize("terminator", ["\n", "\r", "\r\n", "\u2028", "\u2029"]) +def test_regexp_literals_do_not_swallow_messages(terminator): + # https://github.com/python-babel/babel/issues/640: a bogus fallback + # regex token used to span line terminators and silently drop every + # gettext() call inside the swallowed region. Applies to every ECMAScript + # line terminator, not just U+000A. + source = terminator.join(( + 'alert(gettext("First message"));', + 'var expr = />/g;', + 'alert(gettext("Second message"));', + "var expr2 = /'/g;", + 'alert(gettext("Third message"));', + )).encode() + messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})] + assert messages == ["First message", "Second message", "Third message"] diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 00afc78ae..3f4ae5d6c 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -1,3 +1,5 @@ +import pytest + from babel.messages import jslexer @@ -123,3 +125,42 @@ def test_jsx(): ('jsx_tag', '', 8), ] + + +LINE_TERMINATORS = ["\n", "\r", "\r\n", "\u2028", "\u2029"] + + +@pytest.mark.parametrize("terminator", LINE_TERMINATORS) +def test_regexp_literal_cannot_span_lines(terminator): + # A JS regex literal may not contain any of the ECMAScript line terminators; + # the fallback regex match must stop at one instead of swallowing the rest + # of the file (issue #640). + source = terminator.join(( + 'alert(gettext("x"));', + 'var e = />/g;', + 'alert(gettext("y"));', + )) + tokens = list(jslexer.tokenize(source)) + assert any(kind == 'string' and value == '"y"' for kind, value, _ in tokens) + for kind, value, _lineno in tokens: + assert not any(t in value for t in LINE_TERMINATORS), \ + f"{kind} token spans lines: {value!r}" + + +@pytest.mark.parametrize("terminator", LINE_TERMINATORS) +def test_regexp_literal_cannot_continue_after_a_backslash(terminator): + # A backslash is not a line continuation inside a regexp literal, so the + # literal must not be continued past the terminator either. + source = f'var e = /a\\{terminator}b/;\nalert(gettext("still here"));' + tokens = list(jslexer.tokenize(source)) + assert any(kind == 'string' and value == '"still here"' for kind, value, _ in tokens) + for kind, value, _lineno in tokens: + if kind != 'regexp': + continue + assert not any(t in value for t in LINE_TERMINATORS), f"regexp spans lines: {value!r}" + + +def test_ordinary_regexp_literals_still_match(): + # Escaped slashes and flags are unaffected by the line terminator guard. + tokens = list(jslexer.tokenize(r'var e = /a\/b+c/gi;')) + assert ('regexp', r'/a\/b+c/gi', 1) in tokens