jslexer: stop regexp literals from spanning lines

babel/messages/jslexer.py

@@ -27,7 +27,13 @@
 name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE)
 dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE)
 division_re = re.compile(r'/=?')
-regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL)
+# A regexp literal cannot contain an ECMAScript line terminator, and a backslash
+# cannot escape one either: `\` followed by a line terminator is a syntax error
+# inside a regexp literal, so it must not continue the literal on the next line.
+# https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators
+regex_re = re.compile(
+    r'/(?:[^/\\\r\n\u2028\u2029]*(?:\\[^\r\n\u2028\u2029][^/\\\r\n\u2028\u2029]*)*)/[a-zA-Z]*',
+)
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')

tests/messages/test_js_extract.py

@@ -122,7 +122,10 @@ def test_jsx_extraction(jsx_enabled):
     if jsx_enabled:
         assert messages == EXPECTED_JSX_MESSAGES
     else:
-        assert messages != EXPECTED_JSX_MESSAGES
+        # Without the jsx option the regex literals in the source used to
+        # swallow following messages (issue #640); with that lexer bug
+        # fixed, plain JavaScript extraction finds them all too.
+        assert messages == EXPECTED_JSX_MESSAGES
 
 
 def test_dotted_keyword_extract():
@@ -189,3 +192,20 @@ def test_inside_nested_template_string():
     )
 
     assert messages == [(1, 'Greetings!', [], None), (1, 'This is a lovely evening.', [], None), (1, 'The day is really nice!', [], None)]
+
+
+@pytest.mark.parametrize("terminator", ["\n", "\r", "\r\n", "\u2028", "\u2029"])
+def test_regexp_literals_do_not_swallow_messages(terminator):
+    # https://github.com/python-babel/babel/issues/640: a bogus fallback
+    # regex token used to span line terminators and silently drop every
+    # gettext() call inside the swallowed region.  Applies to every ECMAScript
+    # line terminator, not just U+000A.
+    source = terminator.join((
+        'alert(gettext("First message"));',
+        'var expr = />/g;',
+        'alert(gettext("Second message"));',
+        "var expr2 = /'/g;",
+        'alert(gettext("Third message"));',
+    )).encode()
+    messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})]
+    assert messages == ["First message", "Second message", "Third message"]

tests/messages/test_jslexer.py

@@ -1,3 +1,5 @@
+import pytest
+
 from babel.messages import jslexer
 
 
@@ -123,3 +125,42 @@ def test_jsx():
         ('jsx_tag', '</comp2', 8),
         ('operator', '>', 8),
     ]
+
+
+LINE_TERMINATORS = ["\n", "\r", "\r\n", "\u2028", "\u2029"]
+
+
+@pytest.mark.parametrize("terminator", LINE_TERMINATORS)
+def test_regexp_literal_cannot_span_lines(terminator):
+    # A JS regex literal may not contain any of the ECMAScript line terminators;
+    # the fallback regex match must stop at one instead of swallowing the rest
+    # of the file (issue #640).
+    source = terminator.join((
+        'alert(gettext("x"));',
+        'var e = />/g;',
+        'alert(gettext("y"));',
+    ))
+    tokens = list(jslexer.tokenize(source))
+    assert any(kind == 'string' and value == '"y"' for kind, value, _ in tokens)
+    for kind, value, _lineno in tokens:
+        assert not any(t in value for t in LINE_TERMINATORS), \
+            f"{kind} token spans lines: {value!r}"
+
+
+@pytest.mark.parametrize("terminator", LINE_TERMINATORS)
+def test_regexp_literal_cannot_continue_after_a_backslash(terminator):
+    # A backslash is not a line continuation inside a regexp literal, so the
+    # literal must not be continued past the terminator either.
+    source = f'var e = /a\\{terminator}b/;\nalert(gettext("still here"));'
+    tokens = list(jslexer.tokenize(source))
+    assert any(kind == 'string' and value == '"still here"' for kind, value, _ in tokens)
+    for kind, value, _lineno in tokens:
+        if kind != 'regexp':
+            continue
+        assert not any(t in value for t in LINE_TERMINATORS), f"regexp spans lines: {value!r}"
+
+
+def test_ordinary_regexp_literals_still_match():
+    # Escaped slashes and flags are unaffected by the line terminator guard.
+    tokens = list(jslexer.tokenize(r'var e = /a\/b+c/gi;'))
+    assert ('regexp', r'/a\/b+c/gi', 1) in tokens