diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index db08b644f..1366b168d 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -335,17 +335,31 @@ Token Lexer::Lex() { return SetError(start, GetPosition(), "unexpected character"); } -// Consumes characters up to and including the first occurrence of character `c` -// without interpreting backslashes as escapes. -// Returns true if `c` was found and consumed; false if end of input was -// reached. -bool Lexer::ConsumeUntilAfter(char32_t c) { +// Consumes characters up to and including the first occurrence of character +// `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns +// true if `c` was found and consumed; false if end of input or an unescaped +// newline was reached. +bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { ABSL_DCHECK_NE(c, '\n'); - for (int32_t pos = position_; pos < content_.size(); ++pos) { - if (content_.at(pos) == c) { - AdvanceProcessingNewLines(pos + 1); - return true; + ABSL_DCHECK_NE(c, '\r'); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\n' || cc == '\r') { + AdvanceProcessingNewLines(pos); + return false; } + if (!is_raw && cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && (is_raw || !escaped)) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + escaped = false; + } + ++pos; } AdvanceProcessingNewLines(content_.size()); return false; @@ -376,31 +390,6 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { return false; } -// Consumes characters up to and including the first occurrence of `c` that is -// not preceded by an odd number of backslash ('\') escape characters. Returns -// true if an unescaped `c` was found and consumed; false if reached EOF. -bool Lexer::ConsumeUntilAfterUnescaped(char32_t c) { - ABSL_DCHECK_NE(c, '\n'); - ABSL_DCHECK_NE(c, '\\'); - int32_t pos = position_; - bool escaped = false; - while (pos < content_.size()) { - char32_t cc = content_.at(pos); - if (cc == '\\') { - escaped = !escaped; - } else { - if (cc == c && !escaped) { - AdvanceProcessingNewLines(pos + 1); - return true; - } - escaped = false; - } - ++pos; - } - AdvanceProcessingNewLines(content_.size()); - return false; -} - // Consumes characters up to and including the first occurrence of substring `s` // where the first character of `s` is not preceded by an odd number of // backslashes. Returns true if an unescaped `s` was found and consumed; false @@ -564,7 +553,7 @@ TokenType Lexer::ConsumeIntegralSuffix() { Token Lexer::ConsumeQuotedIdent() { int32_t start = GetPosition(); Advance(1); - if (!ConsumeUntilAfter('`')) { + if (!ConsumeUntilAfter('`', /*is_raw=*/true)) { return SetError(start, GetPosition(), "unterminated quoted identifier"); } return MakeToken(TokenType::kIdent, start, GetPosition()); @@ -584,7 +573,7 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, GetPosition()); } - if (is_raw ? !ConsumeUntilAfter(quote) : !ConsumeUntilAfterUnescaped(quote)) { + if (!ConsumeUntilAfter(quote, is_raw)) { return SetError(start, GetPosition(), is_bytes ? "unterminated bytes literal" : "unterminated string literal"); diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index c35dc08e2..5fbdb4feb 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -217,9 +217,10 @@ class Lexer final { void AtEndTokenCreated() { done_ = true; } // Consumes characters up to and including the first occurrence of character - // `c` without interpreting backslashes as escapes. Returns true if `c` was - // found and consumed; false if end of input was reached. - [[nodiscard]] bool ConsumeUntilAfter(char32_t c); + // `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns + // true if `c` was found and consumed; false if end of input or an unescaped + // newline was reached. + [[nodiscard]] bool ConsumeUntilAfter(char32_t c, bool is_raw); // Consumes characters up to and including the first occurrence of substring // `s` without interpreting backslashes as escapes (`s` must not contain @@ -227,11 +228,6 @@ class Lexer final { // input was reached. [[nodiscard]] bool ConsumeUntilAfterString(std::u32string_view s); - // Consumes characters up to and including the first occurrence of `c` that is - // not preceded by an odd number of backslash ('\') escape characters. Returns - // true if an unescaped `c` was found and consumed; false if reached EOF. - [[nodiscard]] bool ConsumeUntilAfterUnescaped(char32_t c); - // Consumes characters up to and including the first occurrence of substring // `s` where the first character of `s` is not preceded by an odd number of // backslashes. Returns true if an unescaped `s` was found and consumed; false diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc index ebe0011d3..eb8f1548e 100644 --- a/parser/internal/lexer_test.cc +++ b/parser/internal/lexer_test.cc @@ -334,29 +334,23 @@ TEST(LexerTest, LineOffsets) { EXPECT_EQ(line_offsets[1], 13); } -TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { - std::string_view source_text = - "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; +TEST(LexerTest, LineOffsetsInMultilineStrings) { + std::string_view source_text = "'''multi\nline'''\n\"\"\"another\nline\"\"\""; ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); Lexer lexer(*source); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kString, "'''multi\nline'''")); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); - EXPECT_THAT(lexer.Lex(), - IsToken(source.get(), TokenType::kString, "\"another\nline\"")); - EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); - EXPECT_THAT(lexer.Lex(), - IsToken(source.get(), TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kString, + "\"\"\"another\nline\"\"\"")); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); auto line_offsets = source->line_offsets(); - ASSERT_GE(line_offsets.size(), 5); + ASSERT_GE(line_offsets.size(), 3); EXPECT_EQ(line_offsets[0], 9); EXPECT_EQ(line_offsets[1], 17); - EXPECT_EQ(line_offsets[2], 26); - EXPECT_EQ(line_offsets[3], 32); - EXPECT_EQ(line_offsets[4], 39); + EXPECT_EQ(line_offsets[2], 28); } struct LexerErrorTestCase { @@ -392,6 +386,12 @@ INSTANTIATE_TEST_SUITE_P( .expected_error_location = "\n | \"unterminated" "\n | .............^", }, + LexerErrorTestCase{ + .source = "\"another\nline\"", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"another" + "\n | ........^", + }, LexerErrorTestCase{ .source = "0x", .expected_error_message = @@ -439,6 +439,12 @@ INSTANTIATE_TEST_SUITE_P( .expected_error_location = "\n | `unterminated quoted" "\n | ....................^", }, + LexerErrorTestCase{ + .source = "`ident\nhere`", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `ident" + "\n | ......^", + }, LexerErrorTestCase{ .source = "'''unterminated multi", .expected_error_message = "unterminated string literal", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 4fd2b02ec..fc6477745 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -361,6 +361,36 @@ std::vector GetParserTestCases() { "\x07\x08\x0c\n\r\t\x0b'\"\\? Legal escapes"^#1:string# )", }, + TestCase{ + .source = "'''hello\nworld'''", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "r\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\\\"\"\"world\"\"\"", + .expected_ast = R"( + "hello\"\"\"world"^#1:string# + )", + }, + TestCase{ + .source = "'''hello\\'''world'''", + .expected_ast = R"( + "hello'''world"^#1:string# + )", + }, TestCase{ .source = "a", .expected_ast = R"( @@ -1394,6 +1424,84 @@ std::vector GetErrorTestCases() { " | \"unterminated\n" " | ^", }, + ErrorTestCase{ + .source = "\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "'''hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "'hello\nworld'", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world'\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "`hello\nworld`", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "\"hello\rworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after " + "expression\n" + " | \"hello\rworld\"\n" + " | .......^", + }, ErrorTestCase{ .source = "b\"unterminated", .expected_error = diff --git a/parser/parser_test.cc b/parser/parser_test.cc index d51339796..922968d3e 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1517,7 +1517,132 @@ std::vector test_cases = { "m^#5:Expr.Ident#.value()^#8:Expr.Call#,\n // LoopCondition\n " "false^#9:bool#,\n // LoopStep\n v^#3:Expr.Ident#,\n // Result\n " " f^#4:Expr.Ident#)^#10:Expr.Comprehension#,\n " - "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}}; + "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}, + + // Multiline and string literal tests + {"'''hello\nworld'''", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"r\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\\\"\"\"world\"\"\"", "\"hello\\\"\\\"\\\"world\"^#1:string#"}, + {"'''hello\\'''world'''", "\"hello'''world\"^#1:string#"}, + {"\"\"\"hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"\"\"hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^"}, + {"'''hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | '''hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^"}, + {"r\"\"\"hello\nworld", "", + "ERROR: :1:4: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"\"\"hello\n" + " | ...^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^"}, + {"\"hello\nworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"'hello\nworld'", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '''\n" + " | world'\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world'\n" + " | ^"}, + {"r\"hello\nworld\"", "", + "ERROR: :1:2: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"hello\n" + " | .^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\"\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"`hello\nworld`", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'`hello\\n'\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '`'\n" + " | world`\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated quoted identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world`\n" + " | ^"}, + {"\"hello\rworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\r'\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:13: Syntax error: token recognition error at: '\"'\n" + " | \"hello\rworld\"\n" + " | ............^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after expression\n" + " | \"hello\rworld\"\n" + " | .......^"}}; absl::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) {