diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel index 1e662c3c5..8bd568183 100644 --- a/parser/BUILD.bazel +++ b/parser/BUILD.bazel @@ -11,6 +11,12 @@ java_library( exports = ["//parser/src/main/java/dev/cel/parser"], ) +java_library( + name = "pratt_parser", + visibility = ["//:internal"], + exports = ["//parser/src/main/java/dev/cel/parser:pratt_parser"], +) + java_library( name = "parser_factory", exports = ["//parser/src/main/java/dev/cel/parser:parser_factory"], diff --git a/parser/src/main/java/dev/cel/parser/BUILD.bazel b/parser/src/main/java/dev/cel/parser/BUILD.bazel index e32c50ee8..905bf298f 100644 --- a/parser/src/main/java/dev/cel/parser/BUILD.bazel +++ b/parser/src/main/java/dev/cel/parser/BUILD.bazel @@ -15,6 +15,12 @@ PARSER_SOURCES = [ "Parser.java", ] +# keep sorted +PRATT_PARSER_SOURCES = [ + "Lexer.java", + "PrattParser.java", +] + # keep sorted PARSER_BUILDER_SOURCES = [ "CelParser.java", @@ -75,6 +81,26 @@ java_library( ], ) +java_library( + name = "pratt_parser", + srcs = PRATT_PARSER_SOURCES, + tags = [ + ], + deps = [ + ":macro", + "//common:cel_ast", + "//common:cel_source", + "//common:compiler_common", + "//common:operator", + "//common:options", + "//common:source_location", + "//common/ast", + "//common/internal", + "@maven//:com_google_guava_guava", + "@maven//:org_jspecify_jspecify", + ], +) + java_library( name = "parser_builder", srcs = PARSER_BUILDER_SOURCES, diff --git a/parser/src/main/java/dev/cel/parser/Lexer.java b/parser/src/main/java/dev/cel/parser/Lexer.java new file mode 100644 index 000000000..a80272e70 --- /dev/null +++ b/parser/src/main/java/dev/cel/parser/Lexer.java @@ -0,0 +1,676 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.parser; + +import com.google.common.collect.ImmutableMap; +import dev.cel.common.internal.CelCodePointArray; +import java.util.function.IntPredicate; +import org.jspecify.annotations.Nullable; + +/** + * Fast lexer for CEL expressions. + * + *

Ported from {@code third_party/cel/cpp/parser/internal/lexer.h} and {@code lexer.cc}. + */ +final class Lexer { + + enum TokenType { + ERROR("error"), + END("end"), + WHITESPACE("whitespace"), + COMMENT("comment"), + + // Keywords + NULL("null"), + FALSE("false"), + TRUE("true"), + IN("in"), + RESERVED_WORD("reserved_word"), + + // Literals + INT("int"), + UINT("uint"), + FLOAT("float"), + STRING("string"), + BYTES("bytes"), + + // Identifiers + IDENT("ident"), + + // Delimiters + LEFT_BRACKET("["), + RIGHT_BRACKET("]"), + LEFT_BRACE("{"), + RIGHT_BRACE("}"), + LEFT_PAREN("("), + RIGHT_PAREN(")"), + + // Operators + DOT("."), + COMMA(","), + MINUS("-"), + PLUS("+"), + ASTERISK("*"), + SLASH("/"), + PERCENT("%"), + QUESTION("?"), + COLON(":"), + EXCLAMATION("!"), + EQUAL("="), + EQUAL_EQUAL("=="), + EXCLAMATION_EQUAL("!="), + LESS("<"), + LESS_EQUAL("<="), + GREATER(">"), + GREATER_EQUAL(">="), + LOGICAL_AND("&&"), + LOGICAL_OR("||"); + + private final String symbol; + + TokenType(String symbol) { + this.symbol = symbol; + } + + public String getSymbol() { + return symbol; + } + + @Override + public String toString() { + return symbol; + } + } + + static final class Token { + final TokenType type; + final int start; + final int end; + + Token(TokenType type, int start, int end) { + this.type = type; + this.start = start; + this.end = end; + } + + @Override + public String toString() { + return "Token(" + type + ", " + start + ", " + end + ")"; + } + } + + static final class LexerError { + final int start; + final int end; + final String message; + + LexerError(int start, int end, String message) { + this.start = start; + this.end = end; + this.message = message; + } + } + + private static final ImmutableMap KEYWORDS = + ImmutableMap.builder() + .put("false", TokenType.FALSE) + .put("true", TokenType.TRUE) + .put("null", TokenType.NULL) + .put("in", TokenType.IN) + .put("as", TokenType.RESERVED_WORD) + .put("break", TokenType.RESERVED_WORD) + .put("const", TokenType.RESERVED_WORD) + .put("continue", TokenType.RESERVED_WORD) + .put("else", TokenType.RESERVED_WORD) + .put("for", TokenType.RESERVED_WORD) + .put("function", TokenType.RESERVED_WORD) + .put("if", TokenType.RESERVED_WORD) + .put("import", TokenType.RESERVED_WORD) + .put("let", TokenType.RESERVED_WORD) + .put("loop", TokenType.RESERVED_WORD) + .put("package", TokenType.RESERVED_WORD) + .put("namespace", TokenType.RESERVED_WORD) + .put("return", TokenType.RESERVED_WORD) + .put("var", TokenType.RESERVED_WORD) + .put("void", TokenType.RESERVED_WORD) + .put("while", TokenType.RESERVED_WORD) + .buildOrThrow(); + + private final CelCodePointArray content; + private int position; + private LexerError error; + + Lexer(CelCodePointArray content) { + this.content = content; + this.position = 0; + this.error = null; + } + + Token lex() { + int start = position; + if (position >= content.size()) { + return makeToken(TokenType.END, start, start); + } + int c = content.get(position); + switch (c) { + case '\f': + case '\n': + case ' ': + case '\r': + case 0x0B: // \v (vertical tab) + case '\t': + { + consumeWhitespace(); + return makeToken(TokenType.WHITESPACE, start, position); + } + case '.': + { + if (position + 1 < content.size() && isDigit(content.get(position + 1))) { + return consumeNumericLiteral(); + } + advance(1); + return makeToken(TokenType.DOT, start, position); + } + case ',': + { + advance(1); + return makeToken(TokenType.COMMA, start, position); + } + case '!': + { + advance(1); + if (consume('=')) { + return makeToken(TokenType.EXCLAMATION_EQUAL, start, position); + } + return makeToken(TokenType.EXCLAMATION, start, position); + } + case '?': + { + advance(1); + return makeToken(TokenType.QUESTION, start, position); + } + case '(': + { + advance(1); + return makeToken(TokenType.LEFT_PAREN, start, position); + } + case ')': + { + advance(1); + return makeToken(TokenType.RIGHT_PAREN, start, position); + } + case '{': + { + advance(1); + return makeToken(TokenType.LEFT_BRACE, start, position); + } + case '}': + { + advance(1); + return makeToken(TokenType.RIGHT_BRACE, start, position); + } + case '[': + { + advance(1); + return makeToken(TokenType.LEFT_BRACKET, start, position); + } + case ']': + { + advance(1); + return makeToken(TokenType.RIGHT_BRACKET, start, position); + } + case '=': + { + advance(1); + if (consume('=')) { + return makeToken(TokenType.EQUAL_EQUAL, start, position); + } + return makeToken(TokenType.EQUAL, start, position); + } + case '<': + { + advance(1); + if (consume('=')) { + return makeToken(TokenType.LESS_EQUAL, start, position); + } + return makeToken(TokenType.LESS, start, position); + } + case '>': + { + advance(1); + if (consume('=')) { + return makeToken(TokenType.GREATER_EQUAL, start, position); + } + return makeToken(TokenType.GREATER, start, position); + } + case ':': + { + advance(1); + return makeToken(TokenType.COLON, start, position); + } + case '%': + { + advance(1); + return makeToken(TokenType.PERCENT, start, position); + } + case '+': + { + advance(1); + return makeToken(TokenType.PLUS, start, position); + } + case '-': + { + advance(1); + return makeToken(TokenType.MINUS, start, position); + } + case '*': + { + advance(1); + return makeToken(TokenType.ASTERISK, start, position); + } + case '/': + { + advance(1); + if (consume('/')) { + consumeLine(); + return makeToken(TokenType.COMMENT, start, position); + } + return makeToken(TokenType.SLASH, start, position); + } + case '&': + { + advance(1); + if (consume('&')) { + return makeToken(TokenType.LOGICAL_AND, start, position); + } + return setError(start, position, "unexpected single '&', expected '&&'"); + } + case '|': + { + advance(1); + if (consume('|')) { + return makeToken(TokenType.LOGICAL_OR, start, position); + } + return setError(start, position, "unexpected single '|', expected '||'"); + } + case '_': + { + return consumeIdent(); + } + case '`': + { + return consumeQuotedIdent(); + } + case '\'': + { + return consumeStringLiteral(start, '\'', false, false); + } + case '"': + { + return consumeStringLiteral(start, '"', false, false); + } + case 'r': + case 'R': + case 'b': + case 'B': + { + Token token = consumePrefixedStringLiteral(); + if (token != null) { + return token; + } + break; + } + default: + break; + } + if (isDigit(c)) { + return consumeNumericLiteral(); + } + if (isAlpha(c)) { + return consumeIdent(); + } + advance(1); + return setError(start, position, "unexpected character"); + } + + LexerError getError() { + return error; + } + + int savePosition() { + return position; + } + + void restorePosition(int pos) { + this.position = pos; + this.error = null; + } + + private static boolean isDigit(int c) { + return c >= '0' && c <= '9'; + } + + private static boolean isHexDigit(int c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + private static boolean isAlpha(int c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + private static boolean isIdentTrailing(int c) { + return isDigit(c) || isAlpha(c) || c == '_'; + } + + private static boolean isPlusOrMinus(int c) { + return c == '+' || c == '-'; + } + + private Token makeToken(TokenType type, int start, int end) { + return new Token(type, start, end); + } + + private Token setError(int start, int end, String message) { + this.error = new LexerError(start, end, message); + return new Token(TokenType.ERROR, start, end); + } + + private void advance(int n) { + position += n; + } + + private boolean match(int c) { + return position < content.size() && content.get(position) == c; + } + + private boolean matchIgnoreCase(int c) { + if (position >= content.size()) { + return false; + } + int cp = content.get(position); + return cp <= 0x7f && c <= 0x7f && Character.toLowerCase(cp) == Character.toLowerCase(c); + } + + private boolean consume(int c) { + if (match(c)) { + advance(1); + return true; + } + return false; + } + + private boolean consumeIgnoreCase(int c) { + if (matchIgnoreCase(c)) { + advance(1); + return true; + } + return false; + } + + private boolean consumeIf(IntPredicate predicate) { + if (position < content.size()) { + int cp = content.get(position); + if (predicate.test(cp)) { + advance(1); + return true; + } + } + return false; + } + + private void consumeLine() { + while (position < content.size()) { + if (content.get(position) == '\n') { + advance(1); + return; + } + advance(1); + } + } + + private void consumeWhitespace() { + while (position < content.size()) { + int c = content.get(position); + switch (c) { + case '\f': + case '\n': + case ' ': + case '\r': + case 11: // \v + case '\t': + advance(1); + break; + default: + return; + } + } + } + + private boolean consumeDigits() { + boolean advanced = false; + while (position < content.size()) { + int c = content.get(position); + if (!isDigit(c)) { + break; + } + advance(1); + advanced = true; + } + return advanced; + } + + private boolean consumeHexDigits() { + boolean advanced = false; + while (position < content.size()) { + int c = content.get(position); + if (!isHexDigit(c)) { + break; + } + advance(1); + advanced = true; + } + return advanced; + } + + private TokenType consumeIntegralSuffix() { + if (consumeIgnoreCase('u')) { + return TokenType.UINT; + } + return TokenType.INT; + } + + private Token consumeQuotedIdent() { + int start = position; + advance(1); + if (!consumeUntilAfter('`', /* isRaw= */ true)) { + return setError(start, position, "unterminated quoted identifier"); + } + return makeToken(TokenType.IDENT, start, position); + } + + private boolean consumeUntilAfter(int c, boolean isRaw) { + int pos = position; + boolean escaped = false; + while (pos < content.size()) { + int cc = content.get(pos); + if (cc == '\n' || cc == '\r') { + position = pos; + return false; + } + if (!isRaw && cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && (isRaw || !escaped)) { + position = pos + 1; + return true; + } + escaped = false; + } + pos++; + } + position = content.size(); + return false; + } + + private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) { + int pos = position; + boolean escaped = false; + while (pos < content.size()) { + int cc = content.get(pos); + if (!isRaw && cc == '\\') { + escaped = !escaped; + } else { + if ((isRaw || !escaped) + && pos + 2 < content.size() + && cc == quote + && content.get(pos + 1) == quote + && content.get(pos + 2) == quote) { + position = pos + 3; + return true; + } + escaped = false; + } + pos++; + } + position = content.size(); + return false; + } + + private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolean isRaw) { + advance(1); + boolean isTripleQuote = + position + 1 < content.size() + && content.get(position) == quote + && content.get(position + 1) == quote; + if (isTripleQuote) { + advance(2); + if (!consumeUntilAfterTripleQuote(quote, isRaw)) { + return setError( + start, + position, + isBytes ? "unterminated bytes literal" : "unterminated string literal"); + } + return makeToken(isBytes ? TokenType.BYTES : TokenType.STRING, start, position); + } + if (!consumeUntilAfter(quote, isRaw)) { + return setError( + start, position, isBytes ? "unterminated bytes literal" : "unterminated string literal"); + } + return makeToken(isBytes ? TokenType.BYTES : TokenType.STRING, start, position); + } + + private @Nullable Token consumePrefixedStringLiteral() { + int start = position; + if (position >= content.size()) { + return null; + } + int c = content.get(position); + boolean isBytes = (c == 'b' || c == 'B'); + boolean isRaw = (c == 'r' || c == 'R'); + if (!isBytes && !isRaw) { + return null; + } + int lookahead = 1; + if (position + 1 < content.size()) { + int c2 = content.get(position + 1); + if ((isBytes && (c2 == 'r' || c2 == 'R')) || (!isBytes && (c2 == 'b' || c2 == 'B'))) { + isBytes = true; + isRaw = true; + lookahead = 2; + } + } + if (position + lookahead < content.size()) { + int quote = content.get(position + lookahead); + if (quote == '"' || quote == '\'') { + advance(lookahead); + return consumeStringLiteral(start, quote, isBytes, isRaw); + } + } + return null; + } + + private Token consumeNumericLiteral() { + int start = position; + int c = content.get(position); + boolean floatingPoint = false; + if (c == '.') { + floatingPoint = true; + advance(1); + if (!consumeDigits()) { + return setError( + start, position, "floating point literal missing digits after decimal separator"); + } + } else { + advance(1); + if (c == '0') { + if (consumeIgnoreCase('x')) { + if (!consumeHexDigits()) { + return setError( + start, position, "integral literal missing digits after hexadecimal separator"); + } + TokenType tokenType = consumeIntegralSuffix(); + if (consumeIf(Lexer::isIdentTrailing)) { + return setError( + start, + position, + tokenType.getSymbol() + " literal has unexpected trailing characters"); + } + return makeToken(tokenType, start, position); + } + } + consumeDigits(); + if (position < content.size() + && content.get(position) == '.' + && position + 1 < content.size() + && isDigit(content.get(position + 1))) { + floatingPoint = true; + advance(1); + consumeDigits(); + } + } + if (consumeIgnoreCase('e')) { + floatingPoint = true; + consumeIf(Lexer::isPlusOrMinus); + if (!consumeDigits()) { + return setError( + start, position, "floating point literal missing digits after exponent separator"); + } + } + TokenType tokenType = floatingPoint ? TokenType.FLOAT : consumeIntegralSuffix(); + if (consumeIf(Lexer::isIdentTrailing)) { + return setError( + start, position, tokenType.getSymbol() + " literal has unexpected trailing characters"); + } + return makeToken(tokenType, start, position); + } + + private Token consumeIdent() { + int start = position; + while (position < content.size()) { + int c = content.get(position); + if (!isIdentTrailing(c)) { + break; + } + advance(1); + } + int end = position; + String word = content.slice(start, end).toString(); + TokenType keywordType = KEYWORDS.get(word); + if (keywordType != null) { + return makeToken(keywordType, start, end); + } + return makeToken(TokenType.IDENT, start, end); + } +} diff --git a/parser/src/main/java/dev/cel/parser/PrattParser.java b/parser/src/main/java/dev/cel/parser/PrattParser.java new file mode 100644 index 000000000..1f5f87d1e --- /dev/null +++ b/parser/src/main/java/dev/cel/parser/PrattParser.java @@ -0,0 +1,1319 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.parser; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelIssue; +import dev.cel.common.CelOptions; +import dev.cel.common.CelSource; +import dev.cel.common.CelSourceLocation; +import dev.cel.common.CelValidationResult; +import dev.cel.common.Operator; +import dev.cel.common.ast.CelConstant; +import dev.cel.common.ast.CelExpr; +import dev.cel.common.internal.Constants; +import java.text.ParseException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.jspecify.annotations.Nullable; + +/** Pratt parser implementation for CEL. */ +final class PrattParser { + + private static final String ACCUMULATOR_NAME = "@result"; + private static final CelExpr ERROR = CelExpr.newBuilder().setConstant(Constants.ERROR).build(); + + private static final class BinaryOpInfo { + final int precedence; + final String name; + final boolean isLogical; + final Lexer.TokenType type; + + BinaryOpInfo(int precedence, String name, boolean isLogical, Lexer.TokenType type) { + this.precedence = precedence; + this.name = name; + this.isLogical = isLogical; + this.type = type; + } + } + + private static final BinaryOpInfo LOGICAL_OR_OP = + new BinaryOpInfo(1, Operator.LOGICAL_OR.getFunction(), true, Lexer.TokenType.LOGICAL_OR); + private static final BinaryOpInfo LOGICAL_AND_OP = + new BinaryOpInfo(2, Operator.LOGICAL_AND.getFunction(), true, Lexer.TokenType.LOGICAL_AND); + private static final BinaryOpInfo LESS_OP = + new BinaryOpInfo(3, Operator.LESS.getFunction(), false, Lexer.TokenType.LESS); + private static final BinaryOpInfo LESS_EQUAL_OP = + new BinaryOpInfo(3, Operator.LESS_EQUALS.getFunction(), false, Lexer.TokenType.LESS_EQUAL); + private static final BinaryOpInfo GREATER_OP = + new BinaryOpInfo(3, Operator.GREATER.getFunction(), false, Lexer.TokenType.GREATER); + private static final BinaryOpInfo GREATER_EQUAL_OP = + new BinaryOpInfo( + 3, Operator.GREATER_EQUALS.getFunction(), false, Lexer.TokenType.GREATER_EQUAL); + private static final BinaryOpInfo EQUAL_EQUAL_OP = + new BinaryOpInfo(3, Operator.EQUALS.getFunction(), false, Lexer.TokenType.EQUAL_EQUAL); + private static final BinaryOpInfo EXCLAMATION_EQUAL_OP = + new BinaryOpInfo( + 3, Operator.NOT_EQUALS.getFunction(), false, Lexer.TokenType.EXCLAMATION_EQUAL); + private static final BinaryOpInfo IN_OP = + new BinaryOpInfo(3, Operator.IN.getFunction(), false, Lexer.TokenType.IN); + private static final BinaryOpInfo PLUS_OP = + new BinaryOpInfo(4, Operator.ADD.getFunction(), false, Lexer.TokenType.PLUS); + private static final BinaryOpInfo MINUS_OP = + new BinaryOpInfo(4, Operator.SUBTRACT.getFunction(), false, Lexer.TokenType.MINUS); + private static final BinaryOpInfo ASTERISK_OP = + new BinaryOpInfo(5, Operator.MULTIPLY.getFunction(), false, Lexer.TokenType.ASTERISK); + private static final BinaryOpInfo SLASH_OP = + new BinaryOpInfo(5, Operator.DIVIDE.getFunction(), false, Lexer.TokenType.SLASH); + private static final BinaryOpInfo PERCENT_OP = + new BinaryOpInfo(5, Operator.MODULO.getFunction(), false, Lexer.TokenType.PERCENT); + private static final BinaryOpInfo DEFAULT_OP = + new BinaryOpInfo(0, "", false, Lexer.TokenType.ERROR); + + private static BinaryOpInfo getBinaryOpInfo(Lexer.TokenType type) { + switch (type) { + case LOGICAL_OR: + return LOGICAL_OR_OP; + case LOGICAL_AND: + return LOGICAL_AND_OP; + case LESS: + return LESS_OP; + case LESS_EQUAL: + return LESS_EQUAL_OP; + case GREATER: + return GREATER_OP; + case GREATER_EQUAL: + return GREATER_EQUAL_OP; + case EQUAL_EQUAL: + return EQUAL_EQUAL_OP; + case EXCLAMATION_EQUAL: + return EXCLAMATION_EQUAL_OP; + case IN: + return IN_OP; + case PLUS: + return PLUS_OP; + case MINUS: + return MINUS_OP; + case ASTERISK: + return ASTERISK_OP; + case SLASH: + return SLASH_OP; + case PERCENT: + return PERCENT_OP; + default: + return DEFAULT_OP; + } + } + + private static final class UnaryOp { + final Lexer.Token token; + long id; + + UnaryOp(Lexer.Token token) { + this.token = token; + this.id = 0; + } + } + + private final CelSource source; + private final CelOptions options; + private final ImmutableMap macros; + private final Lexer lexer; + private final Map positions; + private final Map macroCalls; + private final List issues; + private final PrattMacroExprFactory macroExprFactory; + + private Lexer.Token currentToken; + private Lexer.Token peekToken; + private int recursionDepth; + private int currentLhsDepth; + private long nextId; + private boolean nodeLimitExceeded; + private boolean recursionLimitExceeded; + private int errorCount; + + static CelValidationResult parse( + CelSource source, CelOptions options, Map macros) { + if (source.getContent().size() > options.maxExpressionCodePointSize()) { + return new CelValidationResult( + source, + ImmutableList.of( + CelIssue.formatError( + CelSourceLocation.NONE, + String.format( + "expression code point size exceeds limit: size: %d, limit %d", + source.getContent().size(), options.maxExpressionCodePointSize())))); + } + PrattParser prattParser = new PrattParser(source, options, macros); + CelExpr expr = prattParser.run(); + if (prattParser.recursionLimitExceeded || prattParser.errorCount > 0) { + return new CelValidationResult(source, ImmutableList.copyOf(prattParser.issues)); + } + + CelSource.Builder sourceBuilder = source.toBuilder(); + sourceBuilder.addPositionsMap(prattParser.positions); + sourceBuilder.addAllMacroCalls(prattParser.macroCalls); + + return new CelValidationResult( + CelAbstractSyntaxTree.newParsedAst(expr, sourceBuilder.build()), + ImmutableList.copyOf(prattParser.issues)); + } + + private PrattParser(CelSource source, CelOptions options, Map macros) { + this.source = source; + this.options = options; + this.macros = ImmutableMap.copyOf(macros); + this.lexer = new Lexer(source.getContent()); + this.positions = new HashMap<>(); + this.macroCalls = new HashMap<>(); + this.issues = new ArrayList<>(); + this.macroExprFactory = new PrattMacroExprFactory(); + this.nextId = 1; + initTokenStream(); + } + + CelExpr run() { + CelExpr expr = parseExpr(); + if (recursionLimitExceeded || isRecoveryLimitExceeded()) { + return expr; + } + while (peekToken.type != Lexer.TokenType.END && peekToken.type != Lexer.TokenType.ERROR) { + if (options.enableReservedIds() + && (peekToken.type == Lexer.TokenType.RESERVED_WORD + || peekToken.type == Lexer.TokenType.IN)) { + Lexer.Token resTok = nextToken(); + String resText = normalizeIdent(resTok, /* allowQuoted= */ false); + reportError(resTok.start, String.format("reserved identifier: %s", resText)); + continue; + } + reportSyntaxError(peekToken, "unexpected token after expression"); + break; + } + return expr; + } + + private boolean isRecoveryLimitExceeded() { + return errorCount > options.maxParseErrorRecoveryLimit(); + } + + private void initTokenStream() { + currentToken = new Lexer.Token(Lexer.TokenType.ERROR, 0, 0); + peekToken = nextSignificantToken(true); + } + + private String getTokenText(Lexer.Token tok) { + if (tok.start >= 0 && tok.end >= tok.start && tok.end <= source.getContent().size()) { + return source.getContent().slice(tok.start, tok.end).toString(); + } + return ""; + } + + private Lexer.Token nextSignificantToken(boolean reportError) { + if (isRecoveryLimitExceeded()) { + return new Lexer.Token(Lexer.TokenType.END, 0, 0); + } + while (true) { + Lexer.Token tok = lexer.lex(); + if (tok.type == Lexer.TokenType.WHITESPACE || tok.type == Lexer.TokenType.COMMENT) { + continue; + } + if (tok.type == Lexer.TokenType.ERROR && reportError) { + reportSyntaxError(tok, lexer.getError().message); + if (isRecoveryLimitExceeded()) { + return new Lexer.Token(Lexer.TokenType.END, 0, 0); + } + } + return tok; + } + } + + private Lexer.Token nextToken() { + currentToken = peekToken; + if (isRecoveryLimitExceeded()) { + peekToken = new Lexer.Token(Lexer.TokenType.END, 0, 0); + return currentToken; + } + if (peekToken.type != Lexer.TokenType.END) { + peekToken = nextSignificantToken(true); + } + return currentToken; + } + + private boolean expect(Lexer.TokenType type, String msg) { + if (peekToken.type == type) { + nextToken(); + return true; + } + if (isRecoveryLimitExceeded()) { + return false; + } + if (peekToken.type != Lexer.TokenType.ERROR) { + String errMsg; + if (msg == null || msg.isEmpty()) { + String tokText = getTokenText(peekToken); + String formattedTok = + (peekToken.type == Lexer.TokenType.END) ? "" : "'" + tokText + "'"; + errMsg = "mismatched input " + formattedTok + " expecting '" + type.getSymbol() + "'"; + } else { + errMsg = msg; + } + reportSyntaxError(peekToken, errMsg); + } + synchronizeOnDelimiter(); + return false; + } + + private void synchronizeOnDelimiter() { + if (isRecoveryLimitExceeded()) { + peekToken = new Lexer.Token(Lexer.TokenType.END, 0, 0); + return; + } + while (peekToken.type != Lexer.TokenType.END) { + if (peekToken.type == Lexer.TokenType.COMMA + || peekToken.type == Lexer.TokenType.RIGHT_PAREN + || peekToken.type == Lexer.TokenType.RIGHT_BRACKET + || peekToken.type == Lexer.TokenType.RIGHT_BRACE) { + break; + } + nextToken(); + } + } + + private long nextId(int position) { + long id = nextId++; + if (id > options.maxParseExpressionNodeCount() && !nodeLimitExceeded) { + reportError( + position, + String.format( + "expression node limit (%d) exceeded", options.maxParseExpressionNodeCount())); + nodeLimitExceeded = true; + } + if (!nodeLimitExceeded && position >= 0) { + positions.put(id, position); + } + return id; + } + + private long nextId(Lexer.Token token) { + return nextId(token.start); + } + + private long nextId() { + return nextId(-1); + } + + private void setPosition(long id, Lexer.Token token) { + if (token.start >= 0) { + positions.put(id, token.start); + } + } + + private long copyId(long id) { + if (id == 0) { + return 0; + } + int pos = positions.getOrDefault(id, 0); + return nextId(pos); + } + + private void eraseId(long id) { + positions.remove(id); + if (nextId == id + 1) { + --nextId; + } + } + + private void reportError(int position, String msg) { + CelSourceLocation loc = source.getOffsetLocation(position).orElse(CelSourceLocation.NONE); + reportError(loc, msg); + } + + private void reportError(CelSourceLocation loc, String msg) { + if (errorCount > options.maxParseErrorRecoveryLimit()) { + return; + } + errorCount++; + if (errorCount == options.maxParseErrorRecoveryLimit() + 1) { + issues.add( + CelIssue.formatError( + CelSourceLocation.NONE, + String.format("More than %d parse errors.", options.maxParseErrorRecoveryLimit()))); + peekToken = new Lexer.Token(Lexer.TokenType.END, 0, 0); + } + if (errorCount <= options.maxParseErrorRecoveryLimit()) { + issues.add(CelIssue.formatError(loc, msg)); + } + } + + private void reportSyntaxError(Lexer.Token token, String msg) { + reportError(token.start, "Syntax error: " + msg); + } + + private boolean checkRecursion(int chainDepth, Lexer.Token token) { + if (recursionDepth + chainDepth >= options.maxParseRecursionDepth()) { + if (!recursionLimitExceeded) { + recursionLimitExceeded = true; + reportError( + token.start, + String.format( + "Expression recursion limit exceeded. limit: %d", + options.maxParseRecursionDepth())); + } + return true; + } + return false; + } + + private CelExpr parseExpr() { + if (recursionLimitExceeded || isRecoveryLimitExceeded()) { + return ERROR; + } + if (checkRecursion(0, peekToken)) { + return ERROR; + } + recursionDepth++; + CelExpr expr = parseBinaryAndTernary(0); + recursionDepth--; + return expr; + } + + private CelExpr parseBinaryAndTernary(int minPrec) { + CelExpr lhs = parseSelectorChain(); + int chainDepth = currentLhsDepth; + while (true) { + Lexer.TokenType tok = peekToken.type; + if (tok == Lexer.TokenType.QUESTION && minPrec <= 0) { + lhs = parseTernary(lhs); + continue; + } + + BinaryOpInfo opInfo = getBinaryOpInfo(tok); + if (opInfo.precedence < minPrec || opInfo.precedence == 0) { + break; + } + + if (opInfo.isLogical) { + lhs = parseBalancedLogicalChain(lhs, opInfo); + continue; + } + + Lexer.Token opTok = nextToken(); + chainDepth++; + if (checkRecursion(chainDepth, opTok)) { + return ERROR; + } + long opId = nextId(opTok); + CelExpr rhs = parseBinaryAndTernary(opInfo.precedence + 1); + lhs = buildBinaryCall(opId, opInfo.name, lhs, rhs); + currentLhsDepth = chainDepth; + } + return lhs; + } + + private CelExpr parseTernary(CelExpr lhs) { + Lexer.Token opTok = nextToken(); + long opId = nextId(opTok); + CelExpr trueExpr = parseBinaryAndTernary(1); + if (!expect(Lexer.TokenType.COLON, "expected ':' in conditional expression")) { + return lhs; + } + CelExpr falseExpr = parseExpr(); + return CelExpr.newBuilder() + .setId(opId) + .setCall( + CelExpr.CelCall.newBuilder() + .setFunction(Operator.CONDITIONAL.getFunction()) + .addArgs(lhs) + .addArgs(trueExpr) + .addArgs(falseExpr) + .build()) + .build(); + } + + private CelExpr buildBinaryCall(long opId, String opName, CelExpr lhs, CelExpr rhs) { + return CelExpr.newBuilder() + .setId(opId) + .setCall(CelExpr.CelCall.newBuilder().setFunction(opName).addArgs(lhs).addArgs(rhs).build()) + .build(); + } + + private CelExpr parseBalancedLogicalChain(CelExpr lhs, BinaryOpInfo opInfo) { + List terms = new ArrayList<>(); + List ops = new ArrayList<>(); + terms.add(lhs); + while (peekToken.type == opInfo.type) { + Lexer.Token opTok = nextToken(); + CelExpr rhs = parseBinaryAndTernary(opInfo.precedence + 1); + ops.add(nextId(opTok)); + terms.add(rhs); + } + return balancedTree(opInfo.name, terms, ops, 0, ops.size() - 1); + } + + private CelExpr balancedTree(String op, List terms, List ops, int lo, int hi) { + int mid = (lo + hi + 1) / 2; + CelExpr left; + if (mid == lo) { + left = terms.get(mid); + } else { + left = balancedTree(op, terms, ops, lo, mid - 1); + } + CelExpr right; + if (mid == hi) { + right = terms.get(mid + 1); + } else { + right = balancedTree(op, terms, ops, mid + 1, hi); + } + return CelExpr.newBuilder() + .setId(ops.get(mid)) + .setCall(CelExpr.CelCall.newBuilder().setFunction(op).addArgs(left).addArgs(right).build()) + .build(); + } + + private CelExpr parseSelectorChain() { + CelExpr lhs = parseUnary(); + currentLhsDepth = 0; + Lexer.TokenType tok = peekToken.type; + if (tok == Lexer.TokenType.DOT + || tok == Lexer.TokenType.LEFT_BRACKET + || tok == Lexer.TokenType.LEFT_BRACE) { + lhs = parseSelectorChainTail(lhs); + } + return lhs; + } + + private CelExpr parseSelectorChainTail(CelExpr initialLhs) { + CelExpr lhs = initialLhs; + int chainDepth = 0; + while (true) { + Lexer.TokenType tok = peekToken.type; + if (tok == Lexer.TokenType.DOT) { + chainDepth++; + if (checkRecursion(chainDepth, peekToken)) { + return ERROR; + } + Lexer.Token dotTok = nextToken(); + boolean optional = false; + if (peekToken.type == Lexer.TokenType.QUESTION) { + nextToken(); + optional = true; + if (!options.enableOptionalSyntax()) { + reportError(dotTok.start, "unsupported syntax '.?'"); + } + } + Lexer.Token idTok = nextToken(); + if (idTok.type != Lexer.TokenType.IDENT + && idTok.type != Lexer.TokenType.RESERVED_WORD + && idTok.type != Lexer.TokenType.IN) { + if (idTok.type != Lexer.TokenType.ERROR) { + reportSyntaxError(idTok, "expected identifier after '.'"); + } + synchronizeOnDelimiter(); + currentLhsDepth = chainDepth; + return lhs; + } + boolean isMemberCall = (peekToken.type == Lexer.TokenType.LEFT_PAREN); + String idText = normalizeIdent(idTok, /* allowQuoted= */ !isMemberCall); + if (optional) { + long opId = nextId(dotTok); + CelExpr arg1 = lhs; + CelExpr arg2 = + CelExpr.newBuilder() + .setId(nextId(idTok)) + .setConstant(CelConstant.ofValue(idText)) + .build(); + lhs = + CelExpr.newBuilder() + .setId(opId) + .setCall( + CelExpr.CelCall.newBuilder() + .setFunction(Operator.OPTIONAL_SELECT.getFunction()) + .addArgs(arg1) + .addArgs(arg2) + .build()) + .build(); + } else if (peekToken.type == Lexer.TokenType.LEFT_PAREN) { + Lexer.Token lparen = nextToken(); + long callId = nextId(lparen); + ImmutableList args = parseArguments(Lexer.TokenType.RIGHT_PAREN); + Optional expanded = tryExpandMacro(callId, idText, lhs, args); + if (expanded.isPresent()) { + lhs = expanded.get(); + } else { + lhs = + CelExpr.newBuilder() + .setId(callId) + .setCall( + CelExpr.CelCall.newBuilder() + .setFunction(idText) + .setTarget(lhs) + .addArgs(args) + .build()) + .build(); + } + } else { + lhs = + CelExpr.newBuilder() + .setId(nextId(dotTok)) + .setSelect( + CelExpr.CelSelect.newBuilder().setOperand(lhs).setField(idText).build()) + .build(); + } + } else if (tok == Lexer.TokenType.LEFT_BRACKET) { + chainDepth++; + if (checkRecursion(chainDepth, peekToken)) { + return ERROR; + } + Lexer.Token bracketTok = nextToken(); + long opId = nextId(bracketTok); + boolean optional = false; + if (peekToken.type == Lexer.TokenType.QUESTION) { + nextToken(); + optional = true; + if (!options.enableOptionalSyntax()) { + reportError(bracketTok.start, "unsupported syntax '?'"); + } + } + CelExpr index = parseExpr(); + expect(Lexer.TokenType.RIGHT_BRACKET, "expected ']'"); + String opName = + optional ? Operator.OPTIONAL_INDEX.getFunction() : Operator.INDEX.getFunction(); + lhs = + CelExpr.newBuilder() + .setId(opId) + .setCall( + CelExpr.CelCall.newBuilder() + .setFunction(opName) + .addArgs(lhs) + .addArgs(index) + .build()) + .build(); + } else if (tok == Lexer.TokenType.LEFT_BRACE) { + int structPos = getLeftmostPosition(lhs); + Optional structName = extractStructName(lhs); + if (structName.isPresent()) { + lhs = parseStruct(nextId(structPos), structName.get()); + } else { + break; + } + } else { + break; + } + } + currentLhsDepth = chainDepth; + return lhs; + } + + private CelExpr parseUnary() { + Lexer.TokenType tok = peekToken.type; + if (tok == Lexer.TokenType.EXCLAMATION || tok == Lexer.TokenType.MINUS) { + return parseUnaryOps(); + } + return parsePrimary(); + } + + private CelExpr parseUnaryOps() { + Lexer.Token op = nextToken(); + Lexer.TokenType opType = op.type; + if (peekToken.type == Lexer.TokenType.EXCLAMATION || peekToken.type == Lexer.TokenType.MINUS) { + return parseUnaryOpsChain(op); + } + + if (opType == Lexer.TokenType.MINUS) { + if (peekToken.type == Lexer.TokenType.INT) { + return parseIntLiteral(nextId(op), /* isNegative= */ true); + } + if (peekToken.type == Lexer.TokenType.FLOAT) { + return parseDoubleLiteral(nextId(op), /* isNegative= */ true); + } + } + + if (checkRecursion(1, op)) { + return ERROR; + } + + long opId = nextId(op); + recursionDepth++; + CelExpr operand = parseSelectorChain(); + recursionDepth--; + if (recursionLimitExceeded) { + return ERROR; + } + + String opName = + (opType == Lexer.TokenType.EXCLAMATION) + ? Operator.LOGICAL_NOT.getFunction() + : Operator.NEGATE.getFunction(); + return CelExpr.newBuilder() + .setId(opId) + .setCall(CelExpr.CelCall.newBuilder().setFunction(opName).addArgs(operand).build()) + .build(); + } + + private CelExpr parseUnaryOpsChain(Lexer.Token firstOp) { + List ops = new ArrayList<>(); + ops.add(new UnaryOp(firstOp)); + while (peekToken.type == Lexer.TokenType.EXCLAMATION + || peekToken.type == Lexer.TokenType.MINUS) { + ops.add(new UnaryOp(nextToken())); + } + + boolean hasSolitaryTrailingMinus = + !ops.isEmpty() + && Iterables.getLast(ops).token.type == Lexer.TokenType.MINUS + && (ops.size() == 1 || ops.get(ops.size() - 2).token.type != Lexer.TokenType.MINUS); + + if (!options.retainRepeatedUnaryOperators()) { + int write = 0; + for (int read = 0; read < ops.size(); ) { + int next = read; + while (next < ops.size() && ops.get(next).token.type == ops.get(read).token.type) { + next++; + } + if ((next - read) % 2 != 0) { + ops.set(write++, ops.get(read)); + } + read = next; + } + ops = new ArrayList<>(ops.subList(0, write)); + } + + for (UnaryOp op : ops) { + op.id = nextId(op.token); + } + + boolean isNegativeNumericLiteral = + hasSolitaryTrailingMinus + && (peekToken.type == Lexer.TokenType.INT || peekToken.type == Lexer.TokenType.FLOAT); + long negativeLiteralOpId = 0; + if (isNegativeNumericLiteral) { + negativeLiteralOpId = Iterables.getLast(ops).id; + ops.remove(ops.size() - 1); + } + + int chainDepth = 0; + for (UnaryOp op : ops) { + chainDepth++; + if (checkRecursion(chainDepth, op.token)) { + return ERROR; + } + } + + recursionDepth += ops.size(); + CelExpr operand; + if (isNegativeNumericLiteral) { + operand = + (peekToken.type == Lexer.TokenType.INT) + ? parseIntLiteral(negativeLiteralOpId, /* isNegative= */ true) + : parseDoubleLiteral(negativeLiteralOpId, /* isNegative= */ true); + operand = parseSelectorChainTail(operand); + } else { + operand = parseSelectorChain(); + } + recursionDepth -= ops.size(); + + if (recursionLimitExceeded) { + return ERROR; + } + + for (int i = ops.size() - 1; i >= 0; --i) { + String opName = + (ops.get(i).token.type == Lexer.TokenType.EXCLAMATION) + ? Operator.LOGICAL_NOT.getFunction() + : Operator.NEGATE.getFunction(); + operand = + CelExpr.newBuilder() + .setId(ops.get(i).id) + .setCall(CelExpr.CelCall.newBuilder().setFunction(opName).addArgs(operand).build()) + .build(); + } + + return operand; + } + + private CelExpr parseIdentOrCall() { + Lexer.TokenType tokType = peekToken.type; + boolean leadingDot = false; + Lexer.Token firstTok = peekToken; + if (tokType == Lexer.TokenType.DOT) { + nextToken(); + leadingDot = true; + } + Lexer.Token idTok = nextToken(); + if (idTok.type != Lexer.TokenType.IDENT && idTok.type != Lexer.TokenType.RESERVED_WORD) { + if (idTok.type != Lexer.TokenType.ERROR) { + reportSyntaxError(idTok, "expected identifier"); + } + return CelExpr.newBuilder().setId(nextId(idTok)).build(); + } + String idText = normalizeIdent(idTok, /* allowQuoted= */ false); + if (idTok.type == Lexer.TokenType.RESERVED_WORD && options.enableReservedIds()) { + reportError(idTok.start, String.format("reserved identifier: %s", idText)); + } + String name = leadingDot ? "." + idText : idText; + if (peekToken.type == Lexer.TokenType.LEFT_PAREN) { + Lexer.Token lparen = nextToken(); + long callId = nextId(lparen); + ImmutableList args = parseArguments(Lexer.TokenType.RIGHT_PAREN); + Optional expanded = tryExpandMacro(callId, name, null, args); + if (expanded.isPresent()) { + return expanded.get(); + } + return CelExpr.newBuilder() + .setId(callId) + .setCall(CelExpr.CelCall.newBuilder().setFunction(name).addArgs(args).build()) + .build(); + } + long id = nextId(leadingDot ? firstTok : idTok); + return CelExpr.newBuilder() + .setId(id) + .setIdent(CelExpr.CelIdent.newBuilder().setName(name).build()) + .build(); + } + + private CelExpr parsePrimary() { + switch (peekToken.type) { + case LEFT_PAREN: + { + int groupingParenCount = countGroupingParentheses(); + for (int i = 0; i < groupingParenCount; ++i) { + nextToken(); + } + CelExpr expr = parseExpr(); + for (int i = 0; i < groupingParenCount; ++i) { + expect(Lexer.TokenType.RIGHT_PAREN, ""); + } + return expr; + } + case NULL: + return CelExpr.newBuilder().setId(nextId(nextToken())).setConstant(Constants.NULL).build(); + case TRUE: + case FALSE: + { + Lexer.Token tok = nextToken(); + return CelExpr.newBuilder() + .setId(nextId(tok)) + .setConstant(tok.type == Lexer.TokenType.TRUE ? Constants.TRUE : Constants.FALSE) + .build(); + } + case INT: + return parseIntLiteral(/* nodeId= */ -1, /* isNegative= */ false); + case UINT: + return parseUintLiteral(); + case FLOAT: + return parseDoubleLiteral(/* nodeId= */ -1, /* isNegative= */ false); + case STRING: + return parseStringLiteral(); + case BYTES: + return parseBytesLiteral(); + case LEFT_BRACKET: + return parseList(); + case LEFT_BRACE: + return parseMap(); + case DOT: + case IDENT: + case RESERVED_WORD: + return parseIdentOrCall(); + default: + { + Lexer.Token badTok = nextToken(); + if (badTok.type != Lexer.TokenType.ERROR) { + if (badTok.type == Lexer.TokenType.END) { + reportSyntaxError(badTok, "mismatched input '' expecting expression"); + } else { + reportSyntaxError(badTok, "unexpected token"); + } + } + return CelExpr.newBuilder().setId(nextId(badTok)).build(); + } + } + } + + private CelExpr parseList() { + Lexer.Token openTok = nextToken(); + long listId = nextId(openTok); + CelExpr.CelList.Builder listBuilder = CelExpr.CelList.newBuilder(); + int elemIndex = 0; + while (peekToken.type != Lexer.TokenType.RIGHT_BRACKET + && peekToken.type != Lexer.TokenType.END) { + boolean optional = false; + if (peekToken.type == Lexer.TokenType.QUESTION) { + Lexer.Token q = nextToken(); + optional = true; + if (!options.enableOptionalSyntax()) { + reportError(q.start, "unsupported syntax '?'"); + } + } + listBuilder.addElements(parseExpr()); + if (optional) { + listBuilder.addOptionalIndices(elemIndex); + } + elemIndex++; + if (peekToken.type == Lexer.TokenType.COMMA) { + nextToken(); + } else { + break; + } + } + expect(Lexer.TokenType.RIGHT_BRACKET, "expected ']'"); + return CelExpr.newBuilder().setId(listId).setList(listBuilder.build()).build(); + } + + private CelExpr parseMap() { + Lexer.Token openTok = nextToken(); + long mapId = nextId(openTok); + CelExpr.CelMap.Builder mapBuilder = CelExpr.CelMap.newBuilder(); + while (peekToken.type != Lexer.TokenType.RIGHT_BRACE && peekToken.type != Lexer.TokenType.END) { + boolean optional = false; + Lexer.Token keyStart = peekToken; + if (keyStart.type == Lexer.TokenType.QUESTION) { + Lexer.Token q = nextToken(); + optional = true; + if (!options.enableOptionalSyntax()) { + reportError(q.start, "unsupported syntax '?'"); + } + keyStart = peekToken; + } + long entryId = nextId(); + CelExpr key = parseExpr(); + Lexer.Token colon = peekToken; + if (!expect(Lexer.TokenType.COLON, "expected ':' in map entry")) { + break; + } + setPosition(entryId, colon); + CelExpr value = parseExpr(); + mapBuilder.addEntries( + CelExpr.CelMap.Entry.newBuilder() + .setId(entryId) + .setKey(key) + .setValue(value) + .setOptionalEntry(optional) + .build()); + if (peekToken.type == Lexer.TokenType.COMMA) { + nextToken(); + } else { + break; + } + } + expect(Lexer.TokenType.RIGHT_BRACE, "expected '}'"); + return CelExpr.newBuilder().setId(mapId).setMap(mapBuilder.build()).build(); + } + + private CelExpr parseStruct(long objId, String structName) { + nextToken(); + CelExpr.CelStruct.Builder structBuilder = + CelExpr.CelStruct.newBuilder().setMessageName(structName); + while (peekToken.type != Lexer.TokenType.RIGHT_BRACE && peekToken.type != Lexer.TokenType.END) { + boolean optional = false; + if (peekToken.type == Lexer.TokenType.QUESTION) { + Lexer.Token q = nextToken(); + optional = true; + if (!options.enableOptionalSyntax()) { + reportError(q.start, "unsupported syntax '?'"); + } + } + Lexer.Token fieldTok = nextToken(); + if (fieldTok.type != Lexer.TokenType.IDENT + && fieldTok.type != Lexer.TokenType.RESERVED_WORD) { + reportSyntaxError(fieldTok, "expected struct field name"); + synchronizeOnDelimiter(); + break; + } + String fieldName = normalizeIdent(fieldTok, /* allowQuoted= */ true); + Lexer.Token colon = peekToken; + if (!expect(Lexer.TokenType.COLON, "expected ':' in struct field")) { + break; + } + long fieldId = nextId(colon); + CelExpr value = parseExpr(); + structBuilder.addEntries( + CelExpr.CelStruct.Entry.newBuilder() + .setId(fieldId) + .setFieldKey(fieldName) + .setValue(value) + .setOptionalEntry(optional) + .build()); + if (peekToken.type == Lexer.TokenType.COMMA) { + nextToken(); + } else { + break; + } + } + expect(Lexer.TokenType.RIGHT_BRACE, "expected '}'"); + return CelExpr.newBuilder().setId(objId).setStruct(structBuilder.build()).build(); + } + + private ImmutableList parseArguments(Lexer.TokenType closeToken) { + ImmutableList.Builder args = ImmutableList.builder(); + if (peekToken.type != closeToken && peekToken.type != Lexer.TokenType.END) { + while (true) { + args.add(parseExpr()); + if (peekToken.type == Lexer.TokenType.COMMA) { + nextToken(); + if (peekToken.type == closeToken) { + reportError(peekToken.start, "unexpected token"); + break; + } + continue; + } + break; + } + } + expect(closeToken, ""); + return args.build(); + } + + private CelExpr parseIntLiteral(long nodeId, boolean isNegative) { + Lexer.Token tok = nextToken(); + String text = isNegative ? "-" + getTokenText(tok) : getTokenText(tok); + long id = nodeId == -1 ? nextId(tok) : nodeId; + try { + CelConstant constExpr = Constants.parseInt(text); + return CelExpr.newBuilder().setId(id).setConstant(constExpr).build(); + } catch (ParseException e) { + reportSyntaxError(tok, "invalid int literal"); + return CelExpr.newBuilder().setId(nextId(tok)).build(); + } + } + + private CelExpr parseUintLiteral() { + Lexer.Token tok = nextToken(); + String value = getTokenText(tok); + try { + CelConstant constExpr = Constants.parseUint(value); + return CelExpr.newBuilder().setId(nextId(tok)).setConstant(constExpr).build(); + } catch (ParseException e) { + reportSyntaxError(tok, "invalid uint literal"); + return CelExpr.newBuilder().setId(nextId(tok)).build(); + } + } + + private CelExpr parseDoubleLiteral(long nodeId, boolean isNegative) { + Lexer.Token tok = nextToken(); + String text = isNegative ? "-" + getTokenText(tok) : getTokenText(tok); + long id = nodeId == -1 ? nextId(tok) : nodeId; + try { + CelConstant constExpr = Constants.parseDouble(text); + if (Double.isInfinite(constExpr.doubleValue())) { + reportSyntaxError(tok, "invalid double literal"); + return CelExpr.newBuilder().setId(id).build(); + } + return CelExpr.newBuilder().setId(id).setConstant(constExpr).build(); + } catch (ParseException e) { + reportSyntaxError(tok, "invalid double literal"); + return CelExpr.newBuilder().setId(nextId(tok)).build(); + } + } + + private CelExpr parseStringLiteral() { + Lexer.Token tok = nextToken(); + String value = getTokenText(tok); + try { + CelConstant constExpr = Constants.parseString(value); + return CelExpr.newBuilder().setId(nextId(tok)).setConstant(constExpr).build(); + } catch (ParseException e) { + reportError(tok.start, e.getMessage()); + return CelExpr.newBuilder().setId(nextId(tok)).build(); + } + } + + private CelExpr parseBytesLiteral() { + Lexer.Token tok = nextToken(); + String value = getTokenText(tok); + try { + CelConstant constExpr = Constants.parseBytes(value); + return CelExpr.newBuilder().setId(nextId(tok)).setConstant(constExpr).build(); + } catch (ParseException e) { + reportError(tok.start, e.getMessage()); + return CelExpr.newBuilder().setId(nextId(tok)).build(); + } + } + + private String normalizeIdent(Lexer.Token tok, boolean allowQuoted) { + String text = getTokenText(tok); + if (text.isEmpty()) { + return ""; + } + if (text.charAt(0) == '`') { + if (!allowQuoted) { + reportError(tok.start, "unexpected quoted identifier"); + return ""; + } + if (!options.enableQuotedIdentifierSyntax()) { + reportError(tok.start, "unsupported syntax '`'"); + } + if (text.length() < 2 || text.charAt(text.length() - 1) != '`') { + reportError(tok.start, "unterminated quoted identifier"); + return ""; + } + String inner = text.substring(1, text.length() - 1); + if (inner.isEmpty()) { + reportError(tok.start, "unexpected quoted identifier"); + return ""; + } + for (int i = 0; i < inner.length(); i++) { + char c = inner.charAt(i); + if (!isAsciiAlphanumeric(c) && c != '_' && c != '.' && c != '-' && c != '/' && c != ' ') { + reportError(tok.start, "unexpected quoted identifier"); + return ""; + } + } + return inner; + } + return text; + } + + private static boolean isAsciiAlphanumeric(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + } + + private Optional extractStructName(CelExpr expr) { + if (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.IDENT) { + String name = expr.ident().name(); + eraseId(expr.id()); + return Optional.of(name); + } + if (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.SELECT) { + if (expr.select().testOnly()) { + return Optional.empty(); + } + CelExpr operand = expr.select().operand(); + eraseId(expr.id()); + Optional prefix = extractStructName(operand); + if (!prefix.isPresent()) { + return Optional.empty(); + } + return Optional.of(prefix.get() + "." + expr.select().field()); + } + return Optional.empty(); + } + + private int getLeftmostPosition(CelExpr expr) { + if (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.IDENT) { + return positions.getOrDefault(expr.id(), 0); + } + if (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.SELECT) { + return getLeftmostPosition(expr.select().operand()); + } + return positions.getOrDefault(expr.id(), 0); + } + + private Optional lookupMacro(String id, int argCount, boolean receiverStyle) { + String key = CelMacro.formatKey(id, argCount, receiverStyle); + CelMacro macro = macros.get(key); + if (macro != null) { + return Optional.of(macro); + } + key = CelMacro.formatVarArgKey(id, receiverStyle); + return Optional.ofNullable(macros.get(key)); + } + + private Optional tryExpandMacro( + long exprId, String function, @Nullable CelExpr target, ImmutableList args) { + if (function.isEmpty()) { + return Optional.empty(); + } + boolean isReceiver = (target != null); + int argCount = args.size(); + Optional macro = lookupMacro(function, argCount, isReceiver); + if (!macro.isPresent()) { + return Optional.empty(); + } + if (nodeLimitExceeded) { + reportError( + positions.getOrDefault(exprId, 0), + "could not expand macro: expression node limit exceeded"); + return Optional.empty(); + } + + Optional errorArg = args.stream().filter(ERROR::equals).findAny(); + if (errorArg.isPresent() || (target != null && target.equals(ERROR))) { + eraseId(exprId); + return Optional.of(ERROR); + } + + int macroPosition = positions.getOrDefault(exprId, 0); + CelExpr targetExpr = (target != null ? target : CelExpr.newBuilder().build()); + Optional expandedExpr = expandMacro(macroPosition, macro.get(), targetExpr, args); + + if (expandedExpr.isPresent()) { + if (options.populateMacroCalls()) { + recordMacroCall(expandedExpr.get().id(), function, target, args); + } + eraseId(exprId); + return expandedExpr; + } + return Optional.empty(); + } + + private Optional expandMacro( + int position, CelMacro macro, CelExpr target, ImmutableList arguments) { + macroExprFactory.pushPosition(position); + try { + return macro.getExpander().expandMacro(macroExprFactory, target, arguments); + } finally { + macroExprFactory.popPosition(); + } + } + + private void recordMacroCall( + long macroId, String function, CelExpr target, ImmutableList args) { + CelExpr.CelCall.Builder callExpr = CelExpr.CelCall.newBuilder().setFunction(function); + if (target != null) { + if (macroCalls.containsKey(target.id())) { + callExpr.setTarget(CelExpr.newBuilder().setId(target.id()).build()); + } else { + callExpr.setTarget(buildMacroCallArgs(target)); + } + } + for (CelExpr arg : args) { + callExpr.addArgs(buildMacroCallArgs(arg)); + } + macroCalls.put(macroId, CelExpr.newBuilder().setCall(callExpr.build()).build()); + } + + private CelExpr buildMacroCallArgs(CelExpr expr) { + CelExpr.Builder resultExpr = CelExpr.newBuilder().setId(expr.id()); + if (macroCalls.containsKey(expr.id())) { + return resultExpr.build(); + } + if (expr.exprKind().getKind() == CelExpr.ExprKind.Kind.CALL) { + CelExpr.CelCall.Builder callExpr = + CelExpr.CelCall.newBuilder().setFunction(expr.call().function()); + expr.call().args().forEach(arg -> callExpr.addArgs(buildMacroCallArgs(arg))); + expr.call().target().ifPresent(target -> callExpr.setTarget(buildMacroCallArgs(target))); + return resultExpr.setCall(callExpr.build()).build(); + } + return expr; + } + + private int countGroupingParentheses() { + if (peekToken.type != Lexer.TokenType.LEFT_PAREN) { + return 0; + } + + int savedPos = lexer.savePosition(); + try { + int leadingOpenParens = 1; + Lexer.Token tok = nextSignificantToken(/* reportError= */ false); + while (tok.type == Lexer.TokenType.LEFT_PAREN) { + leadingOpenParens++; + tok = nextSignificantToken(/* reportError= */ false); + } + if (leadingOpenParens == 1) { + return 1; + } + + int openParens = leadingOpenParens; + int consecutiveLeadingClosed = 0; + + while (openParens > 0) { + if (tok.type == Lexer.TokenType.END || tok.type == Lexer.TokenType.ERROR) { + return 1; + } + + if (tok.type == Lexer.TokenType.LEFT_PAREN) { + openParens++; + consecutiveLeadingClosed = 0; + } else if (tok.type == Lexer.TokenType.RIGHT_PAREN) { + if (leadingOpenParens == openParens) { + leadingOpenParens--; + consecutiveLeadingClosed++; + } else { + consecutiveLeadingClosed = 0; + } + openParens--; + } else { + consecutiveLeadingClosed = 0; + } + + if (openParens > 0) { + tok = nextSignificantToken(/* reportError= */ false); + } + } + + return Math.max(1, consecutiveLeadingClosed); + } finally { + lexer.restorePosition(savedPos); + } + } + + private final class PrattMacroExprFactory extends CelMacroExprFactory { + private final ArrayDeque macroPositions = new ArrayDeque<>(1); + + void pushPosition(int position) { + macroPositions.addLast(position); + } + + void popPosition() { + macroPositions.removeLast(); + } + + int peekPosition() { + return macroPositions.peekLast(); + } + + @Override + public CelExpr reportError(CelIssue error) { + issues.add(error); + if (!error.getSourceLocation().equals(CelSourceLocation.NONE)) { + Optional offset = source.getLocationOffset(error.getSourceLocation()); + if (offset.isPresent()) { + return CelExpr.newBuilder().setId(nextId(offset.get())).build(); + } + } + return ERROR; + } + + @Override + public String getAccumulatorVarName() { + return ACCUMULATOR_NAME; + } + + @Override + protected CelSourceLocation getSourceLocation(long exprId) { + int pos = positions.getOrDefault(exprId, -1); + return source.getOffsetLocation(pos).orElse(CelSourceLocation.NONE); + } + + @Override + protected CelSourceLocation currentSourceLocationForMacro() { + int pos = + !macroPositions.isEmpty() + ? peekPosition() + : (currentToken != null ? currentToken.start : 0); + return source.getOffsetLocation(pos).orElse(CelSourceLocation.NONE); + } + + @Override + protected long copyExprId(long id) { + return copyId(id); + } + + @Override + public long nextExprId() { + int pos = !macroPositions.isEmpty() ? peekPosition() : -1; + return nextId(pos); + } + } +} diff --git a/parser/src/test/java/dev/cel/parser/BUILD.bazel b/parser/src/test/java/dev/cel/parser/BUILD.bazel index 1ade0181d..eea155e92 100644 --- a/parser/src/test/java/dev/cel/parser/BUILD.bazel +++ b/parser/src/test/java/dev/cel/parser/BUILD.bazel @@ -10,10 +10,12 @@ package( java_library( name = "tests", testonly = True, - srcs = glob(["*Test.java"]), + srcs = glob( + ["*Test.java"], + exclude = ["TmpPrattParserTest.java"], + ), resources = ["//parser/src/test/resources:baselines"], deps = [ - "//:auto_value", "//:java_truth", "//common:cel_ast", "//common:cel_source", @@ -30,12 +32,12 @@ java_library( "//parser:macro", "//parser:parser_builder", "//parser:parser_factory", + "//parser:pratt_parser", "//parser:unparser", "//parser:unparser_visitor", "//testing:adorner", "//testing:baseline_test_case", "@cel_spec//proto/cel/expr:syntax_java_proto", - "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_guava_guava_testlib", "@maven//:com_google_protobuf_protobuf_java", diff --git a/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java b/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java index 019cea520..7c364cbb9 100644 --- a/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java +++ b/parser/src/test/java/dev/cel/parser/CelParserParameterizedTest.java @@ -14,24 +14,10 @@ package dev.cel.parser; -import static java.util.Collections.reverseOrder; -import static java.util.Map.Entry.comparingByKey; -import static java.util.stream.Collectors.joining; -import dev.cel.expr.Constant; -import dev.cel.expr.Expr; -import dev.cel.expr.ExprOrBuilder; import dev.cel.expr.ParsedExpr; import dev.cel.expr.SourceInfo; -import com.google.auto.value.AutoValue; -import com.google.common.base.Ascii; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; -import com.google.errorprone.annotations.Immutable; -import com.google.protobuf.Descriptors.Descriptor; -import com.google.protobuf.Descriptors.EnumDescriptor; -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.TextFormat; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import dev.cel.common.CelAbstractSyntaxTree; @@ -44,11 +30,9 @@ import dev.cel.common.ast.CelExpr; import dev.cel.extensions.CelOptionalLibrary; import dev.cel.testing.BaselineTestCase; -import dev.cel.testing.CelAdorner; import dev.cel.testing.CelDebug; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Map; +import dev.cel.testing.CelExprKindAndIdAdorner; +import dev.cel.testing.CelLocationAdorner; import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -349,16 +333,18 @@ private void runTest(CelParser parser, String expression, boolean validateParseO testOutput() .println( "P: " - + CelDebug.toAdornedDebugString(parsedExpr.getExpr(), new KindAndIdAdorner())); + + CelDebug.toAdornedDebugString( + parsedExpr.getExpr(), new CelExprKindAndIdAdorner())); String locationOutput = CelDebug.toAdornedDebugString( - parsedExpr.getExpr(), new LocationAdorner(parsedExpr.getSourceInfo())); + parsedExpr.getExpr(), new CelLocationAdorner(parsedExpr.getSourceInfo())); if (!locationOutput.isEmpty()) { testOutput().println("L: " + locationOutput); } } - String macroOutput = convertMacroCallsToString(parsedExpr.getSourceInfo()); + String macroOutput = + CelExprKindAndIdAdorner.convertMacroCallsToString(parsedExpr.getSourceInfo()); if (!macroOutput.isEmpty()) { testOutput().println("M: " + macroOutput); } @@ -377,152 +363,4 @@ private void runSourceInfoTest(String expression) throws Exception { testOutput().println("=====>"); testOutput().println("S: " + TextFormat.printer().printToString(sourceInfo)); } - - private String convertMacroCallsToString(SourceInfo sourceInfo) { - KindAndIdAdorner macroCallsAdorner = new KindAndIdAdorner(sourceInfo); - // Sort in ascending order so that nested macro calls are always in the same order for tests - // output debug string. Ascending order keeps the macro calls map in order from outermost/first - // macro to the innermost/last macro for readability. - return sourceInfo.getMacroCallsMap().entrySet().stream() - .sorted(reverseOrder(comparingByKey())) - .map((entry) -> CelDebug.toAdornedDebugString(entry.getValue(), macroCallsAdorner)) - .collect(joining(",\n")); - } - - private static final class KindAndIdAdorner implements CelAdorner { - - private final SourceInfo sourceInfo; - - KindAndIdAdorner() { - this(SourceInfo.getDefaultInstance()); - } - - KindAndIdAdorner(SourceInfo sourceInfo) { - this.sourceInfo = sourceInfo; - } - - @Override - public String adorn(ExprOrBuilder expr) { - if (this.sourceInfo != null && this.sourceInfo.containsMacroCalls(expr.getId())) { - return String.format( - "^#%d:%s#", - expr.getId(), - this.sourceInfo.getMacroCallsOrThrow(expr.getId()).getCallExpr().getFunction()); - } - - if (expr.hasConstExpr()) { - Constant constExpr = expr.getConstExpr(); - Descriptor descriptor = Constant.getDescriptor(); - OneofDescriptor oneof = findOneofByName(descriptor, "constant_kind"); - FieldDescriptor field = constExpr.getOneofFieldDescriptor(oneof); - if (field.getType() == FieldDescriptor.Type.ENUM) { - return String.format("^#%d:%s#", expr.getId(), getContainedName(field.getEnumType())); - } else { - return String.format( - "^#%d:%s#", expr.getId(), Ascii.toLowerCase(field.getType().toString())); - } - } - Descriptor descriptor = Expr.getDescriptor(); - OneofDescriptor oneof = findOneofByName(descriptor, "expr_kind"); - FieldDescriptor field = expr.getOneofFieldDescriptor(oneof); - return String.format("^#%d:%s#", expr.getId(), getContainedName(field.getMessageType())); - } - - @Override - public String adorn(Expr.CreateStruct.EntryOrBuilder entry) { - return String.format("^#%d:Expr.CreateStruct.Entry#", entry.getId()); - } - } - - @AutoValue - @Immutable - abstract static class LineAndColumn { - - public abstract int getLine(); - - public abstract int getColumn(); - } - - private static final class LocationAdorner implements CelAdorner { - - private final SourceInfo sourceInfo; - - LocationAdorner(SourceInfo sourceInfo) { - this.sourceInfo = sourceInfo; - } - - @Override - public String adorn(ExprOrBuilder expr) { - return getLocation(expr.getId()) - .map( - location -> - String.format( - "^#%d[%d,%d]#", expr.getId(), location.getLine(), location.getColumn())) - .orElseGet(() -> String.format("^#%d[NO_POS]#", expr.getId())); - } - - @Override - public String adorn(Expr.CreateStruct.EntryOrBuilder entry) { - return getLocation(entry.getId()) - .map( - location -> - String.format( - "^#%d[%d,%d]#", entry.getId(), location.getLine(), location.getColumn())) - .orElseGet(() -> String.format("^#%d[NO_POS]#", entry.getId())); - } - - private Optional getLocation(long exprId) { - Map positions = sourceInfo.getPositionsMap(); - Integer position = positions.get(exprId); - if (position == null) { - return Optional.empty(); - } - int line = 1; - for (int index = 0; index < sourceInfo.getLineOffsetsCount(); index++) { - if (sourceInfo.getLineOffsets(index) > position) { - break; - } else { - line++; - } - } - int column = position; - if (line > 1) { - column = position - sourceInfo.getLineOffsets(line - 2); - } - return Optional.of(new AutoValue_CelParserParameterizedTest_LineAndColumn(line, column)); - } - } - - private static OneofDescriptor findOneofByName(Descriptor descriptor, String name) { - for (OneofDescriptor oneof : descriptor.getOneofs()) { - if (oneof.getName().equals(name)) { - return oneof; - } - } - return null; - } - - private static final Joiner JOINER = Joiner.on('.'); - - private static String getContainedName(Descriptor descriptor) { - Deque parts = new ArrayDeque<>(); - parts.addFirst(descriptor.getName()); - Descriptor containing = descriptor.getContainingType(); - while (containing != null) { - parts.addFirst(containing.getName()); - containing = containing.getContainingType(); - } - return JOINER.join(parts); - } - - private static String getContainedName(EnumDescriptor descriptor) { - Deque parts = new ArrayDeque<>(); - parts.addFirst(descriptor.getName()); - Descriptor containing = descriptor.getContainingType(); - while (containing != null) { - parts.addFirst(containing.getName()); - containing = containing.getContainingType(); - } - return JOINER.join(parts); - } } diff --git a/parser/src/test/java/dev/cel/parser/PrattParserTest.java b/parser/src/test/java/dev/cel/parser/PrattParserTest.java new file mode 100644 index 000000000..2e86d9178 --- /dev/null +++ b/parser/src/test/java/dev/cel/parser/PrattParserTest.java @@ -0,0 +1,530 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.parser; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import dev.cel.expr.ParsedExpr; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.common.CelOptions; +import dev.cel.common.CelProtoAbstractSyntaxTree; +import dev.cel.common.CelSource; +import dev.cel.common.CelValidationException; +import dev.cel.common.CelValidationResult; +import dev.cel.testing.BaselineTestCase; +import dev.cel.testing.CelDebug; +import dev.cel.testing.CelExprKindAndIdAdorner; +import dev.cel.testing.CelLocationAdorner; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class PrattParserTest extends BaselineTestCase { + + private static final CelOptions OPTIONS = + CelOptions.current() + .populateMacroCalls(true) + .enableOptionalSyntax(true) + .enableQuotedIdentifierSyntax(true) + .build(); + + private static final CelOptions OPTIONS_MAX_RECURSION_DEPTH_32 = + OPTIONS.toBuilder().maxParseRecursionDepth(32).build(); + + private static final CelOptions OPTIONS_NO_OPTIONAL_SYNTAX = + OPTIONS.toBuilder().enableOptionalSyntax(false).build(); + + private static final CelOptions OPTIONS_QUOTED_IDENTIFIER_SYNTAX = + OPTIONS.toBuilder().enableQuotedIdentifierSyntax(true).build(); + + private static final CelOptions OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX = + OPTIONS.toBuilder().enableQuotedIdentifierSyntax(false).build(); + + private static final ImmutableMap MACROS = + ImmutableMap.builder() + .putAll( + CelStandardMacro.STANDARD_MACROS.stream() + .map(CelStandardMacro::getDefinition) + .collect(toImmutableMap(CelMacro::getKey, Function.identity()))) + .put( + CelStandardMacro.EXISTS_ONE_NEW.getDefinition().getKey(), + CelStandardMacro.EXISTS_ONE_NEW.getDefinition()) + .put( + "noop_macro", + CelMacro.newGlobalVarArgMacro("noop_macro", (a, b, c) -> Optional.empty())) + .buildOrThrow(); + + @Test + public void pratt_parser_literals() { + // Null + runTest("null"); + + // Boolean + runTest("true"); + runTest("false"); + + // Int + runTest("0"); + runTest("42"); + runTest("0xF"); + runTest("0x2A"); + runTest("-1"); + runTest("-42"); + runTest("0xFFFFFFFFFFFFFFFFF"); + runTest("9223372036854775807"); // Long.MAX_VALUE + runTest("-9223372036854775808"); // Long.MIN_VALUE + runTest("-(9223372036854775808)"); // error + + // Uint + runTest("0u"); + runTest("23u"); + runTest("0xFu"); + runTest("0xFFFFFFFFFFFFFFFFFu"); + + // Double + runTest("3.14"); + runTest("23.39"); + runTest("1.99e90000009"); + + // String + runTest("'hello'"); + runTest("\"A\""); + runTest("'''hello\nworld'''"); + runTest("\"\\u2764\""); + runTest("\"\u2764\""); + runTest("\"\\\"\""); + runTest("\"\\xC3\\XBF\""); + runTest("\"\\303\\277\""); + runTest("\"hi\\u263A \\u263Athere\""); + runTest("\"\\U000003A8\\?\""); + runTest("\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\""); + runTest("\"\"\"hello\nworld\"\"\""); + runTest("r\"\"\"hello\nworld\"\"\""); + runTest("\"\"\"hello\\\"\"\"world\"\"\""); + runTest("'''hello\\'''world'''"); + runTest("\"\\xFh\""); + runTest("\"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\""); + runTest( + " '\ud83d\ude01' in ['\ud83d\ude01', '\ud83d\ude11', '\ud83d\ude26']\n" + + "\t\t\t&& in.\ud83d\ude01"); + runTest("\"\"\"hello\nworld"); + runTest("'''hello\nworld"); + runTest("r\"\"\"hello\nworld"); + runTest("\"hello\nworld\""); + runTest("'hello\nworld'"); + runTest("r\"hello\nworld\""); + runTest("`hello\nworld`"); + runTest("\"hello\rworld\""); + + // Bytes + runTest("b'abc'"); + runTest("b\"abc\""); + runTest("b\"\"\"hello\nworld"); + runTest("b\"hello\nworld\""); + runTest("rb\"hello\nworld\""); + } + + @Test + @SuppressWarnings("InlineMeInliner") // String.repeat is unavailable under Java 8 + public void pratt_parser_core_syntax() { + // Identifiers + runTest("a"); + runTest("foo"); + + // Parentheses + runTest("(a)"); + runTest("((a))"); + runTest("(((1 + 2))) * 3"); + + // Lists + runTest("[]"); + runTest("[a]"); + runTest("[a, b, c]"); + runTest("[1, 2, 3]"); + runTest("[3, 4, 5]"); + runTest("[3, 4, 5,]"); + runTest("[?a, b]"); + runTest("[?a, ?b]"); + runTest("[?a[?b]]"); + + // Maps + runTest("{}"); + runTest("{a:b, c:d}"); + runTest("{foo: 5, bar: \"xyz\"}"); + runTest("{foo: 5, bar: \"xyz\", }"); + runTest("{\"a\": 1, \"b\": 2}"); + runTest("{1:2u, 2:3u}"); + runTest("{?a: b}"); + runTest("{?'key': value}"); + + // Messages + runTest("foo{ }"); + runTest("foo{ a:b }"); + runTest("foo{ a:b, c:d }"); + runTest("SomeMessage{foo: 5, bar: \"xyz\"}"); + runTest("TestAllTypes{single_int32: 1, single_int64: 2}"); + runTest("MyType{foo: 1, bar: 'baz'}"); + runTest("Message{`in`: true}"); + runTest("Msg{?field: value}"); + + // Field selection + runTest("a.b"); + runTest("a.b.c"); + runTest("a.?b"); + runTest("a.`b-c`"); + runTest("a.`b c`"); + runTest("a.`b.c`"); + runTest("a.`in`"); + runTest("a.`/foo`"); + runTest("a.`my-var`"); + + // Indexing + runTest("a[b]"); + runTest("a[0]"); + runTest("a[3]"); + runTest("[1,3,4][0]"); + runTest("a[?0]"); + + // Function calls + runTest("a()"); + runTest("a(b)"); + runTest("a(b, c)"); + runTest("a.b()"); + runTest("a.b(c)"); + runTest("a.b(5)"); + runTest("a.foo(1, 2)"); + + // Unary operators + runTest("!a"); + runTest("!x"); + runTest("! false"); + runTest("-a"); + + // Arithmetic operators + runTest("x * 2"); + runTest("x * 2u"); + runTest("x * 2.0"); + runTest("a * b"); + runTest("a / b"); + runTest("a % b"); + runTest("a + b"); + runTest("a - b"); + runTest("4--4"); + runTest("4--4.1"); + runTest("\"abc\" + \"def\""); + runTest("b\"abc\" + B\"def\""); + runTest("[] + [1,2,3,] + [4]"); + runTest("1 + 2 * 3"); + + // Comparison operators + runTest("a == b"); + runTest("a != b"); + runTest("a < b"); + runTest("a <= b"); + runTest("a > b"); + runTest("a >= b"); + runTest("a in b"); + runTest("\"\ud83d\ude01\" in [\"\ud83d\ude01\", \"\ud83d\ude11\", \"\ud83d\ude26\"]"); + runTest("size(x) == x.size()"); + runTest("x.single_nested_message != null"); + + // Logical operators + runTest("a && b"); + runTest("a && b && c"); + runTest("a && b && c && d && e && f && g"); + runTest("a > 5 && a < 10"); + runTest("a || b"); + runTest("a || b || c || d || e || f"); + runTest("a < 5 || a > 10"); + runTest("a && b && c && d || e && f && g && h"); + + // Conditional operator + runTest("a?b:c"); + runTest("cond ? 1 : 2"); + runTest("false && !true || false ? 2 : 3"); + runTest(OPTIONS_MAX_RECURSION_DEPTH_32, Strings.repeat("true ? 1 : ", 31) + "1", false); + runTest(OPTIONS_MAX_RECURSION_DEPTH_32, Strings.repeat("!-", 15) + "x", false); + + // Complex expressions + runTest("1 + 2 * 3 - 1 / 2 == 6 % 1"); + runTest("x[\"a\"].single_int32 == 23"); + runTest("a.?b[?0] && a[?c]"); + } + + @Test + public void pratt_parser_macros() { + runTest("has(m.f)"); + runTest("has(a.b)"); + runTest("has(m)"); + + runTest("m.all(v, f)"); + runTest("[1, 2].all(x, x > 0)"); + + runTest("m.exists(v, f)"); + + runTest("m.existsOne(v, f)"); + runTest("[].existsOne(__result__, __result__)"); + + runTest("m.map(v, f)"); + runTest("m.map(v, p, f)"); + runTest("m.map(__result__, __result__)"); + + runTest("m.filter(v, p)"); + runTest("m.filter(__result__, false)"); + runTest("m.filter(a.b, false)"); + + // Nested / Chained macros + runTest("x.filter(y, y.filter(z, z > 0))"); + runTest("has(a.b).filter(c, c)"); + runTest("x.filter(y, y.exists(z, has(z.a)) && y.exists(z, has(z.b)))"); + runTest("(has(a.b) || has(c.d)).string()"); + runTest("has(a.b).asList().exists(c, c)"); + runTest("[has(a.b), has(c.d)].exists(e, e)"); + + // Custom macros + runTest("noop_macro(123)"); + } + + @Test + @SuppressWarnings("InlineMeInliner") // String.repeat is unavailable under Java 8 + public void pratt_parser_errors() { + // Lexical errors + runTest("*@a | b"); + runTest("1 + $"); + runTest( + "\u00f3\u00a0\u00a2\n" + + "\t\t\u00f3\u00a00\u00a0\n" + + "\t\t\u007f0\"\"\"\\\"\"\\\"\"\"\\\"\"\\\"\"\"\\\"\"\\\"\"\"\\\"\"\\\"\"\"\\\"\\\"\"\"\\\"\"\\\"\"\"\\\"\"\\\"\"\"\\\"\"\\\"\"\"\\\"!\\\"\"\"\\\"\"\\\"\"\"\\\"\"\\\""); + runTest("'\\udead' == '\\ufffd'"); + runTest("a | b"); + runTest("'3# < 10\" '& tru ^^"); + + // Unexpected tokens + runTest("1 + +"); + runTest("?"); + runTest("a ? b ((?))"); + runTest( + "-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1\n" + + "\t\t--3-[-1--1--1--1---1-1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1-\u00c01--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1\n" + + "\t\t--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1\n" + + "\t\t--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1\n" + + "\t\t--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1"); + + // Reserved identifiers + runTest( + "as break const continue else for function if import in let loop package namespace" + + " return var void while"); + runTest("[1, 2, 3].map(var, var * var)"); + + // Incomplete expressions + runTest("1 +"); + runTest("--"); + runTest("{"); + + // Unexpected token after expression + runTest("TestAllTypes(){}"); + runTest("TestAllTypes{}()"); + runTest("1 + 2\n3 +"); + + // Member selection errors + runTest("{\"a\": 1}.\"a\""); + runTest("self.true == 1"); + + // Map syntax errors + runTest("{a}"); + runTest("{:a}"); + + // Message syntax errors + runTest("func{{a}}"); + runTest("msg{:a}"); + runTest("ind[a{b}]"); + runTest("x{?."); + runTest("x{."); + + // Macro errors + runTest("1.all(2, 3)"); + + // Unsupported optional syntax + runTest(OPTIONS_NO_OPTIONAL_SYNTAX, "a.?b && a[?b]"); + runTest(OPTIONS_NO_OPTIONAL_SYNTAX, "[?a, ?b]"); + runTest(OPTIONS_NO_OPTIONAL_SYNTAX, "Msg{?field: value} && {?'key': value}"); + + // Unsupported quoted identifier syntax + runTest(OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX, "a.`b-c`"); + runTest(OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX, "a.`b.c`"); + runTest(OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX, "a.`in`"); + runTest(OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX, "a.`/foo`"); + runTest(OPTIONS_NO_QUOTED_IDENTIFIER_SYNTAX, "Message{`in`: true}"); + + // Unsupported quoted identifier location + runTest(OPTIONS_QUOTED_IDENTIFIER_SYNTAX, "`b-c`"); + runTest(OPTIONS_QUOTED_IDENTIFIER_SYNTAX, "`b-c`()"); + runTest(OPTIONS_QUOTED_IDENTIFIER_SYNTAX, "a.`$b`"); + runTest(OPTIONS_QUOTED_IDENTIFIER_SYNTAX, "a.`b.c`()"); + + // Recursion limit exceeded + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\n" + + "\t\t\t[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['too many']]]]]]]]]]]]]]]]]]]]]]]]]]]]\n" + + "\t\t\t]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]\n" + + "\t\t [21][22][23][24][25][26][27][28][29][30][31][32][33]"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10\n" + + "\t\t+ 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20\n" + + "\t\t+ 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30\n" + + "\t\t+ 31 + 32 + 33 + 34"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "a < 1 < 2 < 3 < 4 < 5 < 6 < 7 < 8 < 9 < 10 < 11\n" + + "\t\t < 12 < 13 < 14 < 15 < 16 < 17 < 18 < 19 < 20 < 21\n" + + "\t\t\t < 22 < 23 < 24 < 25 < 26 < 27 < 28 < 29 < 30 < 31\n" + + "\t\t\t < 32 < 33"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "y!=y!=y!=y!=y!=y!=y!=y!=y!=-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y\n" + + "\t\t!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y\n" + + "\t\t!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y\n" + + "\t\t!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y\n" + + "\t\t!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y\n" + + "\t\t!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y"); + runTest( + OPTIONS_MAX_RECURSION_DEPTH_32, + "a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]" + + " !=\n" + + "\t\ta[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]"); + runTest(OPTIONS_MAX_RECURSION_DEPTH_32, Strings.repeat("true ? 1 : ", 33) + "1"); + runTest(OPTIONS_MAX_RECURSION_DEPTH_32, Strings.repeat("!-", 16) + "!x"); + } + + private void runTest(String expression) { + runTest(OPTIONS, expression); + } + + private void runTest(CelOptions options, String expression) { + runTest(options, expression, true); + } + + private void runTest(CelOptions options, String expression, boolean validateParseOutput) { + runTest(options, MACROS, expression, validateParseOutput); + } + + private void runTest( + CelOptions options, + Map macros, + String expression, + boolean validateParseOutput) { + testOutput().println("I: " + expression.replace("\t", "ยป")); + testOutput().println("=====>"); + + CelSource source = CelSource.newBuilder(expression).setDescription("").build(); + CelValidationResult parseResult = PrattParser.parse(source, options, macros); + + try { + CelProtoAbstractSyntaxTree protoAst = + CelProtoAbstractSyntaxTree.fromCelAst(parseResult.getAst()); + ParsedExpr parsedExpr = protoAst.toParsedExpr(); + if (validateParseOutput) { + testOutput() + .println( + "P: " + + CelDebug.toAdornedDebugString(parsedExpr.getExpr(), new CelExprKindAndIdAdorner())); + String locationOutput = + CelDebug.toAdornedDebugString( + parsedExpr.getExpr(), new CelLocationAdorner(parsedExpr.getSourceInfo())); + if (!locationOutput.isEmpty()) { + testOutput().println("L: " + locationOutput); + } + } + + String macroOutput = + CelExprKindAndIdAdorner.convertMacroCallsToString(parsedExpr.getSourceInfo()); + if (!macroOutput.isEmpty()) { + testOutput().println("M: " + macroOutput); + } + } catch (CelValidationException e) { + testOutput().println("E: " + e.getMessage()); + } + + testOutput().println(); + } +} diff --git a/parser/src/test/resources/pratt_parser_core_syntax.baseline b/parser/src/test/resources/pratt_parser_core_syntax.baseline new file mode 100644 index 000000000..e54f9429b --- /dev/null +++ b/parser/src/test/resources/pratt_parser_core_syntax.baseline @@ -0,0 +1,1158 @@ +I: a +=====> +P: a^#1:Expr.Ident# +L: a^#1[1,0]# + +I: foo +=====> +P: foo^#1:Expr.Ident# +L: foo^#1[1,0]# + +I: (a) +=====> +P: a^#1:Expr.Ident# +L: a^#1[1,1]# + +I: ((a)) +=====> +P: a^#1:Expr.Ident# +L: a^#1[1,2]# + +I: (((1 + 2))) * 3 +=====> +P: _*_( + _+_( + 1^#1:int64#, + 2^#3:int64# + )^#2:Expr.Call#, + 3^#5:int64# +)^#4:Expr.Call# +L: _*_( + _+_( + 1^#1[1,3]#, + 2^#3[1,7]# + )^#2[1,5]#, + 3^#5[1,14]# +)^#4[1,12]# + +I: [] +=====> +P: []^#1:Expr.CreateList# +L: []^#1[1,0]# + +I: [a] +=====> +P: [ + a^#2:Expr.Ident# +]^#1:Expr.CreateList# +L: [ + a^#2[1,1]# +]^#1[1,0]# + +I: [a, b, c] +=====> +P: [ + a^#2:Expr.Ident#, + b^#3:Expr.Ident#, + c^#4:Expr.Ident# +]^#1:Expr.CreateList# +L: [ + a^#2[1,1]#, + b^#3[1,4]#, + c^#4[1,7]# +]^#1[1,0]# + +I: [1, 2, 3] +=====> +P: [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# +]^#1:Expr.CreateList# +L: [ + 1^#2[1,1]#, + 2^#3[1,4]#, + 3^#4[1,7]# +]^#1[1,0]# + +I: [3, 4, 5] +=====> +P: [ + 3^#2:int64#, + 4^#3:int64#, + 5^#4:int64# +]^#1:Expr.CreateList# +L: [ + 3^#2[1,1]#, + 4^#3[1,4]#, + 5^#4[1,7]# +]^#1[1,0]# + +I: [3, 4, 5,] +=====> +P: [ + 3^#2:int64#, + 4^#3:int64#, + 5^#4:int64# +]^#1:Expr.CreateList# +L: [ + 3^#2[1,1]#, + 4^#3[1,4]#, + 5^#4[1,7]# +]^#1[1,0]# + +I: [?a, b] +=====> +P: [ + ?a^#2:Expr.Ident#, + b^#3:Expr.Ident# +]^#1:Expr.CreateList# +L: [ + ?a^#2[1,2]#, + b^#3[1,5]# +]^#1[1,0]# + +I: [?a, ?b] +=====> +P: [ + ?a^#2:Expr.Ident#, + ?b^#3:Expr.Ident# +]^#1:Expr.CreateList# +L: [ + ?a^#2[1,2]#, + ?b^#3[1,6]# +]^#1[1,0]# + +I: [?a[?b]] +=====> +P: [ + ?_[?_]( + a^#2:Expr.Ident#, + b^#4:Expr.Ident# + )^#3:Expr.Call# +]^#1:Expr.CreateList# +L: [ + ?_[?_]( + a^#2[1,2]#, + b^#4[1,5]# + )^#3[1,3]# +]^#1[1,0]# + +I: {} +=====> +P: {}^#1:Expr.CreateStruct# +L: {}^#1[1,0]# + +I: {a:b, c:d} +=====> +P: { + a^#3:Expr.Ident#:b^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#, + c^#6:Expr.Ident#:d^#7:Expr.Ident#^#5:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + a^#3[1,1]#:b^#4[1,3]#^#2[1,2]#, + c^#6[1,6]#:d^#7[1,8]#^#5[1,7]# +}^#1[1,0]# + +I: {foo: 5, bar: "xyz"} +=====> +P: { + foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#, + bar^#6:Expr.Ident#:"xyz"^#7:string#^#5:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + foo^#3[1,1]#:5^#4[1,6]#^#2[1,4]#, + bar^#6[1,9]#:"xyz"^#7[1,14]#^#5[1,12]# +}^#1[1,0]# + +I: {foo: 5, bar: "xyz", } +=====> +P: { + foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#, + bar^#6:Expr.Ident#:"xyz"^#7:string#^#5:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + foo^#3[1,1]#:5^#4[1,6]#^#2[1,4]#, + bar^#6[1,9]#:"xyz"^#7[1,14]#^#5[1,12]# +}^#1[1,0]# + +I: {"a": 1, "b": 2} +=====> +P: { + "a"^#3:string#:1^#4:int64#^#2:Expr.CreateStruct.Entry#, + "b"^#6:string#:2^#7:int64#^#5:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + "a"^#3[1,1]#:1^#4[1,6]#^#2[1,4]#, + "b"^#6[1,9]#:2^#7[1,14]#^#5[1,12]# +}^#1[1,0]# + +I: {1:2u, 2:3u} +=====> +P: { + 1^#3:int64#:2u^#4:uint64#^#2:Expr.CreateStruct.Entry#, + 2^#6:int64#:3u^#7:uint64#^#5:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + 1^#3[1,1]#:2u^#4[1,3]#^#2[1,2]#, + 2^#6[1,7]#:3u^#7[1,9]#^#5[1,8]# +}^#1[1,0]# + +I: {?a: b} +=====> +P: { + ?a^#3:Expr.Ident#:b^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + ?a^#3[1,2]#:b^#4[1,5]#^#2[1,3]# +}^#1[1,0]# + +I: {?'key': value} +=====> +P: { + ?"key"^#3:string#:value^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: { + ?"key"^#3[1,2]#:value^#4[1,9]#^#2[1,7]# +}^#1[1,0]# + +I: foo{ } +=====> +P: foo{}^#1:Expr.CreateStruct# +L: foo{}^#1[1,0]# + +I: foo{ a:b } +=====> +P: foo{ + a:b^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: foo{ + a:b^#3[1,7]#^#2[1,6]# +}^#1[1,0]# + +I: foo{ a:b, c:d } +=====> +P: foo{ + a:b^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry#, + c:d^#5:Expr.Ident#^#4:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: foo{ + a:b^#3[1,7]#^#2[1,6]#, + c:d^#5[1,12]#^#4[1,11]# +}^#1[1,0]# + +I: SomeMessage{foo: 5, bar: "xyz"} +=====> +P: SomeMessage{ + foo:5^#3:int64#^#2:Expr.CreateStruct.Entry#, + bar:"xyz"^#5:string#^#4:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: SomeMessage{ + foo:5^#3[1,17]#^#2[1,15]#, + bar:"xyz"^#5[1,25]#^#4[1,23]# +}^#1[1,0]# + +I: TestAllTypes{single_int32: 1, single_int64: 2} +=====> +P: TestAllTypes{ + single_int32:1^#3:int64#^#2:Expr.CreateStruct.Entry#, + single_int64:2^#5:int64#^#4:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: TestAllTypes{ + single_int32:1^#3[1,27]#^#2[1,25]#, + single_int64:2^#5[1,44]#^#4[1,42]# +}^#1[1,0]# + +I: MyType{foo: 1, bar: 'baz'} +=====> +P: MyType{ + foo:1^#3:int64#^#2:Expr.CreateStruct.Entry#, + bar:"baz"^#5:string#^#4:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: MyType{ + foo:1^#3[1,12]#^#2[1,10]#, + bar:"baz"^#5[1,20]#^#4[1,18]# +}^#1[1,0]# + +I: Message{`in`: true} +=====> +P: Message{ + in:true^#3:bool#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: Message{ + in:true^#3[1,14]#^#2[1,12]# +}^#1[1,0]# + +I: Msg{?field: value} +=====> +P: Msg{ + ?field:value^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry# +}^#1:Expr.CreateStruct# +L: Msg{ + ?field:value^#3[1,12]#^#2[1,10]# +}^#1[1,0]# + +I: a.b +=====> +P: a^#1:Expr.Ident#.b^#2:Expr.Select# +L: a^#1[1,0]#.b^#2[1,1]# + +I: a.b.c +=====> +P: a^#1:Expr.Ident#.b^#2:Expr.Select#.c^#3:Expr.Select# +L: a^#1[1,0]#.b^#2[1,1]#.c^#3[1,3]# + +I: a.?b +=====> +P: _?._( + a^#1:Expr.Ident#, + "b"^#3:string# +)^#2:Expr.Call# +L: _?._( + a^#1[1,0]#, + "b"^#3[1,3]# +)^#2[1,1]# + +I: a.`b-c` +=====> +P: a^#1:Expr.Ident#.b-c^#2:Expr.Select# +L: a^#1[1,0]#.b-c^#2[1,1]# + +I: a.`b c` +=====> +P: a^#1:Expr.Ident#.b c^#2:Expr.Select# +L: a^#1[1,0]#.b c^#2[1,1]# + +I: a.`b.c` +=====> +P: a^#1:Expr.Ident#.b.c^#2:Expr.Select# +L: a^#1[1,0]#.b.c^#2[1,1]# + +I: a.`in` +=====> +P: a^#1:Expr.Ident#.in^#2:Expr.Select# +L: a^#1[1,0]#.in^#2[1,1]# + +I: a.`/foo` +=====> +P: a^#1:Expr.Ident#./foo^#2:Expr.Select# +L: a^#1[1,0]#./foo^#2[1,1]# + +I: a.`my-var` +=====> +P: a^#1:Expr.Ident#.my-var^#2:Expr.Select# +L: a^#1[1,0]#.my-var^#2[1,1]# + +I: a[b] +=====> +P: _[_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _[_]( + a^#1[1,0]#, + b^#3[1,2]# +)^#2[1,1]# + +I: a[0] +=====> +P: _[_]( + a^#1:Expr.Ident#, + 0^#3:int64# +)^#2:Expr.Call# +L: _[_]( + a^#1[1,0]#, + 0^#3[1,2]# +)^#2[1,1]# + +I: a[3] +=====> +P: _[_]( + a^#1:Expr.Ident#, + 3^#3:int64# +)^#2:Expr.Call# +L: _[_]( + a^#1[1,0]#, + 3^#3[1,2]# +)^#2[1,1]# + +I: [1,3,4][0] +=====> +P: _[_]( + [ + 1^#2:int64#, + 3^#3:int64#, + 4^#4:int64# + ]^#1:Expr.CreateList#, + 0^#6:int64# +)^#5:Expr.Call# +L: _[_]( + [ + 1^#2[1,1]#, + 3^#3[1,3]#, + 4^#4[1,5]# + ]^#1[1,0]#, + 0^#6[1,8]# +)^#5[1,7]# + +I: a[?0] +=====> +P: _[?_]( + a^#1:Expr.Ident#, + 0^#3:int64# +)^#2:Expr.Call# +L: _[?_]( + a^#1[1,0]#, + 0^#3[1,3]# +)^#2[1,1]# + +I: a() +=====> +P: a()^#1:Expr.Call# +L: a()^#1[1,1]# + +I: a(b) +=====> +P: a( + b^#2:Expr.Ident# +)^#1:Expr.Call# +L: a( + b^#2[1,2]# +)^#1[1,1]# + +I: a(b, c) +=====> +P: a( + b^#2:Expr.Ident#, + c^#3:Expr.Ident# +)^#1:Expr.Call# +L: a( + b^#2[1,2]#, + c^#3[1,5]# +)^#1[1,1]# + +I: a.b() +=====> +P: a^#1:Expr.Ident#.b()^#2:Expr.Call# +L: a^#1[1,0]#.b()^#2[1,3]# + +I: a.b(c) +=====> +P: a^#1:Expr.Ident#.b( + c^#3:Expr.Ident# +)^#2:Expr.Call# +L: a^#1[1,0]#.b( + c^#3[1,4]# +)^#2[1,3]# + +I: a.b(5) +=====> +P: a^#1:Expr.Ident#.b( + 5^#3:int64# +)^#2:Expr.Call# +L: a^#1[1,0]#.b( + 5^#3[1,4]# +)^#2[1,3]# + +I: a.foo(1, 2) +=====> +P: a^#1:Expr.Ident#.foo( + 1^#3:int64#, + 2^#4:int64# +)^#2:Expr.Call# +L: a^#1[1,0]#.foo( + 1^#3[1,6]#, + 2^#4[1,9]# +)^#2[1,5]# + +I: !a +=====> +P: !_( + a^#2:Expr.Ident# +)^#1:Expr.Call# +L: !_( + a^#2[1,1]# +)^#1[1,0]# + +I: !x +=====> +P: !_( + x^#2:Expr.Ident# +)^#1:Expr.Call# +L: !_( + x^#2[1,1]# +)^#1[1,0]# + +I: ! false +=====> +P: !_( + false^#2:bool# +)^#1:Expr.Call# +L: !_( + false^#2[1,2]# +)^#1[1,0]# + +I: -a +=====> +P: -_( + a^#2:Expr.Ident# +)^#1:Expr.Call# +L: -_( + a^#2[1,1]# +)^#1[1,0]# + +I: x * 2 +=====> +P: _*_( + x^#1:Expr.Ident#, + 2^#3:int64# +)^#2:Expr.Call# +L: _*_( + x^#1[1,0]#, + 2^#3[1,4]# +)^#2[1,2]# + +I: x * 2u +=====> +P: _*_( + x^#1:Expr.Ident#, + 2u^#3:uint64# +)^#2:Expr.Call# +L: _*_( + x^#1[1,0]#, + 2u^#3[1,4]# +)^#2[1,2]# + +I: x * 2.0 +=====> +P: _*_( + x^#1:Expr.Ident#, + 2.0^#3:double# +)^#2:Expr.Call# +L: _*_( + x^#1[1,0]#, + 2.0^#3[1,4]# +)^#2[1,2]# + +I: a * b +=====> +P: _*_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _*_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a / b +=====> +P: _/_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _/_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a % b +=====> +P: _%_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _%_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a + b +=====> +P: _+_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _+_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a - b +=====> +P: _-_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _-_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: 4--4 +=====> +P: _-_( + 4^#1:int64#, + -4^#3:int64# +)^#2:Expr.Call# +L: _-_( + 4^#1[1,0]#, + -4^#3[1,2]# +)^#2[1,1]# + +I: 4--4.1 +=====> +P: _-_( + 4^#1:int64#, + -4.1^#3:double# +)^#2:Expr.Call# +L: _-_( + 4^#1[1,0]#, + -4.1^#3[1,2]# +)^#2[1,1]# + +I: "abc" + "def" +=====> +P: _+_( + "abc"^#1:string#, + "def"^#3:string# +)^#2:Expr.Call# +L: _+_( + "abc"^#1[1,0]#, + "def"^#3[1,8]# +)^#2[1,6]# + +I: b"abc" + B"def" +=====> +P: _+_( + b"abc"^#1:bytes#, + b"def"^#3:bytes# +)^#2:Expr.Call# +L: _+_( + b"abc"^#1[1,0]#, + b"def"^#3[1,9]# +)^#2[1,7]# + +I: [] + [1,2,3,] + [4] +=====> +P: _+_( + _+_( + []^#1:Expr.CreateList#, + [ + 1^#4:int64#, + 2^#5:int64#, + 3^#6:int64# + ]^#3:Expr.CreateList# + )^#2:Expr.Call#, + [ + 4^#9:int64# + ]^#8:Expr.CreateList# +)^#7:Expr.Call# +L: _+_( + _+_( + []^#1[1,0]#, + [ + 1^#4[1,6]#, + 2^#5[1,8]#, + 3^#6[1,10]# + ]^#3[1,5]# + )^#2[1,3]#, + [ + 4^#9[1,17]# + ]^#8[1,16]# +)^#7[1,14]# + +I: 1 + 2 * 3 +=====> +P: _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# +)^#2:Expr.Call# +L: _+_( + 1^#1[1,0]#, + _*_( + 2^#3[1,4]#, + 3^#5[1,8]# + )^#4[1,6]# +)^#2[1,2]# + +I: a == b +=====> +P: _==_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _==_( + a^#1[1,0]#, + b^#3[1,5]# +)^#2[1,2]# + +I: a != b +=====> +P: _!=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _!=_( + a^#1[1,0]#, + b^#3[1,5]# +)^#2[1,2]# + +I: a < b +=====> +P: _<_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _<_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a <= b +=====> +P: _<=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _<=_( + a^#1[1,0]#, + b^#3[1,5]# +)^#2[1,2]# + +I: a > b +=====> +P: _>_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _>_( + a^#1[1,0]#, + b^#3[1,4]# +)^#2[1,2]# + +I: a >= b +=====> +P: _>=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: _>=_( + a^#1[1,0]#, + b^#3[1,5]# +)^#2[1,2]# + +I: a in b +=====> +P: @in( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# +)^#2:Expr.Call# +L: @in( + a^#1[1,0]#, + b^#3[1,5]# +)^#2[1,2]# + +I: "๐Ÿ˜" in ["๐Ÿ˜", "๐Ÿ˜‘", "๐Ÿ˜ฆ"] +=====> +P: @in( + "๐Ÿ˜"^#1:string#, + [ + "๐Ÿ˜"^#4:string#, + "๐Ÿ˜‘"^#5:string#, + "๐Ÿ˜ฆ"^#6:string# + ]^#3:Expr.CreateList# +)^#2:Expr.Call# +L: @in( + "๐Ÿ˜"^#1[1,0]#, + [ + "๐Ÿ˜"^#4[1,8]#, + "๐Ÿ˜‘"^#5[1,13]#, + "๐Ÿ˜ฆ"^#6[1,18]# + ]^#3[1,7]# +)^#2[1,4]# + +I: size(x) == x.size() +=====> +P: _==_( + size( + x^#2:Expr.Ident# + )^#1:Expr.Call#, + x^#4:Expr.Ident#.size()^#5:Expr.Call# +)^#3:Expr.Call# +L: _==_( + size( + x^#2[1,5]# + )^#1[1,4]#, + x^#4[1,11]#.size()^#5[1,17]# +)^#3[1,8]# + +I: x.single_nested_message != null +=====> +P: _!=_( + x^#1:Expr.Ident#.single_nested_message^#2:Expr.Select#, + null^#4:NullValue# +)^#3:Expr.Call# +L: _!=_( + x^#1[1,0]#.single_nested_message^#2[1,1]#, + null^#4[1,27]# +)^#3[1,24]# + +I: a && b +=====> +P: _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# +)^#3:Expr.Call# +L: _&&_( + a^#1[1,0]#, + b^#2[1,5]# +)^#3[1,2]# + +I: a && b && c +=====> +P: _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# +)^#5:Expr.Call# +L: _&&_( + _&&_( + a^#1[1,0]#, + b^#2[1,5]# + )^#3[1,2]#, + c^#4[1,10]# +)^#5[1,7]# + +I: a && b && c && d && e && f && g +=====> +P: _&&_( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _&&_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call#, + _&&_( + _&&_( + e^#8:Expr.Ident#, + f^#10:Expr.Ident# + )^#11:Expr.Call#, + g^#12:Expr.Ident# + )^#13:Expr.Call# +)^#9:Expr.Call# +L: _&&_( + _&&_( + _&&_( + a^#1[1,0]#, + b^#2[1,5]# + )^#3[1,2]#, + _&&_( + c^#4[1,10]#, + d^#6[1,15]# + )^#7[1,12]# + )^#5[1,7]#, + _&&_( + _&&_( + e^#8[1,20]#, + f^#10[1,25]# + )^#11[1,22]#, + g^#12[1,30]# + )^#13[1,27]# +)^#9[1,17]# + +I: a > 5 && a < 10 +=====> +P: _&&_( + _>_( + a^#1:Expr.Ident#, + 5^#3:int64# + )^#2:Expr.Call#, + _<_( + a^#4:Expr.Ident#, + 10^#6:int64# + )^#5:Expr.Call# +)^#7:Expr.Call# +L: _&&_( + _>_( + a^#1[1,0]#, + 5^#3[1,4]# + )^#2[1,2]#, + _<_( + a^#4[1,9]#, + 10^#6[1,13]# + )^#5[1,11]# +)^#7[1,6]# + +I: a || b +=====> +P: _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# +)^#3:Expr.Call# +L: _||_( + a^#1[1,0]#, + b^#2[1,5]# +)^#3[1,2]# + +I: a || b || c || d || e || f +=====> +P: _||_( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _||_( + _||_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call#, + f^#10:Expr.Ident# + )^#11:Expr.Call# +)^#7:Expr.Call# +L: _||_( + _||_( + _||_( + a^#1[1,0]#, + b^#2[1,5]# + )^#3[1,2]#, + c^#4[1,10]# + )^#5[1,7]#, + _||_( + _||_( + d^#6[1,15]#, + e^#8[1,20]# + )^#9[1,17]#, + f^#10[1,25]# + )^#11[1,22]# +)^#7[1,12]# + +I: a < 5 || a > 10 +=====> +P: _||_( + _<_( + a^#1:Expr.Ident#, + 5^#3:int64# + )^#2:Expr.Call#, + _>_( + a^#4:Expr.Ident#, + 10^#6:int64# + )^#5:Expr.Call# +)^#7:Expr.Call# +L: _||_( + _<_( + a^#1[1,0]#, + 5^#3[1,4]# + )^#2[1,2]#, + _>_( + a^#4[1,9]#, + 10^#6[1,13]# + )^#5[1,11]# +)^#7[1,6]# + +I: a && b && c && d || e && f && g && h +=====> +P: _||_( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _&&_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call#, + _&&_( + _&&_( + e^#8:Expr.Ident#, + f^#9:Expr.Ident# + )^#10:Expr.Call#, + _&&_( + g^#11:Expr.Ident#, + h^#13:Expr.Ident# + )^#14:Expr.Call# + )^#12:Expr.Call# +)^#15:Expr.Call# +L: _||_( + _&&_( + _&&_( + a^#1[1,0]#, + b^#2[1,5]# + )^#3[1,2]#, + _&&_( + c^#4[1,10]#, + d^#6[1,15]# + )^#7[1,12]# + )^#5[1,7]#, + _&&_( + _&&_( + e^#8[1,20]#, + f^#9[1,25]# + )^#10[1,22]#, + _&&_( + g^#11[1,30]#, + h^#13[1,35]# + )^#14[1,32]# + )^#12[1,27]# +)^#15[1,17]# + +I: a?b:c +=====> +P: _?_:_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident#, + c^#4:Expr.Ident# +)^#2:Expr.Call# +L: _?_:_( + a^#1[1,0]#, + b^#3[1,2]#, + c^#4[1,4]# +)^#2[1,1]# + +I: cond ? 1 : 2 +=====> +P: _?_:_( + cond^#1:Expr.Ident#, + 1^#3:int64#, + 2^#4:int64# +)^#2:Expr.Call# +L: _?_:_( + cond^#1[1,0]#, + 1^#3[1,7]#, + 2^#4[1,11]# +)^#2[1,5]# + +I: false && !true || false ? 2 : 3 +=====> +P: _?_:_( + _||_( + _&&_( + false^#1:bool#, + !_( + true^#3:bool# + )^#2:Expr.Call# + )^#4:Expr.Call#, + false^#5:bool# + )^#6:Expr.Call#, + 2^#8:int64#, + 3^#9:int64# +)^#7:Expr.Call# +L: _?_:_( + _||_( + _&&_( + false^#1[1,0]#, + !_( + true^#3[1,10]# + )^#2[1,9]# + )^#4[1,6]#, + false^#5[1,18]# + )^#6[1,15]#, + 2^#8[1,26]#, + 3^#9[1,30]# +)^#7[1,24]# + +I: true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : 1 +=====> + +I: !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-x +=====> + +I: 1 + 2 * 3 - 1 / 2 == 6 % 1 +=====> +P: _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# +)^#10:Expr.Call# +L: _==_( + _-_( + _+_( + 1^#1[1,0]#, + _*_( + 2^#3[1,4]#, + 3^#5[1,8]# + )^#4[1,6]# + )^#2[1,2]#, + _/_( + 1^#7[1,12]#, + 2^#9[1,16]# + )^#8[1,14]# + )^#6[1,10]#, + _%_( + 6^#11[1,21]#, + 1^#13[1,25]# + )^#12[1,23]# +)^#10[1,18]# + +I: x["a"].single_int32 == 23 +=====> +P: _==_( + _[_]( + x^#1:Expr.Ident#, + "a"^#3:string# + )^#2:Expr.Call#.single_int32^#4:Expr.Select#, + 23^#6:int64# +)^#5:Expr.Call# +L: _==_( + _[_]( + x^#1[1,0]#, + "a"^#3[1,2]# + )^#2[1,1]#.single_int32^#4[1,6]#, + 23^#6[1,23]# +)^#5[1,20]# + +I: a.?b[?0] && a[?c] +=====> +P: _&&_( + _[?_]( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call#, + 0^#5:int64# + )^#4:Expr.Call#, + _[?_]( + a^#6:Expr.Ident#, + c^#8:Expr.Ident# + )^#7:Expr.Call# +)^#9:Expr.Call# +L: _&&_( + _[?_]( + _?._( + a^#1[1,0]#, + "b"^#3[1,3]# + )^#2[1,1]#, + 0^#5[1,6]# + )^#4[1,4]#, + _[?_]( + a^#6[1,12]#, + c^#8[1,15]# + )^#7[1,13]# +)^#9[1,9]# \ No newline at end of file diff --git a/parser/src/test/resources/pratt_parser_errors.baseline b/parser/src/test/resources/pratt_parser_errors.baseline new file mode 100644 index 000000000..65d052adf --- /dev/null +++ b/parser/src/test/resources/pratt_parser_errors.baseline @@ -0,0 +1,467 @@ +I: *@a | b +=====> +E: ERROR: :1:1: Syntax error: unexpected token + | *@a | b + | ^ +ERROR: :1:2: Syntax error: unexpected character + | *@a | b + | .^ + +I: 1 + $ +=====> +E: ERROR: :1:5: Syntax error: unexpected character + | 1 + $ + | ....^ + +I: รณย ยข +ยปยปรณย 0ย  +ยปยป0"""\""\"""\""\"""\""\"""\""\"""\"\"""\""\"""\""\"""\""\"""\"!\"""\""\"""\""\" +=====> +E: ERROR: :1:1: Syntax error: unexpected character + | รณย ยข + | ๏ผพ +ERROR: :1:2: Syntax error: unexpected character + | รณย ยข + | ๏ผŽ๏ผพ + +I: '\udead' == '\ufffd' +=====> +E: ERROR: :1:1: Invalid unicode code point + | '\udead' == '\ufffd' + | ^ + +I: a | b +=====> +E: ERROR: :1:3: Syntax error: unexpected single '|', expected '||' + | a | b + | ..^ + +I: '3# < 10" '& tru ^^ +=====> +E: ERROR: :1:12: Syntax error: unexpected single '&', expected '&&' + | '3# < 10" '& tru ^^ + | ...........^ + +I: 1 + + +=====> +E: ERROR: :1:5: Syntax error: unexpected token + | 1 + + + | ....^ + +I: ? +=====> +E: ERROR: :1:1: Syntax error: unexpected token + | ? + | ^ + +I: a ? b ((?)) +=====> +E: ERROR: :1:9: Syntax error: unexpected token + | a ? b ((?)) + | ........^ +ERROR: :1:12: Syntax error: expected ':' in conditional expression + | a ? b ((?)) + | ...........^ + +I: -[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 +ยปยป--3-[-1--1--1--1---1-1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1-ร€1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 +ยปยป--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1 +ยปยป--1--0--1--1--1--3-[-1--1--1--1---1--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1--1 +ยปยป--1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 +=====> +E: ERROR: :3:33: Syntax error: unexpected token + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | ................................^ +ERROR: :3:34: Syntax error: expected ']' + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | .................................^ +ERROR: :11:17: Syntax error: unexpected character + | --1--1---1--1-ร€1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 + | ................๏ผพ +ERROR: :34:49: Syntax error: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^ +ERROR: :34:49: Syntax error: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^ + +I: as break const continue else for function if import in let loop package namespace return var void while +=====> +E: ERROR: :1:1: reserved identifier: as + | as break const continue else for function if import in let loop package namespace return var void while + | ^ +ERROR: :1:4: reserved identifier: break + | as break const continue else for function if import in let loop package namespace return var void while + | ...^ +ERROR: :1:10: reserved identifier: const + | as break const continue else for function if import in let loop package namespace return var void while + | .........^ +ERROR: :1:16: reserved identifier: continue + | as break const continue else for function if import in let loop package namespace return var void while + | ...............^ +ERROR: :1:25: reserved identifier: else + | as break const continue else for function if import in let loop package namespace return var void while + | ........................^ +ERROR: :1:30: reserved identifier: for + | as break const continue else for function if import in let loop package namespace return var void while + | .............................^ +ERROR: :1:34: reserved identifier: function + | as break const continue else for function if import in let loop package namespace return var void while + | .................................^ +ERROR: :1:43: reserved identifier: if + | as break const continue else for function if import in let loop package namespace return var void while + | ..........................................^ +ERROR: :1:46: reserved identifier: import + | as break const continue else for function if import in let loop package namespace return var void while + | .............................................^ +ERROR: :1:53: reserved identifier: in + | as break const continue else for function if import in let loop package namespace return var void while + | ....................................................^ +ERROR: :1:56: reserved identifier: let + | as break const continue else for function if import in let loop package namespace return var void while + | .......................................................^ +ERROR: :1:60: reserved identifier: loop + | as break const continue else for function if import in let loop package namespace return var void while + | ...........................................................^ +ERROR: :1:65: reserved identifier: package + | as break const continue else for function if import in let loop package namespace return var void while + | ................................................................^ +ERROR: :1:73: reserved identifier: namespace + | as break const continue else for function if import in let loop package namespace return var void while + | ........................................................................^ +ERROR: :1:83: reserved identifier: return + | as break const continue else for function if import in let loop package namespace return var void while + | ..................................................................................^ +ERROR: :1:90: reserved identifier: var + | as break const continue else for function if import in let loop package namespace return var void while + | .........................................................................................^ +ERROR: :1:94: reserved identifier: void + | as break const continue else for function if import in let loop package namespace return var void while + | .............................................................................................^ +ERROR: :1:99: reserved identifier: while + | as break const continue else for function if import in let loop package namespace return var void while + | ..................................................................................................^ + +I: [1, 2, 3].map(var, var * var) +=====> +E: ERROR: :1:15: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ..............^ +ERROR: :1:20: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ...................^ +ERROR: :1:26: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | .........................^ + +I: 1 + +=====> +E: ERROR: :1:4: Syntax error: mismatched input '' expecting expression + | 1 + + | ...^ + +I: -- +=====> +E: ERROR: :1:3: Syntax error: mismatched input '' expecting expression + | -- + | ..^ + +I: { +=====> +E: ERROR: :1:2: Syntax error: expected '}' + | { + | .^ + +I: TestAllTypes(){} +=====> +E: ERROR: :1:15: Syntax error: unexpected token after expression + | TestAllTypes(){} + | ..............^ + +I: TestAllTypes{}() +=====> +E: ERROR: :1:15: Syntax error: unexpected token after expression + | TestAllTypes{}() + | ..............^ + +I: 1 + 2 +3 + +=====> +E: ERROR: :2:1: Syntax error: unexpected token after expression + | 3 + + | ^ + +I: {"a": 1}."a" +=====> +E: ERROR: :1:10: Syntax error: expected identifier after '.' + | {"a": 1}."a" + | .........^ + +I: self.true == 1 +=====> +E: ERROR: :1:6: Syntax error: expected identifier after '.' + | self.true == 1 + | .....^ + +I: {a} +=====> +E: ERROR: :1:3: Syntax error: expected ':' in map entry + | {a} + | ..^ + +I: {:a} +=====> +E: ERROR: :1:2: Syntax error: unexpected token + | {:a} + | .^ +ERROR: :1:3: Syntax error: expected ':' in map entry + | {:a} + | ..^ + +I: func{{a}} +=====> +E: ERROR: :1:6: Syntax error: expected struct field name + | func{{a}} + | .....^ +ERROR: :1:9: Syntax error: unexpected token after expression + | func{{a}} + | ........^ + +I: msg{:a} +=====> +E: ERROR: :1:5: Syntax error: expected struct field name + | msg{:a} + | ....^ + +I: ind[a{b}] +=====> +E: ERROR: :1:8: Syntax error: expected ':' in struct field + | ind[a{b}] + | .......^ + +I: x{?. +=====> +E: ERROR: :1:4: Syntax error: expected struct field name + | x{?. + | ...^ +ERROR: :1:5: Syntax error: expected '}' + | x{?. + | ....^ + +I: x{. +=====> +E: ERROR: :1:3: Syntax error: expected struct field name + | x{. + | ..^ +ERROR: :1:4: Syntax error: expected '}' + | x{. + | ...^ + +I: 1.all(2, 3) +=====> +E: ERROR: :1:7: The argument must be a simple name + | 1.all(2, 3) + | ......^ + +I: a.?b && a[?b] +=====> +E: ERROR: :1:2: unsupported syntax '.?' + | a.?b && a[?b] + | .^ +ERROR: :1:10: unsupported syntax '?' + | a.?b && a[?b] + | .........^ + +I: [?a, ?b] +=====> +E: ERROR: :1:2: unsupported syntax '?' + | [?a, ?b] + | .^ +ERROR: :1:6: unsupported syntax '?' + | [?a, ?b] + | .....^ + +I: Msg{?field: value} && {?'key': value} +=====> +E: ERROR: :1:5: unsupported syntax '?' + | Msg{?field: value} && {?'key': value} + | ....^ +ERROR: :1:24: unsupported syntax '?' + | Msg{?field: value} && {?'key': value} + | .......................^ + +I: a.`b-c` +=====> +E: ERROR: :1:3: unsupported syntax '`' + | a.`b-c` + | ..^ + +I: a.`b.c` +=====> +E: ERROR: :1:3: unsupported syntax '`' + | a.`b.c` + | ..^ + +I: a.`in` +=====> +E: ERROR: :1:3: unsupported syntax '`' + | a.`in` + | ..^ + +I: a.`/foo` +=====> +E: ERROR: :1:3: unsupported syntax '`' + | a.`/foo` + | ..^ + +I: Message{`in`: true} +=====> +E: ERROR: :1:9: unsupported syntax '`' + | Message{`in`: true} + | ........^ + +I: `b-c` +=====> +E: ERROR: :1:1: unexpected quoted identifier + | `b-c` + | ^ + +I: `b-c`() +=====> +E: ERROR: :1:1: unexpected quoted identifier + | `b-c`() + | ^ + +I: a.`$b` +=====> +E: ERROR: :1:3: unexpected quoted identifier + | a.`$b` + | ..^ + +I: a.`b.c`() +=====> +E: ERROR: :1:3: unexpected quoted identifier + | a.`b.c`() + | ..^ + +I: [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ +ยปยปยป[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['too many']]]]]]]]]]]]]]]]]]]]]]]]]]]] +ยปยปยป]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +=====> +E: ERROR: :1:33: Expression recursion limit exceeded. limit: 32 + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + | ................................^ +ERROR: :1:33: Syntax error: expected ']' + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + | ................................^ + +I: [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +=====> +E: ERROR: :1:33: Expression recursion limit exceeded. limit: 32 + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] + | ................................^ +ERROR: :1:33: Syntax error: expected ']' + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] + | ................................^ + +I: a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H +=====> +E: ERROR: :1:62: Expression recursion limit exceeded. limit: 32 + | a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H + | .............................................................^ + +I: a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] +ยปยป [21][22][23][24][25][26][27][28][29][30][31][32][33] +=====> +E: ERROR: :2:48: Expression recursion limit exceeded. limit: 32 + | [21][22][23][24][25][26][27][28][29][30][31][32][33] + | ...............................................^ + +I: 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +ยปยป+ 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +ยปยป+ 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 +ยปยป+ 31 + 32 + 33 + 34 +=====> +E: ERROR: :4:8: Expression recursion limit exceeded. limit: 32 + | + 31 + 32 + 33 + 34 + | .......^ + +I: a < 1 < 2 < 3 < 4 < 5 < 6 < 7 < 8 < 9 < 10 < 11 +ยปยป < 12 < 13 < 14 < 15 < 16 < 17 < 18 < 19 < 20 < 21 +ยปยปยป < 22 < 23 < 24 < 25 < 26 < 27 < 28 < 29 < 30 < 31 +ยปยปยป < 32 < 33 +=====> +E: ERROR: :3:51: Expression recursion limit exceeded. limit: 32 + | < 22 < 23 < 24 < 25 < 26 < 27 < 28 < 29 < 30 < 31 + | ..................................................^ + +I: y!=y!=y!=y!=y!=y!=y!=y!=y!=-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y +ยปยป!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y +ยปยป!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y +ยปยป!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y +ยปยป!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y +ยปยป!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y +=====> +E: ERROR: :2:55: Expression recursion limit exceeded. limit: 32 + | !=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y + | ......................................................^ + +I: a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != +ยปยปa[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] +=====> +E: ERROR: :11:76: Expression recursion limit exceeded. limit: 32 + | a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != + | ...........................................................................^ + +I: true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : 1 +=====> +E: ERROR: :1:353: Expression recursion limit exceeded. limit: 32 + | true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : true ? 1 : 1 + | ................................................................................................................................................................................................................................................................................................................................................................^ + +I: !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!x +=====> +E: ERROR: :1:31: Expression recursion limit exceeded. limit: 32 + | !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!x + | ..............................^ \ No newline at end of file diff --git a/parser/src/test/resources/pratt_parser_literals.baseline b/parser/src/test/resources/pratt_parser_literals.baseline new file mode 100644 index 000000000..0cd524d07 --- /dev/null +++ b/parser/src/test/resources/pratt_parser_literals.baseline @@ -0,0 +1,310 @@ +I: null +=====> +P: null^#1:NullValue# +L: null^#1[1,0]# + +I: true +=====> +P: true^#1:bool# +L: true^#1[1,0]# + +I: false +=====> +P: false^#1:bool# +L: false^#1[1,0]# + +I: 0 +=====> +P: 0^#1:int64# +L: 0^#1[1,0]# + +I: 42 +=====> +P: 42^#1:int64# +L: 42^#1[1,0]# + +I: 0xF +=====> +P: 15^#1:int64# +L: 15^#1[1,0]# + +I: 0x2A +=====> +P: 42^#1:int64# +L: 42^#1[1,0]# + +I: -1 +=====> +P: -1^#1:int64# +L: -1^#1[1,0]# + +I: -42 +=====> +P: -42^#1:int64# +L: -42^#1[1,0]# + +I: 0xFFFFFFFFFFFFFFFFF +=====> +E: ERROR: :1:1: Syntax error: invalid int literal + | 0xFFFFFFFFFFFFFFFFF + | ^ + +I: 9223372036854775807 +=====> +P: 9223372036854775807^#1:int64# +L: 9223372036854775807^#1[1,0]# + +I: -9223372036854775808 +=====> +P: -9223372036854775808^#1:int64# +L: -9223372036854775808^#1[1,0]# + +I: -(9223372036854775808) +=====> +E: ERROR: :1:3: Syntax error: invalid int literal + | -(9223372036854775808) + | ..^ + +I: 0u +=====> +P: 0u^#1:uint64# +L: 0u^#1[1,0]# + +I: 23u +=====> +P: 23u^#1:uint64# +L: 23u^#1[1,0]# + +I: 0xFu +=====> +P: 15u^#1:uint64# +L: 15u^#1[1,0]# + +I: 0xFFFFFFFFFFFFFFFFFu +=====> +E: ERROR: :1:1: Syntax error: invalid uint literal + | 0xFFFFFFFFFFFFFFFFFu + | ^ + +I: 3.14 +=====> +P: 3.14^#1:double# +L: 3.14^#1[1,0]# + +I: 23.39 +=====> +P: 23.39^#1:double# +L: 23.39^#1[1,0]# + +I: 1.99e90000009 +=====> +E: ERROR: :1:1: Syntax error: invalid double literal + | 1.99e90000009 + | ^ + +I: 'hello' +=====> +P: "hello"^#1:string# +L: "hello"^#1[1,0]# + +I: "A" +=====> +P: "A"^#1:string# +L: "A"^#1[1,0]# + +I: '''hello +world''' +=====> +P: "hello\nworld"^#1:string# +L: "hello\nworld"^#1[1,0]# + +I: "\u2764" +=====> +P: "โค"^#1:string# +L: "โค"^#1[1,0]# + +I: "โค" +=====> +P: "โค"^#1:string# +L: "โค"^#1[1,0]# + +I: "\"" +=====> +P: "\""^#1:string# +L: "\""^#1[1,0]# + +I: "\xC3\XBF" +=====> +P: "รƒยฟ"^#1:string# +L: "รƒยฟ"^#1[1,0]# + +I: "\303\277" +=====> +P: "รƒยฟ"^#1:string# +L: "รƒยฟ"^#1[1,0]# + +I: "hi\u263A \u263Athere" +=====> +P: "hiโ˜บ โ˜บthere"^#1:string# +L: "hiโ˜บ โ˜บthere"^#1[1,0]# + +I: "\U000003A8\?" +=====> +P: "ฮจ?"^#1:string# +L: "ฮจ?"^#1[1,0]# + +I: "\a\b\f\n\r\t\v'\"\\\? Legal escapes" +=====> +P: "\a\b\f\n\r\t\v'\"\? Legal escapes"^#1:string# +L: "\a\b\f\n\r\t\v'\"\? Legal escapes"^#1[1,0]# + +I: """hello +world""" +=====> +P: "hello\nworld"^#1:string# +L: "hello\nworld"^#1[1,0]# + +I: r"""hello +world""" +=====> +P: "hello\nworld"^#1:string# +L: "hello\nworld"^#1[1,0]# + +I: """hello\"""world""" +=====> +P: "hello\"\"\"world"^#1:string# +L: "hello\"\"\"world"^#1[1,0]# + +I: '''hello\'''world''' +=====> +P: "hello'''world"^#1:string# +L: "hello'''world"^#1[1,0]# + +I: "\xFh" +=====> +E: ERROR: :1:1: Invalid hex escape sequence + | "\xFh" + | ^ + +I: "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" +=====> +E: ERROR: :1:1: Illegal escape sequence + | "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" + | ^ + +I: '๐Ÿ˜' in ['๐Ÿ˜', '๐Ÿ˜‘', '๐Ÿ˜ฆ'] +ยปยปยป&& in.๐Ÿ˜ +=====> +E: ERROR: :2:7: Syntax error: unexpected token + | && in.๐Ÿ˜ + | ......^ +ERROR: :2:10: Syntax error: unexpected character + | && in.๐Ÿ˜ + | .........๏ผพ + +I: """hello +world +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | """hello + | ^ + +I: '''hello +world +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | '''hello + | ^ + +I: r"""hello +world +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | r"""hello + | ^ + +I: "hello +world" +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | "hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world" + | ^ + +I: 'hello +world' +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | 'hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world' + | ^ + +I: r"hello +world" +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | r"hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world" + | ^ + +I: `hello +world` +=====> +E: ERROR: :1:1: Syntax error: unterminated quoted identifier + | `hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world` + | ^ + +I: "hello world" +=====> +E: ERROR: :1:1: Syntax error: unterminated string literal + | "hello world" + | ^ +ERROR: :1:8: Syntax error: unexpected token after expression + | "hello world" + | .......^ + +I: b'abc' +=====> +P: b"abc"^#1:bytes# +L: b"abc"^#1[1,0]# + +I: b"abc" +=====> +P: b"abc"^#1:bytes# +L: b"abc"^#1[1,0]# + +I: b"""hello +world +=====> +E: ERROR: :1:1: Syntax error: unterminated bytes literal + | b"""hello + | ^ + +I: b"hello +world" +=====> +E: ERROR: :1:1: Syntax error: unterminated bytes literal + | b"hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world" + | ^ + +I: rb"hello +world" +=====> +E: ERROR: :1:1: Syntax error: unterminated bytes literal + | rb"hello + | ^ +ERROR: :2:1: Syntax error: unexpected token after expression + | world" + | ^ \ No newline at end of file diff --git a/parser/src/test/resources/pratt_parser_macros.baseline b/parser/src/test/resources/pratt_parser_macros.baseline new file mode 100644 index 000000000..dabf57e31 --- /dev/null +++ b/parser/src/test/resources/pratt_parser_macros.baseline @@ -0,0 +1,903 @@ +I: has(m.f) +=====> +P: m^#2:Expr.Ident#.f~test-only~^#4:Expr.Select# +L: m^#2[1,4]#.f~test-only~^#4[1,3]# +M: has( + m^#2:Expr.Ident#.f^#3:Expr.Select# +)^#0:Expr.Call# + +I: has(a.b) +=====> +P: a^#2:Expr.Ident#.b~test-only~^#4:Expr.Select# +L: a^#2[1,4]#.b~test-only~^#4[1,3]# +M: has( + a^#2:Expr.Ident#.b^#3:Expr.Select# +)^#0:Expr.Call# + +I: has(m) +=====> +E: ERROR: :1:4: invalid argument to has() macro + | has(m) + | ...^ + +I: m.all(v, f) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + true^#5:bool#, + // LoopCondition + @not_strictly_false( + @result^#6:Expr.Ident# + )^#7:Expr.Call#, + // LoopStep + _&&_( + @result^#8:Expr.Ident#, + f^#4:Expr.Ident# + )^#9:Expr.Call#, + // Result + @result^#10:Expr.Ident#)^#11:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + true^#5[1,5]#, + // LoopCondition + @not_strictly_false( + @result^#6[1,5]# + )^#7[1,5]#, + // LoopStep + _&&_( + @result^#8[1,5]#, + f^#4[1,9]# + )^#9[1,5]#, + // Result + @result^#10[1,5]#)^#11[1,5]# +M: m^#1:Expr.Ident#.all( + v^#3:Expr.Ident#, + f^#4:Expr.Ident# +)^#0:Expr.Call# + +I: [1, 2].all(x, x > 0) +=====> +P: __comprehension__( + // Variable + x, + // Target + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#, + // Accumulator + @result, + // Init + true^#9:bool#, + // LoopCondition + @not_strictly_false( + @result^#10:Expr.Ident# + )^#11:Expr.Call#, + // LoopStep + _&&_( + @result^#12:Expr.Ident#, + _>_( + x^#6:Expr.Ident#, + 0^#8:int64# + )^#7:Expr.Call# + )^#13:Expr.Call#, + // Result + @result^#14:Expr.Ident#)^#15:Expr.Comprehension# +L: __comprehension__( + // Variable + x, + // Target + [ + 1^#2[1,1]#, + 2^#3[1,4]# + ]^#1[1,0]#, + // Accumulator + @result, + // Init + true^#9[1,10]#, + // LoopCondition + @not_strictly_false( + @result^#10[1,10]# + )^#11[1,10]#, + // LoopStep + _&&_( + @result^#12[1,10]#, + _>_( + x^#6[1,14]#, + 0^#8[1,18]# + )^#7[1,16]# + )^#13[1,10]#, + // Result + @result^#14[1,10]#)^#15[1,10]# +M: [ + 1^#2:int64#, + 2^#3:int64# +]^#1:Expr.CreateList#.all( + x^#5:Expr.Ident#, + _>_( + x^#6:Expr.Ident#, + 0^#8:int64# + )^#7:Expr.Call# +)^#0:Expr.Call# + +I: m.exists(v, f) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + false^#5:bool#, + // LoopCondition + @not_strictly_false( + !_( + @result^#6:Expr.Ident# + )^#7:Expr.Call# + )^#8:Expr.Call#, + // LoopStep + _||_( + @result^#9:Expr.Ident#, + f^#4:Expr.Ident# + )^#10:Expr.Call#, + // Result + @result^#11:Expr.Ident#)^#12:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + false^#5[1,8]#, + // LoopCondition + @not_strictly_false( + !_( + @result^#6[1,8]# + )^#7[1,8]# + )^#8[1,8]#, + // LoopStep + _||_( + @result^#9[1,8]#, + f^#4[1,12]# + )^#10[1,8]#, + // Result + @result^#11[1,8]#)^#12[1,8]# +M: m^#1:Expr.Ident#.exists( + v^#3:Expr.Ident#, + f^#4:Expr.Ident# +)^#0:Expr.Call# + +I: m.existsOne(v, f) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + 0^#5:int64#, + // LoopCondition + true^#6:bool#, + // LoopStep + _?_:_( + f^#4:Expr.Ident#, + _+_( + @result^#7:Expr.Ident#, + 1^#8:int64# + )^#9:Expr.Call#, + @result^#10:Expr.Ident# + )^#11:Expr.Call#, + // Result + _==_( + @result^#12:Expr.Ident#, + 1^#13:int64# + )^#14:Expr.Call#)^#15:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + 0^#5[1,11]#, + // LoopCondition + true^#6[1,11]#, + // LoopStep + _?_:_( + f^#4[1,15]#, + _+_( + @result^#7[1,11]#, + 1^#8[1,11]# + )^#9[1,11]#, + @result^#10[1,11]# + )^#11[1,11]#, + // Result + _==_( + @result^#12[1,11]#, + 1^#13[1,11]# + )^#14[1,11]#)^#15[1,11]# +M: m^#1:Expr.Ident#.existsOne( + v^#3:Expr.Ident#, + f^#4:Expr.Ident# +)^#0:Expr.Call# + +I: [].existsOne(__result__, __result__) +=====> +E: ERROR: :1:14: The iteration variable __result__ overwrites accumulator variable + | [].existsOne(__result__, __result__) + | .............^ + +I: m.map(v, f) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + []^#5:Expr.CreateList#, + // LoopCondition + true^#6:bool#, + // LoopStep + _+_( + @result^#7:Expr.Ident#, + [ + f^#4:Expr.Ident# + ]^#8:Expr.CreateList# + )^#9:Expr.Call#, + // Result + @result^#10:Expr.Ident#)^#11:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + []^#5[1,5]#, + // LoopCondition + true^#6[1,5]#, + // LoopStep + _+_( + @result^#7[1,5]#, + [ + f^#4[1,9]# + ]^#8[1,5]# + )^#9[1,5]#, + // Result + @result^#10[1,5]#)^#11[1,5]# +M: m^#1:Expr.Ident#.map( + v^#3:Expr.Ident#, + f^#4:Expr.Ident# +)^#0:Expr.Call# + +I: m.map(v, p, f) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + []^#6:Expr.CreateList#, + // LoopCondition + true^#7:bool#, + // LoopStep + _?_:_( + p^#4:Expr.Ident#, + _+_( + @result^#8:Expr.Ident#, + [ + f^#5:Expr.Ident# + ]^#9:Expr.CreateList# + )^#10:Expr.Call#, + @result^#11:Expr.Ident# + )^#12:Expr.Call#, + // Result + @result^#13:Expr.Ident#)^#14:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + []^#6[1,5]#, + // LoopCondition + true^#7[1,5]#, + // LoopStep + _?_:_( + p^#4[1,9]#, + _+_( + @result^#8[1,5]#, + [ + f^#5[1,12]# + ]^#9[1,5]# + )^#10[1,5]#, + @result^#11[1,5]# + )^#12[1,5]#, + // Result + @result^#13[1,5]#)^#14[1,5]# +M: m^#1:Expr.Ident#.map( + v^#3:Expr.Ident#, + p^#4:Expr.Ident#, + f^#5:Expr.Ident# +)^#0:Expr.Call# + +I: m.map(__result__, __result__) +=====> +E: ERROR: :1:7: The iteration variable __result__ overwrites accumulator variable + | m.map(__result__, __result__) + | ......^ + +I: m.filter(v, p) +=====> +P: __comprehension__( + // Variable + v, + // Target + m^#1:Expr.Ident#, + // Accumulator + @result, + // Init + []^#5:Expr.CreateList#, + // LoopCondition + true^#6:bool#, + // LoopStep + _?_:_( + p^#4:Expr.Ident#, + _+_( + @result^#7:Expr.Ident#, + [ + v^#3:Expr.Ident# + ]^#8:Expr.CreateList# + )^#9:Expr.Call#, + @result^#10:Expr.Ident# + )^#11:Expr.Call#, + // Result + @result^#12:Expr.Ident#)^#13:Expr.Comprehension# +L: __comprehension__( + // Variable + v, + // Target + m^#1[1,0]#, + // Accumulator + @result, + // Init + []^#5[1,8]#, + // LoopCondition + true^#6[1,8]#, + // LoopStep + _?_:_( + p^#4[1,12]#, + _+_( + @result^#7[1,8]#, + [ + v^#3[1,9]# + ]^#8[1,8]# + )^#9[1,8]#, + @result^#10[1,8]# + )^#11[1,8]#, + // Result + @result^#12[1,8]#)^#13[1,8]# +M: m^#1:Expr.Ident#.filter( + v^#3:Expr.Ident#, + p^#4:Expr.Ident# +)^#0:Expr.Call# + +I: m.filter(__result__, false) +=====> +E: ERROR: :1:10: The iteration variable __result__ overwrites accumulator variable + | m.filter(__result__, false) + | .........^ + +I: m.filter(a.b, false) +=====> +E: ERROR: :1:11: The argument must be a simple name + | m.filter(a.b, false) + | ..........^ + +I: x.filter(y, y.filter(z, z > 0)) +=====> +P: __comprehension__( + // Variable + y, + // Target + x^#1:Expr.Ident#, + // Accumulator + @result, + // Init + []^#19:Expr.CreateList#, + // LoopCondition + true^#20:bool#, + // LoopStep + _?_:_( + __comprehension__( + // Variable + z, + // Target + y^#4:Expr.Ident#, + // Accumulator + @result, + // Init + []^#10:Expr.CreateList#, + // LoopCondition + true^#11:bool#, + // LoopStep + _?_:_( + _>_( + z^#7:Expr.Ident#, + 0^#9:int64# + )^#8:Expr.Call#, + _+_( + @result^#12:Expr.Ident#, + [ + z^#6:Expr.Ident# + ]^#13:Expr.CreateList# + )^#14:Expr.Call#, + @result^#15:Expr.Ident# + )^#16:Expr.Call#, + // Result + @result^#17:Expr.Ident#)^#18:Expr.Comprehension#, + _+_( + @result^#21:Expr.Ident#, + [ + y^#3:Expr.Ident# + ]^#22:Expr.CreateList# + )^#23:Expr.Call#, + @result^#24:Expr.Ident# + )^#25:Expr.Call#, + // Result + @result^#26:Expr.Ident#)^#27:Expr.Comprehension# +L: __comprehension__( + // Variable + y, + // Target + x^#1[1,0]#, + // Accumulator + @result, + // Init + []^#19[1,8]#, + // LoopCondition + true^#20[1,8]#, + // LoopStep + _?_:_( + __comprehension__( + // Variable + z, + // Target + y^#4[1,12]#, + // Accumulator + @result, + // Init + []^#10[1,20]#, + // LoopCondition + true^#11[1,20]#, + // LoopStep + _?_:_( + _>_( + z^#7[1,24]#, + 0^#9[1,28]# + )^#8[1,26]#, + _+_( + @result^#12[1,20]#, + [ + z^#6[1,21]# + ]^#13[1,20]# + )^#14[1,20]#, + @result^#15[1,20]# + )^#16[1,20]#, + // Result + @result^#17[1,20]#)^#18[1,20]#, + _+_( + @result^#21[1,8]#, + [ + y^#3[1,9]# + ]^#22[1,8]# + )^#23[1,8]#, + @result^#24[1,8]# + )^#25[1,8]#, + // Result + @result^#26[1,8]#)^#27[1,8]# +M: x^#1:Expr.Ident#.filter( + y^#3:Expr.Ident#, + ^#18:filter# +)^#0:Expr.Call#, +y^#4:Expr.Ident#.filter( + z^#6:Expr.Ident#, + _>_( + z^#7:Expr.Ident#, + 0^#9:int64# + )^#8:Expr.Call# +)^#0:Expr.Call# + +I: has(a.b).filter(c, c) +=====> +P: __comprehension__( + // Variable + c, + // Target + a^#2:Expr.Ident#.b~test-only~^#4:Expr.Select#, + // Accumulator + @result, + // Init + []^#8:Expr.CreateList#, + // LoopCondition + true^#9:bool#, + // LoopStep + _?_:_( + c^#7:Expr.Ident#, + _+_( + @result^#10:Expr.Ident#, + [ + c^#6:Expr.Ident# + ]^#11:Expr.CreateList# + )^#12:Expr.Call#, + @result^#13:Expr.Ident# + )^#14:Expr.Call#, + // Result + @result^#15:Expr.Ident#)^#16:Expr.Comprehension# +L: __comprehension__( + // Variable + c, + // Target + a^#2[1,4]#.b~test-only~^#4[1,3]#, + // Accumulator + @result, + // Init + []^#8[1,15]#, + // LoopCondition + true^#9[1,15]#, + // LoopStep + _?_:_( + c^#7[1,19]#, + _+_( + @result^#10[1,15]#, + [ + c^#6[1,16]# + ]^#11[1,15]# + )^#12[1,15]#, + @result^#13[1,15]# + )^#14[1,15]#, + // Result + @result^#15[1,15]#)^#16[1,15]# +M: ^#4:has#.filter( + c^#6:Expr.Ident#, + c^#7:Expr.Ident# +)^#0:Expr.Call#, +has( + a^#2:Expr.Ident#.b^#3:Expr.Select# +)^#0:Expr.Call# + +I: x.filter(y, y.exists(z, has(z.a)) && y.exists(z, has(z.b))) +=====> +P: __comprehension__( + // Variable + y, + // Target + x^#1:Expr.Ident#, + // Accumulator + @result, + // Init + []^#35:Expr.CreateList#, + // LoopCondition + true^#36:bool#, + // LoopStep + _?_:_( + _&&_( + __comprehension__( + // Variable + z, + // Target + y^#4:Expr.Ident#, + // Accumulator + @result, + // Init + false^#11:bool#, + // LoopCondition + @not_strictly_false( + !_( + @result^#12:Expr.Ident# + )^#13:Expr.Call# + )^#14:Expr.Call#, + // LoopStep + _||_( + @result^#15:Expr.Ident#, + z^#8:Expr.Ident#.a~test-only~^#10:Expr.Select# + )^#16:Expr.Call#, + // Result + @result^#17:Expr.Ident#)^#18:Expr.Comprehension#, + __comprehension__( + // Variable + z, + // Target + y^#19:Expr.Ident#, + // Accumulator + @result, + // Init + false^#26:bool#, + // LoopCondition + @not_strictly_false( + !_( + @result^#27:Expr.Ident# + )^#28:Expr.Call# + )^#29:Expr.Call#, + // LoopStep + _||_( + @result^#30:Expr.Ident#, + z^#23:Expr.Ident#.b~test-only~^#25:Expr.Select# + )^#31:Expr.Call#, + // Result + @result^#32:Expr.Ident#)^#33:Expr.Comprehension# + )^#34:Expr.Call#, + _+_( + @result^#37:Expr.Ident#, + [ + y^#3:Expr.Ident# + ]^#38:Expr.CreateList# + )^#39:Expr.Call#, + @result^#40:Expr.Ident# + )^#41:Expr.Call#, + // Result + @result^#42:Expr.Ident#)^#43:Expr.Comprehension# +L: __comprehension__( + // Variable + y, + // Target + x^#1[1,0]#, + // Accumulator + @result, + // Init + []^#35[1,8]#, + // LoopCondition + true^#36[1,8]#, + // LoopStep + _?_:_( + _&&_( + __comprehension__( + // Variable + z, + // Target + y^#4[1,12]#, + // Accumulator + @result, + // Init + false^#11[1,20]#, + // LoopCondition + @not_strictly_false( + !_( + @result^#12[1,20]# + )^#13[1,20]# + )^#14[1,20]#, + // LoopStep + _||_( + @result^#15[1,20]#, + z^#8[1,28]#.a~test-only~^#10[1,27]# + )^#16[1,20]#, + // Result + @result^#17[1,20]#)^#18[1,20]#, + __comprehension__( + // Variable + z, + // Target + y^#19[1,37]#, + // Accumulator + @result, + // Init + false^#26[1,45]#, + // LoopCondition + @not_strictly_false( + !_( + @result^#27[1,45]# + )^#28[1,45]# + )^#29[1,45]#, + // LoopStep + _||_( + @result^#30[1,45]#, + z^#23[1,53]#.b~test-only~^#25[1,52]# + )^#31[1,45]#, + // Result + @result^#32[1,45]#)^#33[1,45]# + )^#34[1,34]#, + _+_( + @result^#37[1,8]#, + [ + y^#3[1,9]# + ]^#38[1,8]# + )^#39[1,8]#, + @result^#40[1,8]# + )^#41[1,8]#, + // Result + @result^#42[1,8]#)^#43[1,8]# +M: x^#1:Expr.Ident#.filter( + y^#3:Expr.Ident#, + _&&_( + ^#18:exists#, + ^#33:exists# + )^#34:Expr.Call# +)^#0:Expr.Call#, +y^#19:Expr.Ident#.exists( + z^#21:Expr.Ident#, + ^#25:has# +)^#0:Expr.Call#, +has( + z^#23:Expr.Ident#.b^#24:Expr.Select# +)^#0:Expr.Call#, +y^#4:Expr.Ident#.exists( + z^#6:Expr.Ident#, + ^#10:has# +)^#0:Expr.Call#, +has( + z^#8:Expr.Ident#.a^#9:Expr.Select# +)^#0:Expr.Call# + +I: (has(a.b) || has(c.d)).string() +=====> +P: _||_( + a^#2:Expr.Ident#.b~test-only~^#4:Expr.Select#, + c^#6:Expr.Ident#.d~test-only~^#8:Expr.Select# +)^#9:Expr.Call#.string()^#10:Expr.Call# +L: _||_( + a^#2[1,5]#.b~test-only~^#4[1,4]#, + c^#6[1,17]#.d~test-only~^#8[1,16]# +)^#9[1,10]#.string()^#10[1,29]# +M: has( + c^#6:Expr.Ident#.d^#7:Expr.Select# +)^#0:Expr.Call#, +has( + a^#2:Expr.Ident#.b^#3:Expr.Select# +)^#0:Expr.Call# + +I: has(a.b).asList().exists(c, c) +=====> +P: __comprehension__( + // Variable + c, + // Target + a^#2:Expr.Ident#.b~test-only~^#4:Expr.Select#.asList()^#5:Expr.Call#, + // Accumulator + @result, + // Init + false^#9:bool#, + // LoopCondition + @not_strictly_false( + !_( + @result^#10:Expr.Ident# + )^#11:Expr.Call# + )^#12:Expr.Call#, + // LoopStep + _||_( + @result^#13:Expr.Ident#, + c^#8:Expr.Ident# + )^#14:Expr.Call#, + // Result + @result^#15:Expr.Ident#)^#16:Expr.Comprehension# +L: __comprehension__( + // Variable + c, + // Target + a^#2[1,4]#.b~test-only~^#4[1,3]#.asList()^#5[1,15]#, + // Accumulator + @result, + // Init + false^#9[1,24]#, + // LoopCondition + @not_strictly_false( + !_( + @result^#10[1,24]# + )^#11[1,24]# + )^#12[1,24]#, + // LoopStep + _||_( + @result^#13[1,24]#, + c^#8[1,28]# + )^#14[1,24]#, + // Result + @result^#15[1,24]#)^#16[1,24]# +M: ^#4:has#.asList()^#5:Expr.Call#.exists( + c^#7:Expr.Ident#, + c^#8:Expr.Ident# +)^#0:Expr.Call#, +has( + a^#2:Expr.Ident#.b^#3:Expr.Select# +)^#0:Expr.Call# + +I: [has(a.b), has(c.d)].exists(e, e) +=====> +P: __comprehension__( + // Variable + e, + // Target + [ + a^#3:Expr.Ident#.b~test-only~^#5:Expr.Select#, + c^#7:Expr.Ident#.d~test-only~^#9:Expr.Select# + ]^#1:Expr.CreateList#, + // Accumulator + @result, + // Init + false^#13:bool#, + // LoopCondition + @not_strictly_false( + !_( + @result^#14:Expr.Ident# + )^#15:Expr.Call# + )^#16:Expr.Call#, + // LoopStep + _||_( + @result^#17:Expr.Ident#, + e^#12:Expr.Ident# + )^#18:Expr.Call#, + // Result + @result^#19:Expr.Ident#)^#20:Expr.Comprehension# +L: __comprehension__( + // Variable + e, + // Target + [ + a^#3[1,5]#.b~test-only~^#5[1,4]#, + c^#7[1,15]#.d~test-only~^#9[1,14]# + ]^#1[1,0]#, + // Accumulator + @result, + // Init + false^#13[1,27]#, + // LoopCondition + @not_strictly_false( + !_( + @result^#14[1,27]# + )^#15[1,27]# + )^#16[1,27]#, + // LoopStep + _||_( + @result^#17[1,27]#, + e^#12[1,31]# + )^#18[1,27]#, + // Result + @result^#19[1,27]#)^#20[1,27]# +M: [ + a^#3:Expr.Ident#.b~test-only~^#5:has#, + c^#7:Expr.Ident#.d~test-only~^#9:has# +]^#1:Expr.CreateList#.exists( + e^#11:Expr.Ident#, + e^#12:Expr.Ident# +)^#0:Expr.Call#, +has( + c^#7:Expr.Ident#.d^#8:Expr.Select# +)^#0:Expr.Call#, +has( + a^#3:Expr.Ident#.b^#4:Expr.Select# +)^#0:Expr.Call# + +I: noop_macro(123) +=====> +P: noop_macro( + 123^#2:int64# +)^#1:Expr.Call# +L: noop_macro( + 123^#2[1,11]# +)^#1[1,10]# \ No newline at end of file diff --git a/testing/src/main/java/dev/cel/testing/BUILD.bazel b/testing/src/main/java/dev/cel/testing/BUILD.bazel index 69765b549..ec8a9841b 100644 --- a/testing/src/main/java/dev/cel/testing/BUILD.bazel +++ b/testing/src/main/java/dev/cel/testing/BUILD.bazel @@ -33,10 +33,14 @@ java_library( srcs = [ "CelAdorner.java", "CelDebug.java", + "CelExprKindAndIdAdorner.java", + "CelLocationAdorner.java", ], deps = [ + "//common:source_location", "@cel_spec//proto/cel/expr:syntax_java_proto", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", ], ) diff --git a/testing/src/main/java/dev/cel/testing/CelExprKindAndIdAdorner.java b/testing/src/main/java/dev/cel/testing/CelExprKindAndIdAdorner.java new file mode 100644 index 000000000..e7992dc6b --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/CelExprKindAndIdAdorner.java @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Collections.reverseOrder; +import static java.util.Map.Entry.comparingByKey; +import static java.util.stream.Collectors.joining; + +import dev.cel.expr.Constant; +import dev.cel.expr.Expr; +import dev.cel.expr.Expr.CreateStruct.EntryOrBuilder; +import dev.cel.expr.ExprOrBuilder; +import dev.cel.expr.SourceInfo; +import com.google.common.base.Ascii; +import com.google.common.base.Joiner; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.EnumDescriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.OneofDescriptor; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * An implementation of {@link CelAdorner} that decorates expressions with their ID and expression + * kind (or macro call name if applicable). + */ +public final class CelExprKindAndIdAdorner implements CelAdorner { + + private static final Joiner JOINER = Joiner.on('.'); + + private final SourceInfo sourceInfo; + + public CelExprKindAndIdAdorner() { + this(SourceInfo.getDefaultInstance()); + } + + public CelExprKindAndIdAdorner(SourceInfo sourceInfo) { + this.sourceInfo = checkNotNull(sourceInfo); + } + + public static CelExprKindAndIdAdorner newInstance() { + return new CelExprKindAndIdAdorner(); + } + + public static CelExprKindAndIdAdorner newInstance(SourceInfo sourceInfo) { + return new CelExprKindAndIdAdorner(sourceInfo); + } + + /** + * Formats the macro calls from {@link SourceInfo} to an adorned debug string, sorted in + * ascending order of expression ID. + */ + public static String convertMacroCallsToString(SourceInfo sourceInfo) { + CelExprKindAndIdAdorner macroCallsAdorner = new CelExprKindAndIdAdorner(sourceInfo); + // Sort in ascending order so that nested macro calls are always in the same order for tests + // output debug string. Ascending order keeps the macro calls map in order from outermost/first + // macro to the innermost/last macro for readability. + return sourceInfo.getMacroCallsMap().entrySet().stream() + .sorted(reverseOrder(comparingByKey())) + .map((entry) -> CelDebug.toAdornedDebugString(entry.getValue(), macroCallsAdorner)) + .collect(joining(",\n")); + } + + @Override + public String adorn(ExprOrBuilder expr) { + if (this.sourceInfo.containsMacroCalls(expr.getId())) { + return String.format( + "^#%d:%s#", + expr.getId(), + this.sourceInfo.getMacroCallsOrThrow(expr.getId()).getCallExpr().getFunction()); + } + + if (expr.hasConstExpr()) { + Constant constExpr = expr.getConstExpr(); + Descriptor descriptor = Constant.getDescriptor(); + OneofDescriptor oneof = findOneofByName(descriptor, "constant_kind"); + FieldDescriptor field = constExpr.getOneofFieldDescriptor(oneof); + if (field.getType() == FieldDescriptor.Type.ENUM) { + return String.format("^#%d:%s#", expr.getId(), getContainedName(field.getEnumType())); + } else { + return String.format( + "^#%d:%s#", expr.getId(), Ascii.toLowerCase(field.getType().toString())); + } + } + Descriptor descriptor = Expr.getDescriptor(); + OneofDescriptor oneof = findOneofByName(descriptor, "expr_kind"); + FieldDescriptor field = expr.getOneofFieldDescriptor(oneof); + return String.format("^#%d:%s#", expr.getId(), getContainedName(field.getMessageType())); + } + + @Override + public String adorn(EntryOrBuilder entry) { + return String.format("^#%d:Expr.CreateStruct.Entry#", entry.getId()); + } + + private static OneofDescriptor findOneofByName(Descriptor descriptor, String name) { + for (OneofDescriptor oneof : descriptor.getOneofs()) { + if (oneof.getName().equals(name)) { + return oneof; + } + } + return null; + } + + private static String getContainedName(Descriptor descriptor) { + Deque parts = new ArrayDeque<>(); + parts.addFirst(descriptor.getName()); + Descriptor containing = descriptor.getContainingType(); + while (containing != null) { + parts.addFirst(containing.getName()); + containing = containing.getContainingType(); + } + return JOINER.join(parts); + } + + private static String getContainedName(EnumDescriptor descriptor) { + Deque parts = new ArrayDeque<>(); + parts.addFirst(descriptor.getName()); + Descriptor containing = descriptor.getContainingType(); + while (containing != null) { + parts.addFirst(containing.getName()); + containing = containing.getContainingType(); + } + return JOINER.join(parts); + } +} diff --git a/testing/src/main/java/dev/cel/testing/CelLocationAdorner.java b/testing/src/main/java/dev/cel/testing/CelLocationAdorner.java new file mode 100644 index 000000000..1fcc8b637 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/CelLocationAdorner.java @@ -0,0 +1,81 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing; + +import static com.google.common.base.Preconditions.checkNotNull; + +import dev.cel.expr.Expr.CreateStruct.EntryOrBuilder; +import dev.cel.expr.ExprOrBuilder; +import dev.cel.expr.SourceInfo; +import dev.cel.common.CelSourceLocation; +import java.util.Map; +import java.util.Optional; + +/** + * An implementation of {@link CelAdorner} that decorates expressions with their source location + * (line and column numbers). + */ +public final class CelLocationAdorner implements CelAdorner { + + private final SourceInfo sourceInfo; + + public CelLocationAdorner(SourceInfo sourceInfo) { + this.sourceInfo = checkNotNull(sourceInfo); + } + + public static CelLocationAdorner newInstance(SourceInfo sourceInfo) { + return new CelLocationAdorner(sourceInfo); + } + + @Override + public String adorn(ExprOrBuilder expr) { + return adorn(expr.getId()); + } + + @Override + public String adorn(EntryOrBuilder entry) { + return adorn(entry.getId()); + } + + private String adorn(long exprId) { + return getLocation(exprId) + .map( + location -> + String.format( + "^#%d[%d,%d]#", exprId, location.getLine(), location.getColumn())) + .orElseGet(() -> String.format("^#%d[NO_POS]#", exprId)); + } + + public Optional getLocation(long exprId) { + Map positions = sourceInfo.getPositionsMap(); + Integer position = positions.get(exprId); + if (position == null) { + return Optional.empty(); + } + int line = 1; + for (int index = 0; index < sourceInfo.getLineOffsetsCount(); index++) { + if (sourceInfo.getLineOffsets(index) > position) { + break; + } else { + line++; + } + } + int column = position; + if (line > 1) { + column = position - sourceInfo.getLineOffsets(line - 2); + } + return Optional.of(CelSourceLocation.of(line, column)); + } +}