[Pratt Parser] Optimize call stack depth via in-place AST mutation and lean dispatch

parser/internal/BUILD

@@ -93,6 +93,7 @@ cc_library(
         "//parser:parser_interface",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/base:nullability",
+        "@com_google_absl//absl/cleanup",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",

parser/internal/lexer.h

@@ -140,6 +140,13 @@ class Lexer final {
                                         std::numeric_limits<int32_t>::max()));
   }
 
+  struct Position final {
+    int32_t position = 0;
+    bool at_end = false;
+    bool done = false;
+    LexerError error;
+  };
+
   Lexer(const Lexer&) = delete;
   Lexer(Lexer&&) = delete;
   Lexer& operator=(const Lexer&) = delete;
@@ -158,6 +165,17 @@ class Lexer final {
 
   [[nodiscard]] int32_t GetPosition() const { return position_; }
 
+  [[nodiscard]] Position SavePosition() const {
+    return Position{position_, at_end_, done_, error_};
+  }
+
+  void RestorePosition(const Position& position) {
+    position_ = position.position;
+    at_end_ = position.at_end;
+    done_ = position.done;
+    error_ = position.error;
+  }
+
  private:
   [[nodiscard]] bool Match(char32_t c) const {
     return position_ < content_.size() && content_.at(position_) == c;

parser/internal/lexer_test.cc

@@ -495,5 +495,42 @@ TEST(LexerErrorRecoveryTest, ResumesAfterError) {
   EXPECT_EQ(token.end, 6);
 }
 
+TEST(LexerPositionTest, SaveAndRestorePosition) {
+  ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("foo + bar * 42"));
+  Lexer lexer(*source);
+
+  Token tok1 = lexer.Lex();
+  EXPECT_EQ(tok1.type, TokenType::kIdent);
+
+  Token tok2 = lexer.Lex();
+  EXPECT_EQ(tok2.type, TokenType::kWhitespace);
+
+  // Save position before '+'
+  Lexer::Position saved = lexer.SavePosition();
+
+  Token tok3 = lexer.Lex();
+  EXPECT_EQ(tok3.type, TokenType::kPlus);
+
+  Token tok4 = lexer.Lex();
+  EXPECT_EQ(tok4.type, TokenType::kWhitespace);
+
+  Token tok5 = lexer.Lex();
+  EXPECT_EQ(tok5.type, TokenType::kIdent);
+
+  // Restore position to before '+'
+  lexer.RestorePosition(saved);
+
+  Token tok3_restored = lexer.Lex();
+  EXPECT_EQ(tok3_restored.type, TokenType::kPlus);
+  EXPECT_EQ(tok3_restored.start, tok3.start);
+  EXPECT_EQ(tok3_restored.end, tok3.end);
+
+  Token tok4_restored = lexer.Lex();
+  EXPECT_EQ(tok4_restored.type, TokenType::kWhitespace);
+
+  Token tok5_restored = lexer.Lex();
+  EXPECT_EQ(tok5_restored.type, TokenType::kIdent);
+}
+
 }  // namespace
 }  // namespace cel::parser_internal

parser/internal/pratt_parser_test.cc

@@ -180,6 +180,32 @@ MATCHER_P(AstIs, expected_ast, "") {
   return false;
 }
 
+MATCHER_P(AstEq, expected_expr, "") {
+  KindAndIdAdorner kind_and_id_adorner;
+  cel::ExprPrinter printer(kind_and_id_adorner);
+  ParserOptions options;
+  auto actual_ast = Parse(arg, options);
+  if (!actual_ast.ok()) {
+    *result_listener << "\n  Actual expression failed to parse: "
+                     << actual_ast.status();
+    return false;
+  }
+  std::string actual = Unindent(printer.Print((*actual_ast)->root_expr()));
+  auto expected_ast = Parse(expected_expr, options);
+  if (!expected_ast.ok()) {
+    *result_listener << "\n  Expected expression failed to parse: "
+                     << expected_ast.status();
+    return false;
+  }
+  std::string expected = Unindent(printer.Print((*expected_ast)->root_expr()));
+  if (actual == expected) {
+    return true;
+  }
+  *result_listener << "\n  Actual:   " << actual
+                   << "\n  Expected: " << expected;
+  return false;
+}
+
 TEST_P(PrattParserTest, Parse) {
   const TestCase& test_case = GetParam();
   cel::ParserOptions options;
@@ -1472,6 +1498,44 @@ TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) {
               StatusIs(absl::StatusCode::kCancelled));
 }
 
+TEST(PrattParserRecursionDepthTest, ParseRecursionDepthIgnoreExtraParens) {
+  cel::ParserOptions options;
+  options.max_recursion_depth = 1;
+  EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull()));
+}
+
+TEST(PrattParserRecursionDepthTest, DeeplyNestedParens) {
+  cel::ParserOptions options;
+  options.max_recursion_depth = 1;
+  std::string literal_expr =
+      std::string(1000, '(') + "42" + std::string(1000, ')');
+  EXPECT_THAT(Parse(literal_expr, options), IsOkAndHolds(NotNull()));
+
+  std::string binary_expr =
+      std::string(1000, '(') + "1 + 2" + std::string(1000, ')');
+  EXPECT_THAT(Parse(binary_expr, options), IsOkAndHolds(NotNull()));
+}
+
+TEST(PrattParserRecursionDepthTest, NestedAndGroupingParensCombinations) {
+  EXPECT_THAT("(( (1) + 2 ))", AstEq("1 + 2"));
+  EXPECT_THAT("((1 + 2) * (3 + 4))", AstEq("(1 + 2) * (3 + 4)"));
+  EXPECT_THAT("((((1)) + ((2))))", AstEq("1 + 2"));
+  EXPECT_THAT("(((1 + 2) * 3) + 4)", AstEq("(1 + 2) * 3 + 4"));
+  EXPECT_THAT("f((((1))), (((2))))", AstEq("f(1, 2)"));
+  EXPECT_THAT("[{((1)): ((2))}]", AstEq("[{1: 2}]"));
+  EXPECT_THAT("(((a))).b[0]", AstEq("a.b[0]"));
+}
+
+TEST(PrattParserRecursionDepthTest, MismatchedParensStillReportErrors) {
+  cel::ParserOptions options;
+  EXPECT_THAT(Parse("((((1))", options),
+              StatusIs(absl::StatusCode::kInvalidArgument));
+  EXPECT_THAT(Parse("(((1 + 2]", options),
+              StatusIs(absl::StatusCode::kInvalidArgument));
+  EXPECT_THAT(Parse("(( [ 1 ) ] ))", options),
+              StatusIs(absl::StatusCode::kInvalidArgument));
+}
+
 TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) {
   cel::ParserOptions options;
   options.max_recursion_depth = 2;

parser/internal/pratt_parser_worker.cc

@@ -123,7 +123,7 @@ std::string ParserWorker::GetTokenText(const Token& tok) const {
   return "";
 }
 
-Token ParserWorker::NextSignificantToken() {
+Token ParserWorker::NextSignificantToken(bool report_error) {
   if (is_recovery_limit_exceeded()) {
     return Token{.type = TokenType::kEnd, .start = 0, .end = 0};
   }
@@ -132,7 +132,7 @@ Token ParserWorker::NextSignificantToken() {
     if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) {
       continue;
     }
-    if (tok.type == TokenType::kError) {
+    if (tok.type == TokenType::kError && report_error) {
       ReportError(tok, lexer_.GetError().message);
       if (is_recovery_limit_exceeded()) {
         return Token{.type = TokenType::kEnd, .start = 0, .end = 0};