gh-153568: Factor shared prefixes in expression rules

Author: pablogsal

Grammar/python.gram

@@ -708,10 +708,10 @@ type_param_starred_default[expr_ty]: '=' e=star_expression {
 # -----------
 
 expressions[expr_ty]:
-    | a=expression b=(',' c=expression { c })+ [','] {
-        _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) }
-    | a=expression ',' { _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) }
-    | expression
+    | a=expression b=[(',' c=expression { c })+] trailing=[','] {
+        (b ? b : trailing) ? _PyAST_Tuple(
+            b ? CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)) :
+                CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) : a }
 
 expression[expr_ty] (memo):
     | invalid_if_expression
@@ -729,10 +729,10 @@ yield_expr[expr_ty]:
     | 'yield' a=[star_expressions] { _PyAST_Yield(a, EXTRA) }
 
 star_expressions[expr_ty]:
-    | a=star_expression b=(',' c=star_expression { c })+ [','] {
-        _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) }
-    | a=star_expression ',' { _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) }
-    | star_expression
+    | a=star_expression b=[(',' c=star_expression { c })+] trailing=[','] {
+        (b ? b : trailing) ? _PyAST_Tuple(
+            b ? CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)) :
+                CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) : a }
 
 star_expression[expr_ty] (memo):
     | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) }
@@ -761,18 +761,14 @@ named_expression[expr_ty]:
     | expression !':='
 
 disjunction[expr_ty] (memo):
-    | a=conjunction b=('or' c=conjunction { c })+ { _PyAST_BoolOp(
-        Or,
-        CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)),
-        EXTRA) }
-    | conjunction
+    | a=conjunction b=[('or' c=conjunction { c })+] {
+        b ? _PyAST_BoolOp(Or,
+            CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), EXTRA) : a }
 
 conjunction[expr_ty] (memo):
-    | a=inversion b=('and' c=inversion { c })+ { _PyAST_BoolOp(
-        And,
-        CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)),
-        EXTRA) }
-    | inversion
+    | a=inversion b=[('and' c=inversion { c })+] {
+        b ? _PyAST_BoolOp(And,
+            CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), EXTRA) : a }
 
 inversion[expr_ty] (memo):
     | 'not' a=inversion { _PyAST_UnaryOp(Not, a, EXTRA) }
@@ -782,13 +778,10 @@ inversion[expr_ty] (memo):
 # --------------------
 
 comparison[expr_ty]:
-    | a=bitwise_or b=compare_op_bitwise_or_pair+ {
-        _PyAST_Compare(
-            a,
+    | a=bitwise_or b=[compare_op_bitwise_or_pair+] {
+        b ? _PyAST_Compare(a,
             CHECK(asdl_int_seq*, _PyPegen_get_cmpops(p, b)),
-            CHECK(asdl_expr_seq*, _PyPegen_get_exprs(p, b)),
-            EXTRA) }
-    | bitwise_or
+            CHECK(asdl_expr_seq*, _PyPegen_get_exprs(p, b)), EXTRA) : a }
 
 compare_op_bitwise_or_pair[CmpopExprPair*]:
     | eq_bitwise_or
@@ -832,24 +825,24 @@ bitwise_and[expr_ty]:
     | shift_expr
 
 shift_expr[expr_ty]:
-    | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) }
-    | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) }
+    | a=shift_expr op[Token*]=('<<' | '>>') b=sum {
+        _PyAST_BinOp(a, op->type == LEFTSHIFT ? LShift : RShift, b, EXTRA) }
     | sum
 
 # Arithmetic operators
 # --------------------
 
 sum[expr_ty]:
-    | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) }
-    | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) }
+    | a=sum op[Token*]=('+' | '-') b=term {
+        _PyAST_BinOp(a, op->type == PLUS ? Add : Sub, b, EXTRA) }
     | invalid_arithmetic
     | term
 
 term[expr_ty]:
-    | a=term '*' b=factor { _PyAST_BinOp(a, Mult, b, EXTRA) }
-    | a=term '/' b=factor { _PyAST_BinOp(a, Div, b, EXTRA) }
-    | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) }
-    | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) }
+    | a=term op[Token*]=('*' | '/' | '//' | '%') b=factor {
+        _PyAST_BinOp(a, op->type == STAR ? Mult :
+                       op->type == SLASH ? Div :
+                       op->type == DOUBLESLASH ? FloorDiv : Mod, b, EXTRA) }
     | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) }
     | factor
 
@@ -861,8 +854,8 @@ factor[expr_ty] (memo):
     | invalid_factor
 
 power[expr_ty]:
-    | a=await_primary '**' b=factor { _PyAST_BinOp(a, Pow, b, EXTRA) }
-    | await_primary
+    | a=await_primary b=['**' c=factor { c }] {
+        b ? _PyAST_BinOp(a, Pow, b, EXTRA) : a }
 
 # Primary elements
 # ----------------

Lib/test/test_ast/test_ast.py

@@ -696,6 +696,50 @@ def test_issue18374_binop_col_offset(self):
         self.assertEqual(grandchild_binop.end_col_offset, 3)
         self.assertEqual(grandchild_binop.end_lineno, 1)
 
+    def test_mixed_binop_associativity_and_locations(self):
+        cases = [
+            ('<<', '>>', ast.LShift, ast.RShift),
+            ('+', '-', ast.Add, ast.Sub),
+            ('-', '+', ast.Sub, ast.Add),
+            ('*', '/', ast.Mult, ast.Div),
+            ('/', '//', ast.Div, ast.FloorDiv),
+            ('//', '%', ast.FloorDiv, ast.Mod),
+            ('%', '@', ast.Mod, ast.MatMult),
+            ('@', '*', ast.MatMult, ast.Mult),
+        ]
+        for first, second, first_op, second_op in cases:
+            source = f'(a {first} b\n {second} c)'
+            with self.subTest(source=source):
+                node = ast.parse(source, mode='eval').body
+                self.assertIsInstance(node, ast.BinOp)
+                self.assertIsInstance(node.op, second_op)
+                self.assertIsInstance(node.left, ast.BinOp)
+                self.assertIsInstance(node.left.op, first_op)
+                self.assertEqual(ast.get_source_segment(source, node), source[1:-1])
+                self.assertEqual(ast.get_source_segment(source, node.left),
+                                 f'a {first} b')
+
+    def test_expression_tail_locations(self):
+        cases = [
+            ('a', ast.Name),
+            ('a,', ast.Tuple),
+            ('a, b', ast.Tuple),
+            ('a, b,', ast.Tuple),
+            ('a or b or c', ast.BoolOp),
+            ('a and b and c', ast.BoolOp),
+            ('a < b <= c', ast.Compare),
+            ('a ** b ** c', ast.BinOp),
+        ]
+        for expression, node_type in cases:
+            for mode in ('eval', 'exec'):
+                with self.subTest(expression=expression, mode=mode):
+                    source = expression + '  # comment\n'
+                    tree = ast.parse(source, mode=mode)
+                    node = tree.body if mode == 'eval' else tree.body[0].value
+                    self.assertIsInstance(node, node_type)
+                    self.assertEqual(ast.get_source_segment(source, node),
+                                     expression)
+
     def test_issue39579_dotted_name_end_col_offset(self):
         tree = ast.parse('@a.b.c\ndef f(): pass')
         attr_b = tree.body[0].decorator_list[0].value

Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-15-00-00.gh-issue-153568.GpF4tr.rst

@@ -0,0 +1,2 @@
+Speed up parsing expressions by factoring shared prefixes and optional tails
+in the grammar.