gh-153568: Make FIRST-set dispatch conservative

Author: pablogsal

Lib/test/test_peg_generator/test_c_parser.py

@@ -174,6 +174,100 @@ def test_prefix_respects_cut(self) -> None:
         )
         """)
 
+    def test_first_set_compound_negative_lookahead(self) -> None:
+        self.run_test("""
+        start: choice NEWLINE ENDMARKER
+        choice: !('a' 'b') ('a' | 'c') | 'd'
+        """, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['a', 'c', 'd'], invalid_cases=['a b'],
+        )
+        """)
+
+    def test_first_set_mutual_recursion(self) -> None:
+        self.run_test("""
+        start: a NEWLINE ENDMARKER
+        a: b 'x' | 'a'
+        b: a 'y' | 'b'
+        """, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['a', 'b x', 'a y x', 'b x y x'],
+            invalid_cases=['a y', 'b'],
+        )
+        """)
+
+    def test_first_set_preserves_leading_cut(self) -> None:
+        self.run_test("""
+        start: choice NEWLINE ENDMARKER
+        choice: ~ NAME | NUMBER
+        """, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['name'], invalid_cases=['42'],
+        )
+        """)
+
+    def test_first_set_preserves_forced_parse(self) -> None:
+        self.run_test("""
+        start: choice NEWLINE ENDMARKER
+        choice: &&'+' | NUMBER
+        """, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['+'], invalid_cases=['42'],
+        )
+        """)
+
+    def test_first_set_nullable_prefix_action(self) -> None:
+        self.run_test("""
+        start: choice NEWLINE ENDMARKER
+        choice: guard NAME | NUMBER
+        guard: &NUMBER { RAISE_SYNTAX_ERROR("guard reached") }
+        """, """
+        with self.assertRaisesRegex(SyntaxError, 'guard reached'):
+            parse.parse_string('42', mode=0)
+        """)
+
+    def check_first_set_lookahead_forced_parse(self, predicate: str) -> None:
+        self.run_test(f"""
+        start: choice NEWLINE ENDMARKER
+        choice: {predicate}(NAME &&'+') STRING | NAME
+        """, """
+        with self.assertRaises(SyntaxError):
+            parse.parse_string('name', mode=0)
+        """)
+
+    def test_first_set_positive_lookahead_forced_parse(self) -> None:
+        self.check_first_set_lookahead_forced_parse('&')
+
+    def test_first_set_negative_lookahead_forced_parse(self) -> None:
+        self.check_first_set_lookahead_forced_parse('!')
+
+    def check_first_set_lookahead_action(self, predicate: str) -> None:
+        self.run_test(f"""
+        start: choice NEWLINE ENDMARKER
+        choice: {predicate}guard NAME | NUMBER
+        guard: NUMBER {{ RAISE_SYNTAX_ERROR("guard reached") }}
+        """, """
+        with self.assertRaisesRegex(SyntaxError, 'guard reached'):
+            parse.parse_string('42', mode=0)
+        """)
+
+    def test_first_set_positive_lookahead_action(self) -> None:
+        self.check_first_set_lookahead_action('&')
+
+    def test_first_set_negative_lookahead_action(self) -> None:
+        self.check_first_set_lookahead_action('!')
+
+    def test_first_set_named_soft_keyword(self) -> None:
+        self.run_test('''
+        start: choice NEWLINE ENDMARKER
+        choice: SOFT_KEYWORD ':' | NUMBER
+        spelling: "soft"
+        ''', """
+        self.check_input_strings_for_grammar(
+            valid_cases=['soft :', '42'], invalid_cases=['other :'],
+        )
+        """)
+
     def test_c_parser(self) -> None:
         grammar_source = """
         start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }

Lib/test/test_peg_generator/test_first_sets.py

@@ -1,7 +1,6 @@
 import unittest
 
 from test import test_tools
-from typing import Dict, Set
 
 test_tools.skip_if_missing("peg_generator")
 with test_tools.imports_under_tool("peg_generator"):
@@ -12,7 +11,7 @@
 
 
 class TestFirstSets(unittest.TestCase):
-    def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]:
+    def calculate_first_sets(self, grammar_source: str) -> dict[str, set[str | None]]:
         grammar: Grammar = parse_string(grammar_source, GrammarParser)
         return FirstSetCalculator(grammar.rules).calculate()
 
@@ -164,8 +163,8 @@ def test_positive_lookahead(self) -> None:
         self.assertEqual(
             self.calculate_first_sets(grammar),
             {
-                "expr": {"'a'"},
-                "start": {"'a'"},
+                "expr": {None, "'a'", "'b'", "'c'"},
+                "start": {None, "'a'", "'b'", "'c'"},
                 "opt": {"'b'", "'c'", "'a'"},
             },
         )
@@ -180,8 +179,8 @@ def test_negative_lookahead(self) -> None:
             self.calculate_first_sets(grammar),
             {
                 "opt": {"'b'", "'a'", "'c'"},
-                "expr": {"'b'", "'c'"},
-                "start": {"'b'", "'c'"},
+                "expr": {None, "'a'", "'b'", "'c'"},
+                "start": {None, "'a'", "'b'", "'c'"},
             },
         )
 
@@ -226,21 +225,20 @@ def test_mutual_left_recursion(self) -> None:
             self.calculate_first_sets(grammar),
             {
                 "foo": {"'D'", "'B'"},
-                "bar": {"'D'"},
+                "bar": {"'B'", "'D'"},
                 "start": {"'D'", "'B'"},
             },
         )
 
     def test_nasty_left_recursion(self) -> None:
-        # TODO: Validate this
         grammar = """
         start: target '='
         target: maybe '+' | NAME
         maybe: maybe '-' | target
         """
         self.assertEqual(
             self.calculate_first_sets(grammar),
-            {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
+            {"maybe": {"NAME"}, "target": {"NAME"}, "start": {"NAME"}},
         )
 
     def test_nullable_rule(self) -> None:
@@ -284,3 +282,43 @@ def test_multiple_nullable_rules(self) -> None:
                 "another": {"'/'"},
             },
         )
+
+    def test_compound_negative_lookahead(self) -> None:
+        sets = self.calculate_first_sets("""
+        start: choice NEWLINE ENDMARKER
+        choice: !('a' 'b') ('a' | 'c') | 'd'
+        """)
+        self.assertEqual(sets['choice'], {None, "'a'", "'c'", "'d'"})
+
+    def test_nullable_recursive_rules(self) -> None:
+        sets = self.calculate_first_sets("""
+        start: a NUMBER ENDMARKER
+        a: b | NAME
+        b: a | ['+']
+        """)
+        self.assertEqual(sets['a'], {'', 'NAME', "'+'"})
+        self.assertEqual(sets['b'], {'', 'NAME', "'+'"})
+        self.assertEqual(sets['start'], {'NUMBER', 'NAME', "'+'"})
+
+    def test_control_flow_before_first_token(self) -> None:
+        sets = self.calculate_first_sets("""
+        start: NAME ENDMARKER
+        cut: ~ NAME
+        forced: &&'+'
+        guarded: &forced NUMBER
+        action: [NAME] { _PyPegen_dummy_name(p) }
+        """)
+        self.assertEqual(sets['cut'], {None, 'NAME'})
+        self.assertEqual(sets['forced'], {None, "'+'"})
+        self.assertEqual(sets['guarded'], {None, 'NUMBER'})
+        self.assertEqual(sets['action'], {None, '', 'NAME'})
+
+    def test_nullable_repeat_and_gather(self) -> None:
+        sets = self.calculate_first_sets("""
+        start: NAME ENDMARKER
+        optional: [NAME]
+        repeat: optional+ NUMBER
+        gather: ','.optional+ NUMBER
+        """)
+        self.assertEqual(sets['repeat'], {'NAME', 'NUMBER'})
+        self.assertEqual(sets['gather'], {'NAME', "','", 'NUMBER'})