Fix 'gather' rules in the python parser generator (GH-22021)

Currently, empty sequences in gather rules make the conditional for
gather rules fail as empty sequences evaluate as "False". We need to
explicitly check for "None" (the failure condition) to avoid false
negatives.

diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 5b4e964d698adec6486939b41d1cba922beb1737..bcfee3f2c5f8c318067b36f22aaf3c414ecce08b 100644 (file)
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -74,7 +74,7 @@ class TestPegen(unittest.TestCase):
             "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))"
         )
 
-    def test_repeat_with_separator_rules(self) -> None:
+    def test_gather(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
@@ -85,6 +85,20 @@ class TestPegen(unittest.TestCase):
             "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
         ))
         self.assertEqual(str(rules["thing"]), "thing: NUMBER")
+        parser_class = make_parser(grammar)
+        node = parse_string("42\n", parser_class)
+        assert node == [
+            [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
+            TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
+        ]
+        node = parse_string("1, 2\n", parser_class)
+        assert node == [
+            [
+                [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")],
+                [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")],
+            ],
+            TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"),
+        ]
 
     def test_expr_grammar(self) -> None:
         grammar = """

diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 45a75975dbf5e08a49f1c6b2da317ef7e37d38d1..b786de7fee5b43ce9fdf71e3fbf174f6b8ccd37f 100644 (file)
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -217,6 +217,9 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor):
                     else:
                         self.print("and")
                     self.visit(item)
+                    if is_gather:
+                        self.print("is not None")
+
             self.print("):")
             with self.indent():
                 action = node.action