From: Pablo Galindo <Pablogsal@gmail.com>
Date: Thu, 3 Sep 2020 14:29:55 +0000 (+0100)
Subject: Fix 'gather' rules in the python parser generator (GH-22021)
X-Git-Tag: v3.10.0a1~179
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e55a0e971b4bf9e6473bf0ca6bebdff76c075ef6;p=thirdparty%2FPython%2Fcpython.git

Fix 'gather' rules in the python parser generator (GH-22021)

Currently, empty sequences in gather rules make the conditional for
gather rules fail as empty sequences evaluate as "False". We need to
explicitly check for "None" (the failure condition) to avoid false
negatives.
---

diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 5b4e964d698a..bcfee3f2c5f8 100644
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -74,7 +74,7 @@ class TestPegen(unittest.TestCase):
             "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))"
         )
 
-    def test_repeat_with_separator_rules(self) -> None:
+    def test_gather(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
@@ -85,6 +85,20 @@ class TestPegen(unittest.TestCase):
             "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
         ))
         self.assertEqual(str(rules["thing"]), "thing: NUMBER")
+        parser_class = make_parser(grammar)
+        node = parse_string("42\n", parser_class)
+        assert node == [
+            [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
+            TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
+        ]
+        node = parse_string("1, 2\n", parser_class)
+        assert node == [
+            [
+                [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")],
+                [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")],
+            ],
+            TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"),
+        ]
 
     def test_expr_grammar(self) -> None:
         grammar = """
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 45a75975dbf5..b786de7fee5b 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -217,6 +217,9 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor):
                     else:
                         self.print("and")
                     self.visit(item)
+                    if is_gather:
+                        self.print("is not None")
+
             self.print("):")
             with self.indent():
                 action = node.action