Fix lookahead of soft keywords in the PEG parser (GH-20436)

Automerge-Triggered-By: @gvanrossum

diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py
index 72383d5b5a63157ea85f1290e9123a235cd6891d..a5d88501f77ad7c07da0de903ef27575c339ec32 100644 (file)
--- a/Lib/test/test_peg_generator/test_c_parser.py
+++ b/Lib/test/test_peg_generator/test_c_parser.py
@@ -432,3 +432,15 @@ class TestCParser(TempdirManager, unittest.TestCase):
         self.check_input_strings_for_grammar(valid_cases, invalid_cases)
         """
         self.run_test(grammar_source, test_source)
+
+    def test_soft_keywords_lookahead(self) -> None:
+        grammar_source = """
+        start: &"if" "if" expr '+' expr NEWLINE
+        expr: NAME
+        """
+        test_source = """
+        valid_cases = ["if if + if"]
+        invalid_cases = ["if if"]
+        self.check_input_strings_for_grammar(valid_cases, invalid_cases)
+        """
+        self.run_test(grammar_source, test_source)

diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index ee30c2c0688f89026c405b6e5dd632d3fb037ff0..a0285bcb60e95276bc069f902b8c66930fc651cc 100644 (file)
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -718,6 +718,15 @@ _PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
     return (res != NULL) == positive;
 }
 
+int
+_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
+{
+    int mark = p->mark;
+    void *res = func(p, arg);
+    p->mark = mark;
+    return (res != NULL) == positive;
+}
+
 int
 _PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
 {

diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h
index 9507d9955ae3272e3bca44bce18eda94ab87f236..64cf0ec89291358a71a9484b878a862d1d121cf8 100644 (file)
--- a/Parser/pegen/pegen.h
+++ b/Parser/pegen/pegen.h
@@ -119,6 +119,7 @@ int _PyPegen_is_memoized(Parser *p, int type, void *pres);
 
 int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
 int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
+int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*);
 int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
 
 Token *_PyPegen_expect_token(Parser *p, int type);

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index 885ff05858f6790e70f3b146cbab7859542c9214..ce1d6bb7bf35523cd830272515019e7f6acf7e57 100644 (file)
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -58,7 +58,8 @@ class NodeTypes(Enum):
     STRING_TOKEN = 2
     GENERIC_TOKEN = 3
     KEYWORD = 4
-    CUT_OPERATOR = 5
+    SOFT_KEYWORD = 5
+    CUT_OPERATOR = 6
 
 
 BASE_NODETYPES = {
@@ -123,7 +124,7 @@ class CCallMakerVisitor(GrammarVisitor):
             function="_PyPegen_expect_soft_keyword",
             arguments=["p", value],
             return_type="expr_ty",
-            nodetype=NodeTypes.NAME_TOKEN,
+            nodetype=NodeTypes.SOFT_KEYWORD,
             comment=f"soft_keyword='{value}'",
         )
 
@@ -217,6 +218,12 @@ class CCallMakerVisitor(GrammarVisitor):
                 arguments=[positive, call.function, *call.arguments],
                 return_type="int",
             )
+        elif call.nodetype == NodeTypes.SOFT_KEYWORD:
+            return FunctionCall(
+                function=f"_PyPegen_lookahead_with_string",
+                arguments=[positive, call.function, *call.arguments],
+                return_type="int",
+            )
         elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}:
             return FunctionCall(
                 function=f"_PyPegen_lookahead_with_int",