gh-125588: Allow to regenerate the parser with Python < 3.12 (#127969)

Signed-off-by: Pablo Galindo <pablogsal@gmail.com>

diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py
index 692eb9ed2417d741140d6049f4e1697040379b3b..a987d30a9d6438b91afbc9507aaf4171870c9e42 100644 (file)
--- a/Tools/peg_generator/pegen/parser.py
+++ b/Tools/peg_generator/pegen/parser.py
@@ -207,7 +207,7 @@ class Parser:
 
     @memoize
     def fstring_start(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_START = getattr(token, "FSTRING_START")
+        FSTRING_START = getattr(token, "FSTRING_START", None)
         if not FSTRING_START:
             return None
         tok = self._tokenizer.peek()
@@ -217,7 +217,7 @@ class Parser:
 
     @memoize
     def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE")
+        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None)
         if not FSTRING_MIDDLE:
             return None
         tok = self._tokenizer.peek()
@@ -227,7 +227,7 @@ class Parser:
 
     @memoize
     def fstring_end(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_END = getattr(token, "FSTRING_END")
+        FSTRING_END = getattr(token, "FSTRING_END", None)
         if not FSTRING_END:
             return None
         tok = self._tokenizer.peek()

diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index b42b12c8aa0dee260cd53e43ee5628baf9897c99..6ce0649aefe7ff1b252d12be5d6b7a017d0429f0 100644 (file)
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -1,3 +1,4 @@
+import sys
 import ast
 import contextlib
 import re
@@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor):
     def __init__(self, rules: Dict[str, Rule], tokens: Set[str]):
         self.rules = rules
         self.tokens = tokens
+        # If python < 3.12 add the virtual fstring tokens
+        if sys.version_info < (3, 12):
+            self.tokens.add("FSTRING_START")
+            self.tokens.add("FSTRING_END")
+            self.tokens.add("FSTRING_MIDDLE")
 
     def visit_NameLeaf(self, node: NameLeaf) -> None:
         if node.value not in self.rules and node.value not in self.tokens: