gh-133555: Allow regenerating the parser with Python < 3.14 (#133557)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>

diff --git a/Grammar/Tokens b/Grammar/Tokens
index e40a4437afb009defa185302e957f9ac30b2bf62..0547e6ed08f79ae397836c1b426af7760f7199ce 100644 (file)
--- a/Grammar/Tokens
+++ b/Grammar/Tokens
@@ -1,3 +1,8 @@
+# When adding new tokens, remember to update the PEG generator in
+# Tools/peg_generator/pegen/parser_generator.py
+# This will ensure that older versions of Python can generate a Python parser
+# using "python -m pegen python <GRAMMAR FILE>".
+
 ENDMARKER
 NAME
 NUMBER

diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index 6ce0649aefe7ff1b252d12be5d6b7a017d0429f0..52ae743c26b6b8328976d3fdeb81f50aaf8ec1b2 100644 (file)
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -81,6 +81,11 @@ class RuleCheckingVisitor(GrammarVisitor):
             self.tokens.add("FSTRING_START")
             self.tokens.add("FSTRING_END")
             self.tokens.add("FSTRING_MIDDLE")
+        # If python < 3.14 add the virtual tstring tokens
+        if sys.version_info < (3, 14, 0, 'beta', 1):
+            self.tokens.add("TSTRING_START")
+            self.tokens.add("TSTRING_END")
+            self.tokens.add("TSTRING_MIDDLE")
 
     def visit_NameLeaf(self, node: NameLeaf) -> None:
         if node.value not in self.rules and node.value not in self.tokens: