gh-125588: Teach the python PEG generator the new f-string tokens (#125589)

Signed-off-by: Pablo Galindo <pablogsal@gmail.com>

diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 54c9dce2d0c90dc14f3b58b0e7a235d2fa25db94..d86065213453904b077221bd257c7d186d0dec41 100644 (file)
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -506,6 +506,14 @@ class TestPegen(unittest.TestCase):
         val = eval(code)
         self.assertEqual(val, 3.0)
 
+    def test_f_string_in_action(self) -> None:
+        grammar = """
+        start: n=NAME NEWLINE? $ { f"name -> {n.string}" }
+        """
+        parser_class = make_parser(grammar)
+        node = parse_string("a", parser_class)
+        self.assertEqual(node.strip(), "name ->  a")
+
     def test_nullable(self) -> None:
         grammar_source = """
         start: sign NUMBER

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst
new file mode 100644 (file)
index 0000000..1d59a9c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst
@@ -0,0 +1,2 @@
+The Python PEG generator can now use f-strings in the grammar actions. Patch
+by Pablo Galindo

diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py
index bf31fe5326364559ed4dea46962363459b6ed0cf..2e3a607f7209b0475955e2d358e51f2f6ca9bebb 100644 (file)
--- a/Tools/peg_generator/pegen/grammar_parser.py
+++ b/Tools/peg_generator/pegen/grammar_parser.py
@@ -575,7 +575,7 @@ class GeneratedParser(Parser):
 
     @memoize
     def target_atom(self) -> Optional[str]:
-        # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP
+        # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP
         mark = self._mark()
         cut = False
         if (
@@ -625,6 +625,21 @@ class GeneratedParser(Parser):
         ):
             return string . string
         self._reset(mark)
+        if (
+            (fstring_start := self.fstring_start())
+        ):
+            return fstring_start . string
+        self._reset(mark)
+        if (
+            (fstring_middle := self.fstring_middle())
+        ):
+            return fstring_middle . string
+        self._reset(mark)
+        if (
+            (fstring_end := self.fstring_end())
+        ):
+            return fstring_end . string
+        self._reset(mark)
         if (
             (literal := self.expect("?"))
         ):

diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram
index f22c334ca371b27dc8323c1fded9e39cf157c2c6..f484c4781823bc4fa1682df08cfbf63f8f077d96 100644 (file)
--- a/Tools/peg_generator/pegen/metagrammar.gram
+++ b/Tools/peg_generator/pegen/metagrammar.gram
@@ -126,6 +126,9 @@ target_atom[str]:
     | NAME { name.string }
     | NUMBER { number.string }
     | STRING { string.string }
+    | FSTRING_START { fstring_start.string }
+    | FSTRING_MIDDLE { fstring_middle.string }
+    | FSTRING_END { fstring_end.string }
     | "?" { "?" }
     | ":" { ":" }
     | !"}" !"]" OP { op.string }

diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py
index ed0aec9db2443fab45bb3bc366e6c61c60e5b277..692eb9ed2417d741140d6049f4e1697040379b3b 100644 (file)
--- a/Tools/peg_generator/pegen/parser.py
+++ b/Tools/peg_generator/pegen/parser.py
@@ -205,6 +205,36 @@ class Parser:
             return self._tokenizer.getnext()
         return None
 
+    @memoize
+    def fstring_start(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_START = getattr(token, "FSTRING_START")
+        if not FSTRING_START:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_START:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE")
+        if not FSTRING_MIDDLE:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_MIDDLE:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_end(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_END = getattr(token, "FSTRING_END")
+        if not FSTRING_END:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_END:
+            return self._tokenizer.getnext()
+        return None
+
     @memoize
     def op(self) -> Optional[tokenize.TokenInfo]:
         tok = self._tokenizer.peek()

diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 7057135a9061f6345ef6cabc169b33abdb9d8c0b..4bb26480ebc0af8f5ea49361bd4ba36595d565b1 100644 (file)
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -99,7 +99,8 @@ class PythonCallMakerVisitor(GrammarVisitor):
         name = node.value
         if name == "SOFT_KEYWORD":
             return "soft_keyword", "self.soft_keyword()"
-        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"):
+        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT",
+            "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"):
             name = name.lower()
             return name, f"self.{name}()"
         if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):