gh-105194: Fix format specifier escaped characters in f-strings (#105231)

diff --git a/Grammar/python.gram b/Grammar/python.gram
index 9131835f7421bcb2b8342ef79597264cce253ac0..6b2a46aff0dcf098f68d76fda0c2b33aa3f9f0ae 100644 (file)
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -923,7 +923,7 @@ fstring_conversion[ResultTokenWithMetadata*]:
 fstring_full_format_spec[ResultTokenWithMetadata*]:
     | colon=':' spec=fstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
 fstring_format_spec[expr_ty]:
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+    | t=FSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
     | fstring_replacement_field
 fstring[expr_ty]:
     | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 3ba2f943e6e9689147a659efbb6bbbe7c0c27766..031b94d8d58a3918bde95e4ede6dde9455e91e47 100644 (file)
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -764,6 +764,16 @@ x = (
                              """f'{"s"!{"r"}}'""",
                              ])
 
+    def test_custom_format_specifier(self):
+        class CustomFormat:
+            def __format__(self, format_spec):
+                return format_spec
+
+        self.assertEqual(f'{CustomFormat():\n}', '\n')
+        self.assertEqual(f'{CustomFormat():\u2603}', '☃')
+        with self.assertWarns(SyntaxWarning):
+            exec('f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat})
+
     def test_side_effect_order(self):
         class X:
             def __init__(self):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst
new file mode 100644 (file)
index 0000000..adee74f
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst
@@ -0,0 +1,2 @@
+Do not escape with backslashes f-string format specifiers. Patch by Pablo
+Galindo

diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 2411da2c705e788555547d36fb4a4ee6cbb98790..7786124fde82d0385bd1ce8300302ca73b85f855 100644 (file)
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1350,6 +1350,25 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
                             p->arena);
 }
 
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
+    Py_ssize_t bsize;
+    char* bstr;
+    if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
+        return NULL;
+    }
+    PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok);
+    if (str == NULL) {
+        return NULL;
+    }
+    if (_PyArena_AddPyObject(p->arena, str) < 0) {
+        Py_DECREF(str);
+        return NULL;
+    }
+    return _PyAST_Constant(str, NULL, tok->lineno, tok->col_offset,
+                           tok->end_lineno, tok->end_col_offset,
+                           p->arena);
+}
+
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
     char* bstr = PyBytes_AsString(tok->bytes);
     if (bstr == NULL) {

diff --git a/Parser/parser.c b/Parser/parser.c
index 1705ebd456b5ff99b7a9311bad9562e36dcf7ed7..006ee297974a615ba65a3a5753f9a54190229ffb 100644 (file)
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -16323,7 +16323,7 @@ fstring_format_spec_rule(Parser *p)
         )
         {
             D(fprintf(stderr, "%*c+ fstring_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE"));
-            _res = _PyPegen_constant_from_token ( p , t );
+            _res = _PyPegen_decoded_constant_from_token ( p , t );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 p->level--;

diff --git a/Parser/pegen.h b/Parser/pegen.h
index 8800e9f97f5e04821a00cc596e41d1b5123226cb..fe13d10e6b83e3f732b706214101516a72c2789e 100644 (file)
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -328,6 +328,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
                      int lineno, int col_offset, int end_lineno,
                      int end_col_offset, PyArena *arena);
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
 expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);