From: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
Date: Sun, 25 May 2025 16:48:19 +0000 (+0200)
Subject: [3.14] gh-134582: Fix t-strings untokenize() roundtrip removing space between braces... 
X-Git-Tag: v3.14.0b2~17
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=c61fd7c0f2df757fb485f60a8e6a5f8164477566;p=thirdparty%2FPython%2Fcpython.git

[3.14] gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (GH-134603) (#134659)

gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (GH-134603)
(cherry picked from commit 52509cc94b1a18cb325dbfa7e5f830b32759a903)

Co-authored-by: Loïc Simon <loic.pano@gmail.com>
---

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 2d41a5e5ac06..e6b19fe1812d 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1975,6 +1975,10 @@ if 1:
         for case in cases:
             self.check_roundtrip(case)
 
+        self.check_roundtrip(r"t'{ {}}'")
+        self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'")
+        self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'")
+
 
     def test_continuation(self):
         # Balancing continuation
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 8d01fd7bce41..559a7aecbde2 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -274,7 +274,7 @@ class Untokenizer:
         toks_append = self.tokens.append
         startline = token[0] in (NEWLINE, NL)
         prevstring = False
-        in_fstring = 0
+        in_fstring_or_tstring = 0
 
         for tok in _itertools.chain([token], iterable):
             toknum, tokval = tok[:2]
@@ -293,10 +293,10 @@ class Untokenizer:
             else:
                 prevstring = False
 
-            if toknum == FSTRING_START:
-                in_fstring += 1
-            elif toknum == FSTRING_END:
-                in_fstring -= 1
+            if toknum in {FSTRING_START, TSTRING_START}:
+                in_fstring_or_tstring += 1
+            elif toknum in {FSTRING_END, TSTRING_END}:
+                in_fstring_or_tstring -= 1
             if toknum == INDENT:
                 indents.append(tokval)
                 continue
@@ -311,8 +311,8 @@ class Untokenizer:
             elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}:
                 tokval = self.escape_brackets(tokval)
 
-            # Insert a space between two consecutive brackets if we are in an f-string
-            if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring:
+            # Insert a space between two consecutive brackets if we are in an f-string or t-string
+            if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring:
                 tokval = ' ' + tokval
 
             # Insert a space between two consecutive f-strings
diff --git a/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
new file mode 100644
index 000000000000..23e1d5891b68
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
@@ -0,0 +1 @@
+Fix tokenize.untokenize() round-trip errors related to t-strings braces escaping