gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (#134603)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 2d41a5e5ac0697500d3ef119b674f0aa081c8d4c..e6b19fe1812d441d1b261dbf621501c1e38dd207 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1975,6 +1975,10 @@ if 1:
         for case in cases:
             self.check_roundtrip(case)
 
+        self.check_roundtrip(r"t'{ {}}'")
+        self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'")
+        self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'")
+
 
     def test_continuation(self):
         # Balancing continuation

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 8d01fd7bce41b043b2827a89cb2bfd828bbdfbbb..559a7aecbde2d187b698b09ee8339ab106631ad1 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -274,7 +274,7 @@ class Untokenizer:
         toks_append = self.tokens.append
         startline = token[0] in (NEWLINE, NL)
         prevstring = False
-        in_fstring = 0
+        in_fstring_or_tstring = 0
 
         for tok in _itertools.chain([token], iterable):
             toknum, tokval = tok[:2]
@@ -293,10 +293,10 @@ class Untokenizer:
             else:
                 prevstring = False
 
-            if toknum == FSTRING_START:
-                in_fstring += 1
-            elif toknum == FSTRING_END:
-                in_fstring -= 1
+            if toknum in {FSTRING_START, TSTRING_START}:
+                in_fstring_or_tstring += 1
+            elif toknum in {FSTRING_END, TSTRING_END}:
+                in_fstring_or_tstring -= 1
             if toknum == INDENT:
                 indents.append(tokval)
                 continue
@@ -311,8 +311,8 @@ class Untokenizer:
             elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}:
                 tokval = self.escape_brackets(tokval)
 
-            # Insert a space between two consecutive brackets if we are in an f-string
-            if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring:
+            # Insert a space between two consecutive brackets if we are in an f-string or t-string
+            if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring:
                 tokval = ' ' + tokval
 
             # Insert a space between two consecutive f-strings

diff --git a/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
new file mode 100644 (file)
index 0000000..23e1d58
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
@@ -0,0 +1 @@
+Fix tokenize.untokenize() round-trip errors related to t-strings braces escaping