From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Tue, 25 Apr 2023 01:31:21 +0000 (-0600)
Subject: gh-103718: Correctly set f-string buffers in all cases (GH-103815)
X-Git-Tag: v3.12.0b1~429
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=57f8f9a66d4d0b5e590f7746a58136b3b45b1336;p=thirdparty%2FPython%2Fcpython.git

gh-103718: Correctly set f-string buffers in all cases (GH-103815)

Turns out we always need to remember/restore fstring buffers in all of
the stack of tokenizer modes, cause they might change to
`TOK_REGULAR_MODE` and have newlines inside the braces (which is when we
need to reallocate the buffer and restore the fstring ones).
---

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5244ab7d4f7e..a8649b8547e2 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -371,10 +371,8 @@ remember_fstring_buffers(struct tok_state *tok)
 
     for (index = tok->tok_mode_stack_index; index >= 0; --index) {
         mode = &(tok->tok_mode_stack[index]);
-        if (mode->kind == TOK_FSTRING_MODE) {
-            mode->f_string_start_offset = mode->f_string_start - tok->buf;
-            mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
-        }
+        mode->f_string_start_offset = mode->f_string_start - tok->buf;
+        mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
     }
 }
 
@@ -387,10 +385,8 @@ restore_fstring_buffers(struct tok_state *tok)
 
     for (index = tok->tok_mode_stack_index; index >= 0; --index) {
         mode = &(tok->tok_mode_stack[index]);
-        if (mode->kind == TOK_FSTRING_MODE) {
-            mode->f_string_start = tok->buf + mode->f_string_start_offset;
-            mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
-        }
+        mode->f_string_start = tok->buf + mode->f_string_start_offset;
+        mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
     }
 }
 
@@ -1081,6 +1077,7 @@ tok_underflow_interactive(struct tok_state *tok) {
         restore_fstring_buffers(tok);
     }
     else {
+        remember_fstring_buffers(tok);
         ADVANCE_LINENO();
         PyMem_Free(tok->buf);
         tok->buf = newtok;
@@ -1088,6 +1085,7 @@ tok_underflow_interactive(struct tok_state *tok) {
         tok->line_start = tok->buf;
         tok->inp = strchr(tok->buf, '\0');
         tok->end = tok->inp + 1;
+        restore_fstring_buffers(tok);
     }
     if (tok->done != E_OK) {
         if (tok->prompt != NULL) {