gh-103718: Correctly set f-string buffers in all cases (GH-103815)

Turns out we always need to remember/restore fstring buffers in all of
the stack of tokenizer modes, cause they might change to
`TOK_REGULAR_MODE` and have newlines inside the braces (which is when we
need to reallocate the buffer and restore the fstring ones).

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5244ab7d4f7e0272696a58227b172d3b91c3e634..a8649b8547e2567d15797eec6c86b8fc8f71e840 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -371,10 +371,8 @@ remember_fstring_buffers(struct tok_state *tok)
 
     for (index = tok->tok_mode_stack_index; index >= 0; --index) {
         mode = &(tok->tok_mode_stack[index]);
-        if (mode->kind == TOK_FSTRING_MODE) {
-            mode->f_string_start_offset = mode->f_string_start - tok->buf;
-            mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
-        }
+        mode->f_string_start_offset = mode->f_string_start - tok->buf;
+        mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
     }
 }
 
@@ -387,10 +385,8 @@ restore_fstring_buffers(struct tok_state *tok)
 
     for (index = tok->tok_mode_stack_index; index >= 0; --index) {
         mode = &(tok->tok_mode_stack[index]);
-        if (mode->kind == TOK_FSTRING_MODE) {
-            mode->f_string_start = tok->buf + mode->f_string_start_offset;
-            mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
-        }
+        mode->f_string_start = tok->buf + mode->f_string_start_offset;
+        mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
     }
 }
 
@@ -1081,6 +1077,7 @@ tok_underflow_interactive(struct tok_state *tok) {
         restore_fstring_buffers(tok);
     }
     else {
+        remember_fstring_buffers(tok);
         ADVANCE_LINENO();
         PyMem_Free(tok->buf);
         tok->buf = newtok;
@@ -1088,6 +1085,7 @@ tok_underflow_interactive(struct tok_state *tok) {
         tok->line_start = tok->buf;
         tok->inp = strchr(tok->buf, '\0');
         tok->end = tok->inp + 1;
+        restore_fstring_buffers(tok);
     }
     if (tok->done != E_OK) {
         if (tok->prompt != NULL) {