[3.12] gh-105718: Fix buffer allocation in tokenizer with readline (GH-105728) (...

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index df9c9db322dc948a9c215717aba3dfce90857def..15f53632cff81411f9e811ddfe4cfe6c76e42148 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2239,6 +2239,16 @@ def"', """\
     FSTRING_START \'f"\'          (1, 0) (1, 2)
     FSTRING_MIDDLE 'abc\\\\\\ndef'  (1, 2) (2, 3)
     FSTRING_END '"'           (2, 3) (2, 4)
+    """)
+
+        self.check_tokenize('''\
+f"{
+a}"''', """\
+    FSTRING_START 'f"'          (1, 0) (1, 2)
+    LBRACE     '{'           (1, 2) (1, 3)
+    NAME       'a'           (2, 0) (2, 1)
+    RBRACE     '}'           (2, 1) (2, 2)
+    FSTRING_END '"'           (2, 2) (2, 3)
     """)
 
         self.check_tokenize(r'Rf"abc\

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 57c98fe5fcaa4085f3825af97116044d96a5aa03..f41dd130af1f7d83c342bf71c5f5cbc301cf41b3 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1106,11 +1106,7 @@ tok_readline_string(struct tok_state* tok) {
     tok->inp += buflen;
     *tok->inp = '\0';
 
-    if (tok->start == NULL) {
-        tok->buf = tok->cur;
-    }
     tok->line_start = tok->cur;
-
     Py_DECREF(line);
     return 1;
 error:

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 16e919a8931edd4f6610c6c137de53eb3ce5b923..cb44845c1d306ece3a5a22d98d34a9ff2af2aa13 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -68,7 +68,7 @@ typedef struct _tokenizer_mode {
 struct tok_state {
     /* Input state; buf <= cur <= inp <= end */
     /* NB an entire line is held in the buffer */
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL */
+    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
     int fp_interactive; /* If the file descriptor is interactive */