Ensure the str member of the tokenizer is always initialised (GH-29681)

diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 6eeab0a97226fa763e9be3889c45277f25acb6da..694184a03b075ae6d667e5de92fb8adcd89c3b4d 100644 (file)
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -245,7 +245,7 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
      * (multi-line) statement are stored in p->tok->interactive_src_start.
      * If not, we're parsing from a string, which means that the whole source
      * is stored in p->tok->str. */
-    assert(p->tok->fp == NULL || p->tok->fp == stdin);
+    assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin);
 
     char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
     assert(cur_line != NULL);

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 69d2c08b439262fb8277f7fa0495ca15007f13f4..6358cdf654e1854601a84572bb427a96dd8066d6 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -87,7 +87,7 @@ tok_new(void)
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
     tok->interactive_underflow = IUNDERFLOW_NORMAL;
-
+    tok->str = NULL;
     return tok;
 }
 

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 1d1cfd639d9d50f2ef28da684d8f7316165ee64d..0cb665104b2b86f9d57c07dc811a782da7fbab88 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -71,7 +71,7 @@ struct tok_state {
     PyObject *decoding_readline; /* open(...).readline */
     PyObject *decoding_buffer;
     const char* enc;        /* Encoding for the current str. */
-    char* str;
+    char* str;          /* Source string being tokenized (if tokenizing from a string)*/
     char* input;       /* Tokenizer's newline translated copy of the string. */
 
     int type_comments;      /* Whether to look for type comments */