bpo-47117: Don't crash if we fail to decode characters when the tokenizer buffers...

Automerge-Triggered-By: GH:pablogsal

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-26-15-45-57.bpo-47117.60W6GQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-26-15-45-57.bpo-47117.60W6GQ.rst
new file mode 100644 (file)
index 0000000..5098ed8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-26-15-45-57.bpo-47117.60W6GQ.rst
@@ -0,0 +1,2 @@
+Fix a crash if we fail to decode characters in interactive mode if the
+tokenizer buffers are uninitialized. Patch by Pablo Galindo.

diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 0be9df0ae55357e4f11099471ab16570369c6f20..489699679633e9908186a9519ea318809c4ec505 100644 (file)
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -248,7 +248,12 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
     assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin);
 
     char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
-    assert(cur_line != NULL);
+    if (cur_line == NULL) {
+        assert(p->tok->fp_interactive);
+        // We can reach this point if the tokenizer buffers for interactive source have not been
+        // initialized because we failed to decode the original source with the given locale.
+        return PyUnicode_FromStringAndSize("", 0);
+    }
 
     Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
     const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
@@ -311,7 +316,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
         goto error;
     }
 
-    if (p->tok->fp_interactive) {
+    if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
         error_line = get_error_line_from_tokenizer_buffers(p, lineno);
     }
     else if (p->start_rule == Py_file_input) {