bpo-44335: Ensure the tokenizer doesn't go into Python with the error set (GH-26608)

(cherry picked from commit bafe0aade5741ab0d13143ee261711fdd65e8a1f)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>

diff --git a/Parser/pegen.c b/Parser/pegen.c
index c69a042f8de12f13e52b0f965cb1d25d22248c53..42a992251da97550746c493975b5b5085a89ae98 100644 (file)
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1251,9 +1251,14 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         return 0;
     }
 
+    PyObject *type, *value, *traceback;
+    PyErr_Fetch(&type, &value, &traceback);
+
     Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
     Py_ssize_t current_err_line = current_token->lineno;
 
+    int ret = 0;
+
     for (;;) {
         const char *start;
         const char *end;
@@ -1262,9 +1267,9 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
                 if (p->tok->level != 0) {
                     int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
                     if (current_err_line > error_lineno) {
-                        PyErr_Clear();
                         raise_unclosed_parentheses_error(p);
-                        return -1;
+                        ret = -1;
+                        goto exit;
                     }
                 }
                 break;
@@ -1276,7 +1281,16 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         break;
     }
 
-    return 0;
+
+exit:
+    if (PyErr_Occurred()) {
+        Py_XDECREF(value);
+        Py_XDECREF(type);
+        Py_XDECREF(traceback);
+    } else {
+        PyErr_Restore(type, value, traceback);
+    }
+    return ret;
 }
 
 void *