bpo-42864: Simplify the tokenizer exceptions after generic SyntaxError (GH-24273)

Automerge-Triggered-By: GH:pablogsal

diff --git a/Parser/pegen.c b/Parser/pegen.c
index 6c279806021057ba34f6e2c6a1c38db10756be9f..0d39030ea6ed180c8f0630461844ddf79898b632 100644 (file)
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1177,14 +1177,9 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         return 0;
     }
 
-
     Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
     Py_ssize_t current_err_line = current_token->lineno;
 
-    // Save the tokenizer state to restore them later in case we found nothing
-    struct tok_state saved_tok;
-    memcpy(&saved_tok, p->tok, sizeof(struct tok_state));
-
     for (;;) {
         const char *start;
         const char *end;
@@ -1206,8 +1201,6 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
         break;
     }
 
-    // Restore the tokenizer state
-    memcpy(p->tok, &saved_tok, sizeof(struct tok_state));
     return 0;
 }
 
@@ -1239,10 +1232,10 @@ _PyPegen_run_parser(Parser *p)
                 RAISE_INDENTATION_ERROR("unexpected unindent");
             }
             else {
-                if (_PyPegen_check_tokenizer_errors(p)) {
-                    return NULL;
-                }
                 RAISE_SYNTAX_ERROR("invalid syntax");
+                // _PyPegen_check_tokenizer_errors will override the existing
+                // generic SyntaxError we just raised if errors are found.
+                _PyPegen_check_tokenizer_errors(p);
             }
         }
         return NULL;