bpo-44201: Avoid side effects of "invalid_*" rules in the REPL (GH-26298) (GH-26313)

When the parser does a second pass to check for errors, these rules can
have some small side-effects as they may advance the parser more than
the point reached in the first pass. This can cause the tokenizer to ask
for extra tokens in interactive mode causing the tokenizer to show the
prompt instead of failing instantly.

To avoid this, add a new mode to the tokenizer that is activated in the
second pass and deactivates asking for new tokens when the interactive
line is finished. As the parsing should have reached the last line in
the first pass, the second pass should not need to ask for more tokens.

(cherry picked from commit bd7476dae337e905e7b1bbf33ddb96cc270fdc84)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>

diff --git a/Include/errcode.h b/Include/errcode.h
index f2671d6c9b30b41543a419b3b68893ad9d9866b8..2e07fc2c963280bf6dd99c69a1919fe5da2692a5 100644 (file)
--- a/Include/errcode.h
+++ b/Include/errcode.h
@@ -28,6 +28,7 @@ extern "C" {
 #define E_DECODE        22      /* Error in decoding into Unicode */
 #define E_LINECONT      25      /* Unexpected characters after a line continuation */
 #define E_BADSINGLE     27      /* Ill-formed single statement input */
+#define E_INTERACT_STOP 28      /* Interactive mode stopped tokenization */
 
 #ifdef __cplusplus
 }

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst
new file mode 100644 (file)
index 0000000..6f61aac
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst
@@ -0,0 +1,3 @@
+Avoid side effects of checking for specialized syntax errors in the REPL
+that was causing it to ask for extra tokens after a syntax error had been
+detected. Patch by Pablo Galindo

diff --git a/Parser/pegen.c b/Parser/pegen.c
index 3c25e4dbae0ffc43be371db978e6f6a76cbd49ca..548a64788dec4b86072d296e10706db7745d3c48 100644 (file)
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1234,6 +1234,9 @@ reset_parser_state(Parser *p)
     }
     p->mark = 0;
     p->call_invalid_rules = 1;
+    // Don't try to get extra tokens in interactive mode when trying to
+    // raise specialized errors in the second pass.
+    p->tok->interactive_underflow = IUNDERFLOW_STOP;
 }
 
 static int

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ad32293d70b7855c5a8ae28e58936381b7ec5348..a86af9bc0620ca8910b2fe6cc639b969eb94c633 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -85,6 +85,7 @@ tok_new(void)
     tok->async_def = 0;
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
+    tok->interactive_underflow = IUNDERFLOW_NORMAL;
 
     return tok;
 }
@@ -845,6 +846,10 @@ tok_underflow_string(struct tok_state *tok) {
 
 static int
 tok_underflow_interactive(struct tok_state *tok) {
+    if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+        tok->done = E_INTERACT_STOP;
+        return 1;
+    }
     char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
     if (newtok != NULL) {
         char *translated = translate_newlines(newtok, 0, tok);
@@ -1399,6 +1404,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         }
     }
 
+    if (tok->done == E_INTERACT_STOP) {
+        return ENDMARKER;
+    }
+
     /* Check for EOF and errors now */
     if (c == EOF) {
         if (tok->level) {

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index aaa31f37962076b52a71ba28dcecebaecea2eaa0..ff563d57fa8b15a8804c84e87503e29d4b1724f9 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -19,6 +19,14 @@ enum decoding_state {
     STATE_NORMAL
 };
 
+enum interactive_underflow_t {
+    /* Normal mode of operation: return a new token when asked in interactie mode */
+    IUNDERFLOW_NORMAL,
+    /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
+     * can be used to prevent the tokenizer to promt the user for new tokens */
+    IUNDERFLOW_STOP,
+};
+
 /* Tokenizer state */
 struct tok_state {
     /* Input state; buf <= cur <= inp <= end */
@@ -74,6 +82,8 @@ struct tok_state {
     int async_def_indent; /* Indentation level of the outermost 'async def'. */
     int async_def_nl;     /* =1 if the outermost 'async def' had at least one
                              NEWLINE token after it. */
+    /* How to proceed when asked for a new token in interactive mode */
+    enum interactive_underflow_t interactive_underflow; 
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *, int);