Fix tokenizer error when raw decoding null bytes (GH-25080)

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index d18fffaf3dbda3eda748dada8a23dca48b26a6cf..ad32293d70b7855c5a8ae28e58936381b7ec5348 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -813,6 +813,9 @@ tok_readline_raw(struct tok_state *tok)
             tok_concatenate_interactive_new_line(tok, line) == -1) {
             return 0;
         }
+        if (*tok->inp == '\0') {
+            return 0;
+        }
         tok->inp = strchr(tok->inp, '\0');
     } while (tok->inp[-1] != '\n');
     return 1;
@@ -963,7 +966,7 @@ tok_underflow_file(struct tok_state *tok) {
         if (tok->lineno > 2) {
             tok->decoding_state = STATE_NORMAL;
         }
-        else if (!check_coding_spec(tok->cur, tok->end - tok->cur,
+        else if (!check_coding_spec(tok->cur, strlen(tok->cur),
                                     tok, fp_setreadl))
         {
             return 0;