[3.13] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH...

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 9aff15eb530845812fb6b7da8024068384c5b51b..4a7b02ee05b89595cc3f5571d938ce76f998381a 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -3097,6 +3097,7 @@ async def f():
             f'__{
                 x:d
             }__'""",
+            " a\n\x00",
         ]:
             with self.subTest(case=case):
                 self.assertRaises(tokenize.TokenError, get_tokens, case)

diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
new file mode 100644 (file)
index 0000000..2c27525
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
@@ -0,0 +1,2 @@
+Fixed crash in :func:`tokenize.generate_tokens` in case of
+specific incorrect input. Patch by Mikhail Efimov.

diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index 66a7cbb769aa4653796cba7d1a9e42c8b56520b7..cc89f0b9cc9af05c025abe622cb4c6c4d7c5d045 100644 (file)
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -481,6 +481,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
                     return MAKE_TOKEN(ERRORTOKEN);
                 }
             }
+            else if (c == EOF && PyErr_Occurred()) {
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
             else {
                 break;
             }