From: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 29 Oct 2025 13:54:37 +0000 (+0100)
Subject: [3.14] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH... 
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=c45ab05a3ad779ba7316714eacae75d25944cd73;p=thirdparty%2FPython%2Fcpython.git

[3.14] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH-140583) (#140757)

gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH-140583)
(cherry picked from commit 8706167474e9a625e5f6613d3c7ac77a62faff58)

Co-authored-by: Mikhail Efimov <efimov.mikhail@gmail.com>
---

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index d274726eed2e..ca67e3819587 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -3183,6 +3183,7 @@ async def f():
             f'__{
                 x:d
             }__'""",
+            " a\n\x00",
         ]:
             with self.subTest(case=case):
                 self.assertRaises(tokenize.TokenError, get_tokens, case)
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
new file mode 100644
index 000000000000..2c27525d9f78
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst
@@ -0,0 +1,2 @@
+Fixed crash in :func:`tokenize.generate_tokens` in case of
+specific incorrect input. Patch by Mikhail Efimov.
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a69994e9b3d0..7f25afec302c 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -539,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
                     return MAKE_TOKEN(ERRORTOKEN);
                 }
             }
+            else if (c == EOF && PyErr_Occurred()) {
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
             else {
                 break;
             }