From 8706167474e9a625e5f6613d3c7ac77a62faff58 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 29 Oct 2025 16:27:35 +0300 Subject: [PATCH] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (#140583) --- Lib/test/test_tokenize.py | 1 + .../2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 ++ Parser/lexer/lexer.c | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index d274726eed2e..ca67e3819587 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3183,6 +3183,7 @@ async def f(): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst new file mode 100644 index 000000000000..2c27525d9f78 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst @@ -0,0 +1,2 @@ +Fixed crash in :func:`tokenize.generate_tokens` in case of +specific incorrect input. Patch by Mikhail Efimov. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a69994e9b3d0..7f25afec302c 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -539,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; } -- 2.47.3