From 08cacb2a14dd2db133448b24bb9c1d7d21bed07d Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 29 Oct 2025 17:33:55 +0300 Subject: [PATCH] [3.13] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (GH-140583) (#140762) --- Lib/test/test_tokenize.py | 1 + .../2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 ++ Parser/lexer/lexer.c | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 9aff15eb5308..4a7b02ee05b8 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3097,6 +3097,7 @@ async def f(): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst new file mode 100644 index 000000000000..2c27525d9f78 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst @@ -0,0 +1,2 @@ +Fixed crash in :func:`tokenize.generate_tokens` in case of +specific incorrect input. Patch by Mikhail Efimov. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 66a7cbb769aa..cc89f0b9cc9a 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -481,6 +481,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; } -- 2.47.3