From: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
Date: Sat, 6 Jun 2026 02:46:57 +0000 (+0200)
Subject: [3.14] gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashi... 
X-Git-Url: http://git.ipfire.org/gitweb/index.cgi?a=commitdiff_plain;h=d12f86092476dd1ed93f2de78989b1871a606c0a;p=thirdparty%2FPython%2Fcpython.git

[3.14] gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275) (#150997)

gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275)
(cherry picked from commit 262625fa30e5a1b5cf33c9dbce5d2b713093c7be)

Co-authored-by: Grant Herman <grantlouisherman041@gmail.com>
---

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
new file mode 100644
index 000000000000..12fbffcd1706
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst
@@ -0,0 +1 @@
+Fix a crash when a memory allocation fails during tokenizer initialization. A proper :exc:`MemoryError` is now raised instead.
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2de9004fe084..a1f645029021 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void)
     struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
                                             1,
                                             sizeof(struct tok_state));
-    if (tok == NULL)
+    if (tok == NULL) {
+        PyErr_NoMemory();
         return NULL;
+    }
+
     tok->buf = tok->cur = tok->inp = NULL;
     tok->fp_interactive = 0;
     tok->interactive_src_start = NULL;
diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c
index 8c836a3f7258..a11702557a07 100644
--- a/Parser/tokenizer/file_tokenizer.c
+++ b/Parser/tokenizer/file_tokenizer.c
@@ -378,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc,
         return NULL;
     if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
         _PyTokenizer_Free(tok);
+        PyErr_NoMemory();
         return NULL;
     }
     tok->cur = tok->inp = tok->buf;
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index 7bdf6367671f..af0efd37e8c0 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -193,6 +193,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
     char* result = (char *)PyMem_Malloc(len + 1);
     if (!result) {
         tok->done = E_NOMEM;
+        PyErr_NoMemory();
         return NULL;
     }
     memcpy(result, s, len);
@@ -221,6 +222,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf
     buf = PyMem_Malloc(needed_length);
     if (buf == NULL) {
         tok->done = E_NOMEM;
+        PyErr_NoMemory();
         return NULL;
     }
     for (current = buf; *s; s++, current++) {
diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c
index 0f7769aeb8fd..917f7b40cfbb 100644
--- a/Parser/tokenizer/readline_tokenizer.c
+++ b/Parser/tokenizer/readline_tokenizer.c
@@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc,
         return NULL;
     if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
         _PyTokenizer_Free(tok);
+        PyErr_NoMemory();
         return NULL;
     }
     tok->cur = tok->inp = tok->buf;