]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-131876: extract `_hashlib` helpers into a separate directory (#136995)
authorBénédikt Tran <10796600+picnixz@users.noreply.github.com>
Mon, 28 Jul 2025 09:28:48 +0000 (11:28 +0200)
committerGitHub <noreply@github.com>
Mon, 28 Jul 2025 09:28:48 +0000 (11:28 +0200)
The `Modules/hashlib.h` helper file is now removed and split into multiple files:

* `Modules/_hashlib/hashlib_buffer.[ch]` -- Utilities for getting a buffer view and handling buffer inputs.
* `Modules/_hashlib/hashlib_fetch.h` -- Utilities used when fetching a message digest from a digest-like identifier.
  Currently, this file only contains common error messages as the fetching API is not yet implemented.
* `Modules/_hashlib/hashlib_mutex.h` -- Utilities for managing the lock on cryptographic hash objects.

20 files changed:
Makefile.pre.in
Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst [new file with mode: 0644]
Modules/_hashlib/hashlib_buffer.c [new file with mode: 0644]
Modules/_hashlib/hashlib_buffer.h [new file with mode: 0644]
Modules/_hashlib/hashlib_fetch.h [new file with mode: 0644]
Modules/_hashlib/hashlib_mutex.h [new file with mode: 0644]
Modules/_hashopenssl.c
Modules/blake2module.c
Modules/hashlib.h [deleted file]
Modules/hmacmodule.c
Modules/md5module.c
Modules/sha1module.c
Modules/sha2module.c
Modules/sha3module.c
PCbuild/_hashlib.vcxproj
PCbuild/_hashlib.vcxproj.filters
PCbuild/pythoncore.vcxproj
PCbuild/pythoncore.vcxproj.filters
configure
configure.ac

index 0c070131cda20005c812bbcb8507765a84415328..b7b16ef4cb9d19040d04a99f157d7d0d43e546f3 100644 (file)
@@ -227,6 +227,7 @@ ENSUREPIP=      @ENSUREPIP@
 # Internal static libraries
 LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
 LIBEXPAT_A= Modules/expat/libexpat.a
+LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a
 
 # HACL* build configuration
 LIBHACL_CFLAGS=@LIBHACL_CFLAGS@
@@ -761,6 +762,17 @@ LIBHACL_HMAC_HEADERS= \
                $(LIBHACL_BLAKE2_HEADERS) \
                $(LIBHACL_HEADERS)
 
+##########################################################################
+# Internal library for cryptographic primitives
+
+LIBHASHLIB_INTERNAL_OBJS= \
+               Modules/_hashlib/hashlib_buffer.o
+
+LIBHASHLIB_INTERNAL_HEADERS= \
+               Modules/_hashlib/hashlib_buffer.h \
+               Modules/_hashlib/hashlib_fetch.h \
+               Modules/_hashlib/hashlib_mutex.h
+
 #########################################################################
 # Rules
 
@@ -1511,6 +1523,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
        -rm -f $@
        $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS)
 
+##########################################################################
+# '_hashlib', '_hmac' and HACL*-based modules helpers
+LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED)
+
+Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS)
+       $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c
+
+$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS)
+       -rm -f $@
+       $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS)
+
 ##########################################################################
 # HACL* library build
 #
@@ -3353,21 +3376,21 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h
 MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@
 MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@
 MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@
-MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
+MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@
 MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
 
 # HACL*-based cryptographic primitives
-MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
-MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)
-MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
-MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)
-MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
-MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)
+MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)
 
 MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
diff --git a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst b/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst
new file mode 100644 (file)
index 0000000..304f2c3
--- /dev/null
@@ -0,0 +1,2 @@
+Remove :file:`!Modules/hashlib.h` and move its content into dedicated files
+now located in ``Modules/_hashlib``. Patch by Bénédikt Tran.
diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c
new file mode 100644 (file)
index 0000000..032f93a
--- /dev/null
@@ -0,0 +1,65 @@
+#include "hashlib_buffer.h"
+
+int
+_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string)
+{
+    if (data != NULL && string == NULL) {
+        // called as H(data) or H(data=...)
+        *res = data;
+        return 1;
+    }
+    else if (data == NULL && string != NULL) {
+        // called as H(string=...)
+        if (PyErr_WarnEx(PyExc_DeprecationWarning,
+                         "the 'string' keyword parameter is deprecated since "
+                         "Python 3.15 and slated for removal in Python 3.19; "
+                         "use the 'data' keyword parameter or pass the data "
+                         "to hash as a positional argument instead", 1) < 0)
+        {
+            *res = NULL;
+            return -1;
+        }
+        *res = string;
+        return 1;
+    }
+    else if (data == NULL && string == NULL) {
+        // fast path when no data is given
+        assert(!PyErr_Occurred());
+        *res = NULL;
+        return 0;
+    }
+    else {
+        // called as H(data=..., string)
+        *res = NULL;
+        PyErr_SetString(PyExc_TypeError,
+                        "'data' and 'string' are mutually exclusive "
+                        "and support for 'string' keyword parameter "
+                        "is slated for removal in a future version.");
+        return -1;
+    }
+}
+
+int
+_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view)
+{
+    if (PyUnicode_Check(obj)) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Strings must be encoded before hashing");
+        return -1;
+    }
+    if (!PyObject_CheckBuffer(obj)) {
+        PyErr_SetString(PyExc_TypeError,
+                        "object supporting the buffer API required");
+        return -1;
+    }
+    if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) {
+        return -1;
+    }
+    if (view->ndim > 1) {
+        PyErr_SetString(PyExc_BufferError,
+                        "Buffer must be single dimension");
+        PyBuffer_Release(view);
+        return -1;
+    }
+    return 0;
+}
diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h
new file mode 100644 (file)
index 0000000..809f198
--- /dev/null
@@ -0,0 +1,48 @@
+#ifndef _HASHLIB_HASHLIB_BUFFER_H
+#define _HASHLIB_HASHLIB_BUFFER_H
+
+#include "Python.h"
+
+/*
+ * Allow to use the 'data' or 'string' keyword in hashlib.new()
+ * and other hash functions named constructors.
+ *
+ * - If 'data' and 'string' are both non-NULL, set an exception and return -1.
+ * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0.
+ * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation
+ *   warning is set when 'string' is specified.
+ *
+ * The symbol is exported for '_hashlib' and HACL*-based extension modules.
+ */
+PyAPI_FUNC(int)
+_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string);
+
+/*
+ * Obtain a buffer view from a buffer-like object 'obj'.
+ *
+ * On success, store the result in 'view' and return 0.
+ * On error, set an exception and return -1.
+ *
+ * The symbol is exported for '_hashlib' and HACL*-based extension modules.
+ */
+PyAPI_FUNC(int)
+_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view);
+
+/*
+ * Call _Py_hashlib_get_buffer_view() and check if it succeeded.
+ *
+ * On error, set an exception and execute the ERRACTION statements.
+ */
+#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION)      \
+    do {                                                    \
+        if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) {   \
+            assert(PyErr_Occurred());                       \
+            ERRACTION;                                      \
+        }                                                   \
+    } while (0)
+
+/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */
+#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW)                \
+    GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL)
+
+#endif // !_HASHLIB_HASHLIB_BUFFER_H
diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h
new file mode 100644 (file)
index 0000000..09add71
--- /dev/null
@@ -0,0 +1,19 @@
+/*
+ * Utilities used when fetching a message digest from a digest-like identifier.
+ */
+
+#ifndef _HASHLIB_HASHLIB_FETCH_H
+#define _HASHLIB_HASHLIB_FETCH_H
+
+#include "Python.h"
+
+/*
+ * Internal error messages used for reporting an unsupported hash algorithm.
+ * The algorithm can be given by its name, a callable or a PEP-247 module.
+ * The same message is raised by Lib/hashlib.py::__get_builtin_constructor()
+ * and _hmacmodule.c::find_hash_info().
+ */
+#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM       "unsupported hash algorithm %S"
+#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM   "unsupported hash algorithm %s"
+
+#endif // !_HASHLIB_HASHLIB_FETCH_H
diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h
new file mode 100644 (file)
index 0000000..d6924a2
--- /dev/null
@@ -0,0 +1,82 @@
+#ifndef _HASHLIB_HASHLIB_MUTEX_H
+#define _HASHLIB_HASHLIB_MUTEX_H
+
+#include "Python.h"
+#include "pycore_lock.h"    // PyMutex
+
+/*
+ * Message length above which the GIL is to be released
+ * when performing hashing operations.
+ */
+#define HASHLIB_GIL_MINSIZE 2048
+
+/*
+ * Helper code to synchronize access to the hash object when the GIL is
+ * released around a CPU consuming hashlib operation.
+ *
+ * Code accessing a mutable part of the hash object must be enclosed in
+ * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release
+ * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if
+ * they wish to release the GIL for an operation.
+ */
+
+#define HASHLIB_OBJECT_HEAD                                             \
+    PyObject_HEAD                                                       \
+    /* Guard against race conditions during incremental update(). */    \
+    PyMutex mutex;
+
+#define HASHLIB_INIT_MUTEX(OBJ)         \
+    do {                                \
+        (OBJ)->mutex = (PyMutex){0};    \
+    } while (0)
+
+#define HASHLIB_ACQUIRE_LOCK(OBJ)   PyMutex_Lock(&(OBJ)->mutex)
+#define HASHLIB_RELEASE_LOCK(OBJ)   PyMutex_Unlock(&(OBJ)->mutex)
+
+// Macros for executing code while conditionally holding the GIL.
+//
+// These only drop the GIL if the lock acquisition itself is likely to
+// block. Thus the non-blocking acquire gating the GIL release for a
+// blocking lock acquisition. The intent of these macros is to surround
+// the assumed always "fast" operations that you aren't releasing the
+// GIL around.
+
+/*
+ * Execute a suite of C statements 'STATEMENTS'.
+ *
+ * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold.
+ */
+#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS)    \
+    do {                                                            \
+        if ((SIZE) > HASHLIB_GIL_MINSIZE) {                         \
+            Py_BEGIN_ALLOW_THREADS                                  \
+            STATEMENTS;                                             \
+            Py_END_ALLOW_THREADS                                    \
+        }                                                           \
+        else {                                                      \
+            STATEMENTS;                                             \
+        }                                                           \
+    } while (0)
+
+/*
+ * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'.
+ *
+ * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold.
+ */
+#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \
+    do {                                                            \
+        if ((SIZE) > HASHLIB_GIL_MINSIZE) {                         \
+            Py_BEGIN_ALLOW_THREADS                                  \
+            HASHLIB_ACQUIRE_LOCK(OBJ);                              \
+            STATEMENTS;                                             \
+            HASHLIB_RELEASE_LOCK(OBJ);                              \
+            Py_END_ALLOW_THREADS                                    \
+        }                                                           \
+        else {                                                      \
+            HASHLIB_ACQUIRE_LOCK(OBJ);                              \
+            STATEMENTS;                                             \
+            HASHLIB_RELEASE_LOCK(OBJ);                              \
+        }                                                           \
+    } while (0)
+
+#endif // !_HASHLIB_HASHLIB_MUTEX_H
index 00f98c090b3952a313530d38599bd855e564a3f0..26412cb62430c9dca31f8405b155e803f5f50287 100644 (file)
 
 #include "Python.h"
 #include "pycore_hashtable.h"
-#include "pycore_strhex.h"               // _Py_strhex()
-#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED
-#include "hashlib.h"
+#include "pycore_strhex.h"                  // _Py_strhex()
+#include "pycore_pyatomic_ft_wrappers.h"    // FT_ATOMIC_LOAD_PTR_RELAXED
+
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_fetch.h"
+#include "_hashlib/hashlib_mutex.h"
 
 /* EVP is the preferred interface to hashing in OpenSSL */
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
-#include <openssl/crypto.h>              // FIPS_mode()
+#include <openssl/crypto.h>                 // FIPS_mode()
 /* We use the object interface to discover what hashes OpenSSL supports. */
 #include <openssl/objects.h>
 #include <openssl/err.h>
@@ -532,7 +535,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod)
 {
     raise_unsupported_algorithm_impl(
         state->unsupported_digestmod_error,
-        HASHLIB_UNSUPPORTED_ALGORITHM,
+        _Py_HASHLIB_UNSUPPORTED_ALGORITHM,
         digestmod
     );
 }
@@ -542,7 +545,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name)
 {
     raise_unsupported_algorithm_impl(
         state->unsupported_digestmod_error,
-        HASHLIB_UNSUPPORTED_STR_ALGORITHM,
+        _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM,
         name
     );
 }
index 163f238a4268d0cec57dbb59255314775684a8af..13c969056be354c5dd7039b1955625a9beaffdac 100644 (file)
 #endif
 
 #include "Python.h"
-#include "hashlib.h"
-#include "pycore_strhex.h"       // _Py_strhex()
-#include "pycore_typeobject.h"
 #include "pycore_moduleobject.h"
+#include "pycore_strhex.h"              // _Py_strhex()
+#include "pycore_typeobject.h"
+
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_mutex.h"
 
 // QUICK CPU AUTODETECTION
 //
diff --git a/Modules/hashlib.h b/Modules/hashlib.h
deleted file mode 100644 (file)
index 5ada4ef..0000000
+++ /dev/null
@@ -1,173 +0,0 @@
-/* Common code for use by all hashlib related modules. */
-
-#include "pycore_lock.h"        // PyMutex
-
-/*
- * Internal error messages used for reporting an unsupported hash algorithm.
- * The algorithm can be given by its name, a callable or a PEP-247 module.
- * The same message is raised by Lib/hashlib.py::__get_builtin_constructor()
- * and _hmacmodule.c::find_hash_info().
- */
-#define HASHLIB_UNSUPPORTED_ALGORITHM       "unsupported hash algorithm %S"
-#define HASHLIB_UNSUPPORTED_STR_ALGORITHM   "unsupported hash algorithm %s"
-
-/*
- * Obtain a buffer view from a buffer-like object 'obj'.
- *
- * On success, store the result in 'view' and return 0.
- * On error, set an exception and return -1.
- */
-static inline int
-_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view)
-{
-    if (PyUnicode_Check(obj)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "Strings must be encoded before hashing");
-        return -1;
-    }
-    if (!PyObject_CheckBuffer(obj)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "object supporting the buffer API required");
-        return -1;
-    }
-    if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) {
-        return -1;
-    }
-    if (view->ndim > 1) {
-        PyErr_SetString(PyExc_BufferError,
-                        "Buffer must be single dimension");
-        PyBuffer_Release(view);
-        return -1;
-    }
-    return 0;
-}
-
-/*
- * Call _Py_hashlib_get_buffer_view() and check if it succeeded.
- *
- * On error, set an exception and execute the ERRACTION statements.
- */
-#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION)      \
-    do {                                                    \
-        if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) {   \
-            assert(PyErr_Occurred());                       \
-            ERRACTION;                                      \
-        }                                                   \
-    } while (0)
-
-#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW)                \
-    GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL)
-
-/*
- * Helper code to synchronize access to the hash object when the GIL is
- * released around a CPU consuming hashlib operation.
- *
- * Code accessing a mutable part of the hash object must be enclosed in
- * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release
- * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if
- * they wish to release the GIL for an operation.
- */
-
-#define HASHLIB_OBJECT_HEAD                                             \
-    PyObject_HEAD                                                       \
-    /* Guard against race conditions during incremental update(). */    \
-    PyMutex mutex;
-
-#define HASHLIB_INIT_MUTEX(OBJ)         \
-    do {                                \
-        (OBJ)->mutex = (PyMutex){0};    \
-    } while (0)
-
-#define HASHLIB_ACQUIRE_LOCK(OBJ)   PyMutex_Lock(&(OBJ)->mutex)
-#define HASHLIB_RELEASE_LOCK(OBJ)   PyMutex_Unlock(&(OBJ)->mutex)
-
-/*
- * Message length above which the GIL is to be released
- * when performing hashing operations.
- */
-#define HASHLIB_GIL_MINSIZE         2048
-
-// Macros for executing code while conditionally holding the GIL.
-//
-// These only drop the GIL if the lock acquisition itself is likely to
-// block. Thus the non-blocking acquire gating the GIL release for a
-// blocking lock acquisition. The intent of these macros is to surround
-// the assumed always "fast" operations that you aren't releasing the
-// GIL around.
-
-/*
- * Execute a suite of C statements 'STATEMENTS'.
- *
- * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold.
- */
-#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS)    \
-    do {                                                            \
-        if ((SIZE) > HASHLIB_GIL_MINSIZE) {                         \
-            Py_BEGIN_ALLOW_THREADS                                  \
-            STATEMENTS;                                             \
-            Py_END_ALLOW_THREADS                                    \
-        }                                                           \
-        else {                                                      \
-            STATEMENTS;                                             \
-        }                                                           \
-    } while (0)
-
-/*
- * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'.
- *
- * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold.
- */
-#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \
-    do {                                                            \
-        if ((SIZE) > HASHLIB_GIL_MINSIZE) {                         \
-            Py_BEGIN_ALLOW_THREADS                                  \
-            HASHLIB_ACQUIRE_LOCK(OBJ);                              \
-            STATEMENTS;                                             \
-            HASHLIB_RELEASE_LOCK(OBJ);                              \
-            Py_END_ALLOW_THREADS                                    \
-        }                                                           \
-        else {                                                      \
-            HASHLIB_ACQUIRE_LOCK(OBJ);                              \
-            STATEMENTS;                                             \
-            HASHLIB_RELEASE_LOCK(OBJ);                              \
-        }                                                           \
-    } while (0)
-
-static inline int
-_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string)
-{
-    if (data != NULL && string == NULL) {
-        // called as H(data) or H(data=...)
-        *res = data;
-        return 1;
-    }
-    else if (data == NULL && string != NULL) {
-        // called as H(string=...)
-        if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                         "the 'string' keyword parameter is deprecated since "
-                         "Python 3.15 and slated for removal in Python 3.19; "
-                         "use the 'data' keyword parameter or pass the data "
-                         "to hash as a positional argument instead", 1) < 0)
-        {
-            *res = NULL;
-            return -1;
-        }
-        *res = string;
-        return 1;
-    }
-    else if (data == NULL && string == NULL) {
-        // fast path when no data is given
-        assert(!PyErr_Occurred());
-        *res = NULL;
-        return 0;
-    }
-    else {
-        // called as H(data=..., string)
-        *res = NULL;
-        PyErr_SetString(PyExc_TypeError,
-                        "'data' and 'string' are mutually exclusive "
-                        "and support for 'string' keyword parameter "
-                        "is slated for removal in a future version.");
-        return -1;
-    }
-}
index 694e2a095744ffc71a3cab8eed0d7cf13c70b7b0..92be49c5a879f351d2114542983b046a39e92f15 100644 (file)
 #include "pycore_hashtable.h"
 #include "pycore_strhex.h"              // _Py_strhex()
 
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_fetch.h"
+#include "_hashlib/hashlib_mutex.h"
+
 /*
  * Taken from blake2module.c. In the future, detection of SIMD support
  * should be delegated to https://github.com/python/cpython/pull/125011.
@@ -47,8 +51,6 @@
 
 #include <stdbool.h>
 
-#include "hashlib.h"
-
 // --- Reusable error messages ------------------------------------------------
 
 static inline void
@@ -656,7 +658,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref)
     }
     if (rc == 0) {
         PyErr_Format(state->unknown_hash_error,
-                     HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref);
+                     _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref);
         return NULL;
     }
     assert(info != NULL);
index 8b6dd4a8195dfb597c6b9155a3c40b2994cb304c..d5dc4f60a575d440817036ad06141453bd3fa253 100644 (file)
 #endif
 
 #include "Python.h"
-#include "pycore_strhex.h" // _Py_strhex()
+#include "pycore_strhex.h"              // _Py_strhex()
 
-#include "hashlib.h"
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_mutex.h"
 
 #include "_hacl/Hacl_Hash_MD5.h"
 
index faa9dcccc5755bc9f0b487215e7dbe53f46dfabc..86e5691e8463e4cadd520ff22bcc7b0d10cb0684 100644 (file)
 #endif
 
 #include "Python.h"
-#include "hashlib.h"
-#include "pycore_strhex.h"        // _Py_strhex()
-#include "pycore_typeobject.h"    // _PyType_GetModuleState()
+#include "pycore_strhex.h"              // _Py_strhex()
+#include "pycore_typeobject.h"          // _PyType_GetModuleState()
+
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_mutex.h"
 
 #include "_hacl/Hacl_Hash_SHA1.h"
 
index 36300ba899fd4444e6647bf5e186a299443e3eb0..dbf6dde1b8c121ccb13238d43445a62bdbb64fa3 100644 (file)
 #endif
 
 #include "Python.h"
-#include "pycore_moduleobject.h"  // _PyModule_GetState()
-#include "pycore_typeobject.h"    // _PyType_GetModuleState()
-#include "pycore_strhex.h"        // _Py_strhex()
+#include "pycore_moduleobject.h"        // _PyModule_GetState()
+#include "pycore_strhex.h"              // _Py_strhex()
+#include "pycore_typeobject.h"          // _PyType_GetModuleState()
 
-#include "hashlib.h"
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_mutex.h"
 
 #include "_hacl/Hacl_Hash_SHA2.h"
 
index 5764556bb680f358bc7a6d6d4d5166e56e7dadc9..c67bfadbe4664a66d70e5192302176f045798d92 100644 (file)
 #endif
 
 #include "Python.h"
-#include "pycore_strhex.h"        // _Py_strhex()
-#include "pycore_typeobject.h"    // _PyType_GetModuleState()
-#include "hashlib.h"
+#include "pycore_strhex.h"              // _Py_strhex()
+#include "pycore_typeobject.h"          // _PyType_GetModuleState()
+
+#include "_hashlib/hashlib_buffer.h"
+#include "_hashlib/hashlib_mutex.h"
 
 #include "_hacl/Hacl_Hash_SHA3.h"
 
index 2cd205224bc0891d380e7fc7ef26e7cac0bc0769..cfb43cee935b8681cdb3728695a106ffe2c7573c 100644 (file)
   <ItemGroup>
     <ClCompile Include="..\Modules\_hashopenssl.c" />
   </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\Modules\_hashlib\hashlib_buffer.h" />
+    <ClInclude Include="..\Modules\_hashlib\hashlib_fetch.h" />
+    <ClInclude Include="..\Modules\_hashlib\hashlib_mutex.h" />
+  </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />
   </ItemGroup>
index 7a0700c007f6442d7197bf2ac9f69f1f5e695ba0..d465d92a956edac22698788650851517f958fb52 100644 (file)
@@ -18,4 +18,4 @@
       <Filter>Resource Files</Filter>
     </ResourceCompile>
   </ItemGroup>
-</Project>
\ No newline at end of file
+</Project>
index 517103acea8d8e83f495c248ec175e6340b6910a..c59b380d814ed950415979053ad5823bece87161 100644 (file)
       <PreprocessorDefinitions>HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalOptions>/arch:AVX %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
+    <ClCompile Include="..\Modules\_hashlib\hashlib_buffer.c" />
+    <ClInclude Include="..\Modules\_hashlib\hashlib_buffer.h" />
+    <ClInclude Include="..\Modules\_hashlib\hashlib_fetch.h" />
+    <ClInclude Include="..\Modules\_hashlib\hashlib_mutex.h" />
     <ClCompile Include="..\Modules\_heapqmodule.c" />
     <ClCompile Include="..\Modules\_json.c" />
     <ClCompile Include="..\Modules\_localemodule.c" />
index e9eedfd1312fae109b2858addc7af1e630c3094e..1410cbbef6c8492cffb873c6bd0cc8fe1e3ad6aa 100644 (file)
     <ClInclude Include="..\Include\weakrefobject.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Modules\_hashlib\hashlib_buffer.h">
+      <Filter>Modules\_hashlib</Filter>
+    </ClInclude>
+    <ClInclude Include="..\Modules\_hashlib\hashlib_fetch.h">
+      <Filter>Modules\_hashlib</Filter>
+    </ClInclude>
+    <ClInclude Include="..\Modules\_hashlib\hashlib_mutex.h">
+      <Filter>Modules\_hashlib</Filter>
+    </ClInclude>
     <ClInclude Include="..\Modules\_math.h">
       <Filter>Modules</Filter>
     </ClInclude>
     <ClCompile Include="..\Modules\_hacl\Hacl_Streaming_HMAC.c">
       <Filter>Modules</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_hashlib\hashlib_buffer.c">
+      <Filter>Modules\_hashlib</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\_heapqmodule.c">
       <Filter>Modules</Filter>
     </ClCompile>
index 74df430d10d6bdddb8da83e1713c5f47ae6f8919..0e7aefed5ee62d79e97e7faa4067a1a8b7f6b45e 100755 (executable)
--- a/configure
+++ b/configure
@@ -725,6 +725,8 @@ LIBHACL_BLAKE2_SIMD128_OBJS
 LIBHACL_SIMD128_FLAGS
 LIBHACL_LDFLAGS
 LIBHACL_CFLAGS
+LIBHASHLIB_INTERNAL
+LIBHASHLIB_INTERNAL_CFLAGS
 MODULE_UNICODEDATA_FALSE
 MODULE_UNICODEDATA_TRUE
 MODULE__MULTIBYTECODEC_FALSE
@@ -29949,6 +29951,7 @@ SRCDIRS="\
   Modules/_decimal \
   Modules/_decimal/libmpdec \
   Modules/_hacl \
+  Modules/_hashlib \
   Modules/_io \
   Modules/_multiprocessing \
   Modules/_sqlite \
@@ -32525,6 +32528,15 @@ then :
 fi
 
 
+###############################################################################
+# Cryptographic primitives
+LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib"
+LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)"
+LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)"
+
+
+
+
 ###############################################################################
 # HACL* compilation and linking configuration (contact: @picnixz)
 #
@@ -32773,8 +32785,8 @@ fi
   if test "x$py_cv_module__md5" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__md5" = yes; then
@@ -32818,8 +32830,8 @@ fi
   if test "x$py_cv_module__sha1" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__sha1" = yes; then
@@ -32863,8 +32875,8 @@ fi
   if test "x$py_cv_module__sha2" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__sha2" = yes; then
@@ -32908,8 +32920,8 @@ fi
   if test "x$py_cv_module__sha3" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__sha3" = yes; then
@@ -32953,8 +32965,8 @@ fi
   if test "x$py_cv_module__blake2" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__blake2" = yes; then
@@ -32999,8 +33011,8 @@ fi
   if test "x$py_cv_module__hmac" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__hmac" = yes; then
@@ -33681,8 +33693,8 @@ fi
   if test "x$py_cv_module__hashlib" = xyes
 then :
 
-    as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl"
-    as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl"
 
 fi
    if test "$py_cv_module__hashlib" = yes; then
index 29e36f217402f13bc1caa16ddcaa014f14304194..1e590e1d0fd7278c9d9918cc49cdcd3093c0ec0c 100644 (file)
@@ -7187,6 +7187,7 @@ SRCDIRS="\
   Modules/_decimal \
   Modules/_decimal/libmpdec \
   Modules/_hacl \
+  Modules/_hashlib \
   Modules/_io \
   Modules/_multiprocessing \
   Modules/_sqlite \
@@ -7957,6 +7958,15 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw])
 PY_STDLIB_MOD_SIMPLE([_multibytecodec])
 PY_STDLIB_MOD_SIMPLE([unicodedata])
 
+###############################################################################
+# Cryptographic primitives
+LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib"
+LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)"
+LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)"
+
+AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS])
+AC_SUBST([LIBHASHLIB_INTERNAL])
+
 ###############################################################################
 # HACL* compilation and linking configuration (contact: @picnixz)
 #
@@ -8093,7 +8103,9 @@ dnl The EXTNAME is the name of the extension module being built.
 AC_DEFUN([PY_HACL_CREATE_MODULE], [
   AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]])
   AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]])
-  PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)])
+  PY_STDLIB_MOD([$2], [$3], [],
+    [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS],
+    [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS])
   AS_VAR_POPDEF([v])
 ])
 
@@ -8174,7 +8186,8 @@ dnl OpenSSL bindings
 PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes],
   [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS])
 PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes],
-  [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS])
+  [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS],
+  [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS])
 
 dnl test modules
 PY_STDLIB_MOD([_testcapi],