mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
void _mi_thread_done(mi_heap_t* heap);
void _mi_thread_data_collect(void);
+void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
// os.c
void _mi_os_init(void); // called from process init
uint8_t _mi_bin(size_t size); // for stats
// "heap.c"
+void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id);
void _mi_heap_destroy_pages(mi_heap_t* heap);
void _mi_heap_collect_abandon(mi_heap_t* heap);
void _mi_heap_set_default_direct(mi_heap_t* heap);
# error "pycore_mimalloc.h must be included before mimalloc.h"
#endif
+typedef enum {
+ _Py_MIMALLOC_HEAP_MEM = 0, // PyMem_Malloc() and friends
+ _Py_MIMALLOC_HEAP_OBJECT = 1, // non-GC objects
+ _Py_MIMALLOC_HEAP_GC = 2, // GC objects without pre-header
+ _Py_MIMALLOC_HEAP_GC_PRE = 3, // GC objects with pre-header
+ _Py_MIMALLOC_HEAP_COUNT
+} _Py_mimalloc_heap_id;
+
#include "pycore_pymem.h"
+
+#ifdef WITH_MIMALLOC
#define MI_DEBUG_UNINIT PYMEM_CLEANBYTE
#define MI_DEBUG_FREED PYMEM_DEADBYTE
#define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE
+#ifdef Py_DEBUG
+# define MI_DEBUG 1
+#else
+# define MI_DEBUG 0
+#endif
#include "mimalloc.h"
+#include "mimalloc/types.h"
+#include "mimalloc/internal.h"
+#endif
+
+#ifdef Py_GIL_DISABLED
+struct _mimalloc_thread_state {
+ mi_heap_t *current_object_heap;
+ mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT];
+ mi_tld_t tld;
+};
+#endif
#endif // Py_INTERNAL_MIMALLOC_H
int whence);
extern void _PyThreadState_Bind(PyThreadState *tstate);
extern void _PyThreadState_DeleteExcept(PyThreadState *tstate);
+extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate);
// Export for '_testinternalcapi' shared extension
PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate);
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
+
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
// PyThreadState fields are exposed as part of the C API, although most fields
// semi-public fields are in PyThreadState.
PyThreadState base;
- // TODO: add private fields here
+#ifdef Py_GIL_DISABLED
+ struct _mimalloc_thread_state mimalloc;
+#endif
+
} _PyThreadStateImpl;
const bool force = collect >= MI_FORCE;
_mi_deferred_free(heap, force);
+ // gh-112532: we may be called from a thread that is not the owner of the heap
+ bool is_main_thread = _mi_is_main_thread() && heap->thread_id == _mi_thread_id();
+
// note: never reclaim on collect but leave it to threads that need storage to reclaim
const bool force_main =
#ifdef NDEBUG
#else
collect >= MI_FORCE
#endif
- && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim;
+ && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim;
if (force_main) {
// the main thread is abandoned (end-of-program), try to reclaim all abandoned segments.
}
// collect regions on program-exit (or shared library unload)
- if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
+ if (force && is_main_thread && mi_heap_is_backing(heap)) {
_mi_thread_data_collect(); // collect thread data cache
_mi_arena_collect(true /* force purge */, &heap->tld->stats);
}
return bheap;
}
-mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {
- mi_heap_t* bheap = mi_heap_get_backing();
- mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
- if (heap == NULL) return NULL;
+void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id)
+{
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
- heap->tld = bheap->tld;
+ heap->tld = tld;
heap->thread_id = _mi_thread_id();
heap->arena_id = arena_id;
- _mi_random_split(&bheap->random, &heap->random);
+ if (heap == tld->heap_backing) {
+ _mi_random_init(&heap->random);
+ }
+ else {
+ _mi_random_split(&tld->heap_backing->random, &heap->random);
+ }
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
+}
+
+mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {
+ mi_heap_t* bheap = mi_heap_get_backing();
+ mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
+ if (heap == NULL) return NULL;
+ _mi_heap_init_ex(heap, bheap->tld, arena_id);
heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe
// push on the thread local heaps list
heap->next = heap->tld->heaps;
mi_thread_data_t* td = mi_thread_data_zalloc();
if (td == NULL) return false;
- mi_tld_t* tld = &td->tld;
- mi_heap_t* heap = &td->heap;
+ _mi_tld_init(&td->tld, &td->heap);
+ _mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none());
+ _mi_heap_set_default_direct(&td->heap);
+ }
+ return false;
+}
+
+void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) {
_mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld));
- _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap));
- heap->thread_id = _mi_thread_id();
- _mi_random_init(&heap->random);
- heap->cookie = _mi_heap_random_next(heap) | 1;
- heap->keys[0] = _mi_heap_random_next(heap);
- heap->keys[1] = _mi_heap_random_next(heap);
- heap->tld = tld;
- tld->heap_backing = heap;
- tld->heaps = heap;
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
- _mi_heap_set_default_direct(heap);
- }
- return false;
+ tld->heap_backing = bheap;
+ tld->heaps = bheap;
}
// Free the thread local default heap (called from `mi_thread_done`)
void *
_PyMem_MiMalloc(void *ctx, size_t size)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];
+ return mi_heap_malloc(heap, size);
+#else
return mi_malloc(size);
+#endif
}
void *
_PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];
+ return mi_heap_calloc(heap, nelem, elsize);
+#else
return mi_calloc(nelem, elsize);
+#endif
}
void *
_PyMem_MiRealloc(void *ctx, void *ptr, size_t size)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];
+ return mi_heap_realloc(heap, ptr, size);
+#else
return mi_realloc(ptr, size);
+#endif
}
void
void *
_PyObject_MiMalloc(void *ctx, size_t nbytes)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = tstate->mimalloc.current_object_heap;
+ return mi_heap_malloc(heap, nbytes);
+#else
return mi_malloc(nbytes);
+#endif
}
void *
_PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = tstate->mimalloc.current_object_heap;
+ return mi_heap_calloc(heap, nelem, elsize);
+#else
return mi_calloc(nelem, elsize);
+#endif
}
void *
_PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes)
{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ mi_heap_t *heap = tstate->mimalloc.current_object_heap;
+ return mi_heap_realloc(heap, ptr, nbytes);
+#else
return mi_realloc(ptr, nbytes);
+#endif
}
void
}
finalize_interp_types(tstate->interp);
+
+ /* finalize_interp_types may allocate Python objects so we may need to
+ abandon mimalloc segments again */
+ _PyThreadState_ClearMimallocHeaps(tstate);
}
static void bind_gilstate_tstate(PyThreadState *);
static void unbind_gilstate_tstate(PyThreadState *);
+static void tstate_mimalloc_bind(PyThreadState *);
+
static void
bind_tstate(PyThreadState *tstate)
{
tstate->native_thread_id = PyThread_get_thread_native_id();
#endif
+ // mimalloc state needs to be initialized from the active thread.
+ tstate_mimalloc_bind(tstate);
+
tstate->_status.bound = 1;
}
tstate->on_delete(tstate->on_delete_data);
}
+ _PyThreadState_ClearMimallocHeaps(tstate);
+
tstate->_status.cleared = 1;
// XXX Call _PyThreadStateSwap(runtime, NULL) here if "current".
}
return 1;
}
+
+/********************/
+/* mimalloc support */
+/********************/
+
+static void
+tstate_mimalloc_bind(PyThreadState *tstate)
+{
+#ifdef Py_GIL_DISABLED
+ struct _mimalloc_thread_state *mts = &((_PyThreadStateImpl*)tstate)->mimalloc;
+
+ // Initialize the mimalloc thread state. This must be called from the
+ // same thread that will use the thread state. The "mem" heap doubles as
+ // the "backing" heap.
+ mi_tld_t *tld = &mts->tld;
+ _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]);
+
+ // Initialize each heap
+ for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) {
+ _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none());
+ }
+
+ // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT.
+ // _PyObject_GC_New() and similar functions temporarily override this to
+ // use one of the GC heaps.
+ mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+#endif
+}
+
+void
+_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate)
+{
+#ifdef Py_GIL_DISABLED
+ if (!tstate->_status.bound) {
+ // The mimalloc heaps are only initialized when the thread is bound.
+ return;
+ }
+
+ _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+ for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) {
+ // Abandon all segments in use by this thread. This pushes them to
+ // a shared pool to later be reclaimed by other threads. It's important
+ // to do this before the thread state is destroyed so that objects
+ // remain visible to the GC.
+ _mi_heap_collect_abandon(&tstate_impl->mimalloc.heaps[i]);
+ }
+#endif
+}
projects / thirdparty / Python / cpython.git / commitdiff
