+++ /dev/null
-
-/* An object allocator for Python.
-
- Here is an introduction to the layers of the Python memory architecture,
- showing where the object allocator is actually used (layer +2), It is
- called for every object allocation and deallocation (PyObject_New/Del),
- unless the object-specific allocators implement a proprietary allocation
- scheme (ex.: ints use a simple free list). This is also the place where
- the cyclic garbage collector operates selectively on container objects.
-
-
- Object-specific allocators
- _____ ______ ______ ________
- [ int ] [ dict ] [ list ] ... [ string ] Python core |
-+3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
- _______________________________ | |
- [ Python's object allocator ] | |
-+2 | ####### Object memory ####### | <------ Internal buffers ------> |
- ______________________________________________________________ |
- [ Python's raw memory allocator (PyMem_ API) ] |
-+1 | <----- Python memory (under PyMem manager's control) ------> | |
- __________________________________________________________________
- [ Underlying general-purpose allocator (ex: C library malloc) ]
- 0 | <------ Virtual memory allocated for the python process -------> |
-
- =========================================================================
- _______________________________________________________________________
- [ OS-specific Virtual Memory Manager (VMM) ]
--1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
- __________________________________ __________________________________
- [ ] [ ]
--2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
-
-*/
-/*==========================================================================*/
-
-/* A fast, special-purpose memory allocator for small blocks, to be used
- on top of a general-purpose malloc -- heavily based on previous art. */
-
-/* Vladimir Marangozov -- August 2000 */
-
-/*
- * "Memory management is where the rubber meets the road -- if we do the wrong
- * thing at any level, the results will not be good. And if we don't make the
- * levels work well together, we are in serious trouble." (1)
- *
- * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
- * "Dynamic Storage Allocation: A Survey and Critical Review",
- * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
- */
-
-#ifndef Py_INTERNAL_PYMALLOC_H
-#define Py_INTERNAL_PYMALLOC_H
-
-/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */
-
-/*==========================================================================*/
-
-/*
- * Allocation strategy abstract:
- *
- * For small requests, the allocator sub-allocates <Big> blocks of memory.
- * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the
- * system's allocator.
- *
- * Small requests are grouped in size classes spaced 8 bytes apart, due
- * to the required valid alignment of the returned address. Requests of
- * a particular size are serviced from memory pools of 4K (one VMM page).
- * Pools are fragmented on demand and contain free lists of blocks of one
- * particular size class. In other words, there is a fixed-size allocator
- * for each size class. Free pools are shared by the different allocators
- * thus minimizing the space reserved for a particular size class.
- *
- * This allocation strategy is a variant of what is known as "simple
- * segregated storage based on array of free lists". The main drawback of
- * simple segregated storage is that we might end up with lot of reserved
- * memory for the different free lists, which degenerate in time. To avoid
- * this, we partition each free list in pools and we share dynamically the
- * reserved space between all free lists. This technique is quite efficient
- * for memory intensive programs which allocate mainly small-sized blocks.
- *
- * For small requests we have the following table:
- *
- * Request in bytes Size of allocated block Size class idx
- * ----------------------------------------------------------------
- * 1-8 8 0
- * 9-16 16 1
- * 17-24 24 2
- * 25-32 32 3
- * 33-40 40 4
- * 41-48 48 5
- * 49-56 56 6
- * 57-64 64 7
- * 65-72 72 8
- * ... ... ...
- * 497-504 504 62
- * 505-512 512 63
- *
- * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying
- * allocator.
- */
-
-/*==========================================================================*/
-
-/*
- * -- Main tunable settings section --
- */
-
-/*
- * Alignment of addresses returned to the user. 8-bytes alignment works
- * on most current architectures (with 32-bit or 64-bit address busses).
- * The alignment value is also used for grouping small requests in size
- * classes spaced ALIGNMENT bytes apart.
- *
- * You shouldn't change this unless you know what you are doing.
- */
-#define ALIGNMENT 8 /* must be 2^N */
-#define ALIGNMENT_SHIFT 3
-
-/* Return the number of bytes in size class I, as a uint. */
-#define INDEX2SIZE(I) (((unsigned int)(I) + 1) << ALIGNMENT_SHIFT)
-
-/*
- * Max size threshold below which malloc requests are considered to be
- * small enough in order to use preallocated memory pools. You can tune
- * this value according to your application behaviour and memory needs.
- *
- * Note: a size threshold of 512 guarantees that newly created dictionaries
- * will be allocated from preallocated memory pools on 64-bit.
- *
- * The following invariants must hold:
- * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512
- * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT
- *
- * Although not required, for better performance and space efficiency,
- * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.
- */
-#define SMALL_REQUEST_THRESHOLD 512
-#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT)
-
-#if NB_SMALL_SIZE_CLASSES > 64
-#error "NB_SMALL_SIZE_CLASSES should be less than 64"
-#endif /* NB_SMALL_SIZE_CLASSES > 64 */
-
-/*
- * The system's VMM page size can be obtained on most unices with a
- * getpagesize() call or deduced from various header files. To make
- * things simpler, we assume that it is 4K, which is OK for most systems.
- * It is probably better if this is the native page size, but it doesn't
- * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page
- * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation
- * violation fault. 4K is apparently OK for all the platforms that python
- * currently targets.
- */
-#define SYSTEM_PAGE_SIZE (4 * 1024)
-#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1)
-
-/*
- * Maximum amount of memory managed by the allocator for small requests.
- */
-#ifdef WITH_MEMORY_LIMITS
-#ifndef SMALL_MEMORY_LIMIT
-#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MiB -- more? */
-#endif
-#endif
-
-/*
- * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
- * on a page boundary. This is a reserved virtual address space for the
- * current process (obtained through a malloc()/mmap() call). In no way this
- * means that the memory arenas will be used entirely. A malloc(<Big>) is
- * usually an address range reservation for <Big> bytes, unless all pages within
- * this space are referenced subsequently. So malloc'ing big blocks and not
- * using them does not mean "wasting memory". It's an addressable range
- * wastage...
- *
- * Arenas are allocated with mmap() on systems supporting anonymous memory
- * mappings to reduce heap fragmentation.
- */
-#define ARENA_SIZE (256 << 10) /* 256 KiB */
-
-#ifdef WITH_MEMORY_LIMITS
-#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE)
-#endif
-
-/*
- * Size of the pools used for small blocks. Should be a power of 2,
- * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
- */
-#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */
-#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK
-
-/*
- * -- End of tunable settings section --
- */
-
-/*==========================================================================*/
-
-/*
- * Locking
- *
- * To reduce lock contention, it would probably be better to refine the
- * crude function locking with per size class locking. I'm not positive
- * however, whether it's worth switching to such locking policy because
- * of the performance penalty it might introduce.
- *
- * The following macros describe the simplest (should also be the fastest)
- * lock object on a particular platform and the init/fini/lock/unlock
- * operations on it. The locks defined here are not expected to be recursive
- * because it is assumed that they will always be called in the order:
- * INIT, [LOCK, UNLOCK]*, FINI.
- */
-
-/*
- * Python's threads are serialized, so object malloc locking is disabled.
- */
-#define SIMPLELOCK_DECL(lock) /* simple lock declaration */
-#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */
-#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */
-#define SIMPLELOCK_LOCK(lock) /* acquire released lock */
-#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */
-
-/* When you say memory, my mind reasons in terms of (pointers to) blocks */
-typedef uint8_t pyblock;
-
-/* Pool for small blocks. */
-struct pool_header {
- union { pyblock *_padding;
- unsigned int count; } ref; /* number of allocated blocks */
- pyblock *freeblock; /* pool's free list head */
- struct pool_header *nextpool; /* next pool of this size class */
- struct pool_header *prevpool; /* previous pool "" */
- unsigned int arenaindex; /* index into arenas of base adr */
- unsigned int szidx; /* block size class index */
- unsigned int nextoffset; /* bytes to virgin block */
- unsigned int maxnextoffset; /* largest valid nextoffset */
-};
-
-typedef struct pool_header *poolp;
-
-/* Record keeping for arenas. */
-struct arena_object {
- /* The address of the arena, as returned by malloc. Note that 0
- * will never be returned by a successful malloc, and is used
- * here to mark an arena_object that doesn't correspond to an
- * allocated arena.
- */
- uintptr_t address;
-
- /* Pool-aligned pointer to the next pool to be carved off. */
- pyblock* pool_address;
-
- /* The number of available pools in the arena: free pools + never-
- * allocated pools.
- */
- unsigned int nfreepools;
-
- /* The total number of pools in the arena, whether or not available. */
- unsigned int ntotalpools;
-
- /* Singly-linked list of available pools. */
- struct pool_header* freepools;
-
- /* Whenever this arena_object is not associated with an allocated
- * arena, the nextarena member is used to link all unassociated
- * arena_objects in the singly-linked `unused_arena_objects` list.
- * The prevarena member is unused in this case.
- *
- * When this arena_object is associated with an allocated arena
- * with at least one available pool, both members are used in the
- * doubly-linked `usable_arenas` list, which is maintained in
- * increasing order of `nfreepools` values.
- *
- * Else this arena_object is associated with an allocated arena
- * all of whose pools are in use. `nextarena` and `prevarena`
- * are both meaningless in this case.
- */
- struct arena_object* nextarena;
- struct arena_object* prevarena;
-};
-
-#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT)
-
-#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */
-
-/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */
-#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE))
-
-/* Return total number of blocks in pool of size index I, as a uint. */
-#define NUMBLOCKS(I) \
- ((unsigned int)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I))
-
-/*==========================================================================*/
-
-/*
- * This malloc lock
- */
-SIMPLELOCK_DECL(_malloc_lock)
-#define LOCK() SIMPLELOCK_LOCK(_malloc_lock)
-#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock)
-#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock)
-#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock)
-
-/*
- * Pool table -- headed, circular, doubly-linked lists of partially used pools.
-
-This is involved. For an index i, usedpools[i+i] is the header for a list of
-all partially used pools holding small blocks with "size class idx" i. So
-usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size
-16, and so on: index 2*i <-> blocks of size (i+1)<<ALIGNMENT_SHIFT.
-
-Pools are carved off an arena's highwater mark (an arena_object's pool_address
-member) as needed. Once carved off, a pool is in one of three states forever
-after:
-
-used == partially used, neither empty nor full
- At least one block in the pool is currently allocated, and at least one
- block in the pool is not currently allocated (note this implies a pool
- has room for at least two blocks).
- This is a pool's initial state, as a pool is created only when malloc
- needs space.
- The pool holds blocks of a fixed size, and is in the circular list headed
- at usedpools[i] (see above). It's linked to the other used pools of the
- same size class via the pool_header's nextpool and prevpool members.
- If all but one block is currently allocated, a malloc can cause a
- transition to the full state. If all but one block is not currently
- allocated, a free can cause a transition to the empty state.
-
-full == all the pool's blocks are currently allocated
- On transition to full, a pool is unlinked from its usedpools[] list.
- It's not linked to from anything then anymore, and its nextpool and
- prevpool members are meaningless until it transitions back to used.
- A free of a block in a full pool puts the pool back in the used state.
- Then it's linked in at the front of the appropriate usedpools[] list, so
- that the next allocation for its size class will reuse the freed block.
-
-empty == all the pool's blocks are currently available for allocation
- On transition to empty, a pool is unlinked from its usedpools[] list,
- and linked to the front of its arena_object's singly-linked freepools list,
- via its nextpool member. The prevpool member has no meaning in this case.
- Empty pools have no inherent size class: the next time a malloc finds
- an empty list in usedpools[], it takes the first pool off of freepools.
- If the size class needed happens to be the same as the size class the pool
- last had, some pool initialization can be skipped.
-
-
-Block Management
-
-Blocks within pools are again carved out as needed. pool->freeblock points to
-the start of a singly-linked list of free blocks within the pool. When a
-block is freed, it's inserted at the front of its pool's freeblock list. Note
-that the available blocks in a pool are *not* linked all together when a pool
-is initialized. Instead only "the first two" (lowest addresses) blocks are
-set up, returning the first such block, and setting pool->freeblock to a
-one-block list holding the second such block. This is consistent with that
-pymalloc strives at all levels (arena, pool, and block) never to touch a piece
-of memory until it's actually needed.
-
-So long as a pool is in the used state, we're certain there *is* a block
-available for allocating, and pool->freeblock is not NULL. If pool->freeblock
-points to the end of the free list before we've carved the entire pool into
-blocks, that means we simply haven't yet gotten to one of the higher-address
-blocks. The offset from the pool_header to the start of "the next" virgin
-block is stored in the pool_header nextoffset member, and the largest value
-of nextoffset that makes sense is stored in the maxnextoffset member when a
-pool is initialized. All the blocks in a pool have been passed out at least
-once when and only when nextoffset > maxnextoffset.
-
-
-Major obscurity: While the usedpools vector is declared to have poolp
-entries, it doesn't really. It really contains two pointers per (conceptual)
-poolp entry, the nextpool and prevpool members of a pool_header. The
-excruciating initialization code below fools C so that
-
- usedpool[i+i]
-
-"acts like" a genuine poolp, but only so long as you only reference its
-nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is
-compensating for that a pool_header's nextpool and prevpool members
-immediately follow a pool_header's first two members:
-
- union { block *_padding;
- uint count; } ref;
- block *freeblock;
-
-each of which consume sizeof(block *) bytes. So what usedpools[i+i] really
-contains is a fudged-up pointer p such that *if* C believes it's a poolp
-pointer, then p->nextpool and p->prevpool are both p (meaning that the headed
-circular list is empty).
-
-It's unclear why the usedpools setup is so convoluted. It could be to
-minimize the amount of cache required to hold this heavily-referenced table
-(which only *needs* the two interpool pointer members of a pool_header). OTOH,
-referencing code has to remember to "double the index" and doing so isn't
-free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying
-on that C doesn't insert any padding anywhere in a pool_header at or before
-the prevpool member.
-**************************************************************************** */
-
-#define MAX_POOLS (2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8)
-
-/*==========================================================================
-Arena management.
-
-`arenas` is a vector of arena_objects. It contains maxarenas entries, some of
-which may not be currently used (== they're arena_objects that aren't
-currently associated with an allocated arena). Note that arenas proper are
-separately malloc'ed.
-
-Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5,
-we do try to free() arenas, and use some mild heuristic strategies to increase
-the likelihood that arenas eventually can be freed.
-
-unused_arena_objects
-
- This is a singly-linked list of the arena_objects that are currently not
- being used (no arena is associated with them). Objects are taken off the
- head of the list in new_arena(), and are pushed on the head of the list in
- PyObject_Free() when the arena is empty. Key invariant: an arena_object
- is on this list if and only if its .address member is 0.
-
-usable_arenas
-
- This is a doubly-linked list of the arena_objects associated with arenas
- that have pools available. These pools are either waiting to be reused,
- or have not been used before. The list is sorted to have the most-
- allocated arenas first (ascending order based on the nfreepools member).
- This means that the next allocation will come from a heavily used arena,
- which gives the nearly empty arenas a chance to be returned to the system.
- In my unscientific tests this dramatically improved the number of arenas
- that could be freed.
-
-Note that an arena_object associated with an arena all of whose pools are
-currently in use isn't on either list.
-*/
-
-/* How many arena_objects do we initially allocate?
- * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4 MiB before growing the
- * `arenas` vector.
- */
-#define INITIAL_ARENA_OBJECTS 16
-
-#endif /* Py_INTERNAL_PYMALLOC_H */
#include "Python.h"
-#include "internal/mem.h"
-#include "internal/pystate.h"
#include <stdbool.h>
#define PYDBG_FUNCS \
_PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
+static PyMemAllocatorEx _PyMem_Raw = {
+#ifdef Py_DEBUG
+ &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+#else
+ NULL, PYRAW_FUNCS
+#endif
+ };
-#define _PyMem_Raw _PyRuntime.mem.allocators.raw
-
-#define _PyMem _PyRuntime.mem.allocators.mem
+static PyMemAllocatorEx _PyMem = {
+#ifdef Py_DEBUG
+ &_PyMem_Debug.mem, PYDBG_FUNCS
+#else
+ NULL, PYMEM_FUNCS
+#endif
+ };
-#define _PyObject _PyRuntime.mem.allocators.obj
+static PyMemAllocatorEx _PyObject = {
+#ifdef Py_DEBUG
+ &_PyMem_Debug.obj, PYDBG_FUNCS
+#else
+ NULL, PYOBJ_FUNCS
+#endif
+ };
void
_PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p)
{
- PyMemAllocatorEx pymem_raw = {
#ifdef Py_DEBUG
- &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+ PyMemAllocatorEx alloc = {&_PyMem_Debug.raw, PYDBG_FUNCS};
#else
- NULL, PYRAW_FUNCS
+ PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS};
#endif
- };
- *alloc_p = pymem_raw;
+ *alloc_p = alloc;
}
int
return 0;
}
+#undef PYRAW_FUNCS
+#undef PYMEM_FUNCS
+#undef PYOBJ_FUNCS
+#undef PYRAWDBG_FUNCS
+#undef PYDBG_FUNCS
-void
-_PyObject_Initialize(struct _pyobj_runtime_state *state)
-{
- PyObjectArenaAllocator _PyObject_Arena = {NULL,
+static PyObjectArenaAllocator _PyObject_Arena = {NULL,
#ifdef MS_WINDOWS
- _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
+ _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
#elif defined(ARENAS_USE_MMAP)
- _PyObject_ArenaMmap, _PyObject_ArenaMunmap
+ _PyObject_ArenaMmap, _PyObject_ArenaMunmap
#else
- _PyObject_ArenaMalloc, _PyObject_ArenaFree
+ _PyObject_ArenaMalloc, _PyObject_ArenaFree
#endif
};
- state->allocator_arenas = _PyObject_Arena;
-}
-
-
-void
-_PyMem_Initialize(struct _pymem_runtime_state *state)
-{
- PyMemAllocatorEx pymem = {
-#ifdef Py_DEBUG
- &_PyMem_Debug.mem, PYDBG_FUNCS
-#else
- NULL, PYMEM_FUNCS
-#endif
- };
- PyMemAllocatorEx pyobject = {
-#ifdef Py_DEBUG
- &_PyMem_Debug.obj, PYDBG_FUNCS
-#else
- NULL, PYOBJ_FUNCS
-#endif
- };
-
- _PyMem_GetDefaultRawAllocator(&state->allocators.raw);
- state->allocators.mem = pymem;
- state->allocators.obj = pyobject;
-
-#ifdef WITH_PYMALLOC
- Py_BUILD_ASSERT(NB_SMALL_SIZE_CLASSES == 64);
-
- for (int i = 0; i < 8; i++) {
- for (int j = 0; j < 8; j++) {
- int x = i * 8 + j;
- poolp *addr = &(state->usedpools[2*(x)]);
- poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *));
- state->usedpools[x * 2] = val;
- state->usedpools[x * 2 + 1] = val;
- };
- };
-#endif /* WITH_PYMALLOC */
-}
-
-
#ifdef WITH_PYMALLOC
static int
_PyMem_DebugEnabled(void)
void
PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
{
- *allocator = _PyRuntime.obj.allocator_arenas;
+ *allocator = _PyObject_Arena;
}
void
PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
{
- _PyRuntime.obj.allocator_arenas = *allocator;
+ _PyObject_Arena = *allocator;
}
void *
return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
}
-void
-PyMem_RawFree(void *ptr)
+void PyMem_RawFree(void *ptr)
{
_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
}
#endif
+/* An object allocator for Python.
+
+ Here is an introduction to the layers of the Python memory architecture,
+ showing where the object allocator is actually used (layer +2), It is
+ called for every object allocation and deallocation (PyObject_New/Del),
+ unless the object-specific allocators implement a proprietary allocation
+ scheme (ex.: ints use a simple free list). This is also the place where
+ the cyclic garbage collector operates selectively on container objects.
+
+
+ Object-specific allocators
+ _____ ______ ______ ________
+ [ int ] [ dict ] [ list ] ... [ string ] Python core |
++3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
+ _______________________________ | |
+ [ Python's object allocator ] | |
++2 | ####### Object memory ####### | <------ Internal buffers ------> |
+ ______________________________________________________________ |
+ [ Python's raw memory allocator (PyMem_ API) ] |
++1 | <----- Python memory (under PyMem manager's control) ------> | |
+ __________________________________________________________________
+ [ Underlying general-purpose allocator (ex: C library malloc) ]
+ 0 | <------ Virtual memory allocated for the python process -------> |
+
+ =========================================================================
+ _______________________________________________________________________
+ [ OS-specific Virtual Memory Manager (VMM) ]
+-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
+ __________________________________ __________________________________
+ [ ] [ ]
+-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
+
+*/
+/*==========================================================================*/
+
+/* A fast, special-purpose memory allocator for small blocks, to be used
+ on top of a general-purpose malloc -- heavily based on previous art. */
+
+/* Vladimir Marangozov -- August 2000 */
+
+/*
+ * "Memory management is where the rubber meets the road -- if we do the wrong
+ * thing at any level, the results will not be good. And if we don't make the
+ * levels work well together, we are in serious trouble." (1)
+ *
+ * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
+ * "Dynamic Storage Allocation: A Survey and Critical Review",
+ * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
+ */
+
+/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */
+
+/*==========================================================================*/
+
+/*
+ * Allocation strategy abstract:
+ *
+ * For small requests, the allocator sub-allocates <Big> blocks of memory.
+ * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the
+ * system's allocator.
+ *
+ * Small requests are grouped in size classes spaced 8 bytes apart, due
+ * to the required valid alignment of the returned address. Requests of
+ * a particular size are serviced from memory pools of 4K (one VMM page).
+ * Pools are fragmented on demand and contain free lists of blocks of one
+ * particular size class. In other words, there is a fixed-size allocator
+ * for each size class. Free pools are shared by the different allocators
+ * thus minimizing the space reserved for a particular size class.
+ *
+ * This allocation strategy is a variant of what is known as "simple
+ * segregated storage based on array of free lists". The main drawback of
+ * simple segregated storage is that we might end up with lot of reserved
+ * memory for the different free lists, which degenerate in time. To avoid
+ * this, we partition each free list in pools and we share dynamically the
+ * reserved space between all free lists. This technique is quite efficient
+ * for memory intensive programs which allocate mainly small-sized blocks.
+ *
+ * For small requests we have the following table:
+ *
+ * Request in bytes Size of allocated block Size class idx
+ * ----------------------------------------------------------------
+ * 1-8 8 0
+ * 9-16 16 1
+ * 17-24 24 2
+ * 25-32 32 3
+ * 33-40 40 4
+ * 41-48 48 5
+ * 49-56 56 6
+ * 57-64 64 7
+ * 65-72 72 8
+ * ... ... ...
+ * 497-504 504 62
+ * 505-512 512 63
+ *
+ * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying
+ * allocator.
+ */
+
+/*==========================================================================*/
+
+/*
+ * -- Main tunable settings section --
+ */
+
+/*
+ * Alignment of addresses returned to the user. 8-bytes alignment works
+ * on most current architectures (with 32-bit or 64-bit address busses).
+ * The alignment value is also used for grouping small requests in size
+ * classes spaced ALIGNMENT bytes apart.
+ *
+ * You shouldn't change this unless you know what you are doing.
+ */
+#define ALIGNMENT 8 /* must be 2^N */
+#define ALIGNMENT_SHIFT 3
+
+/* Return the number of bytes in size class I, as a uint. */
+#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)
+
+/*
+ * Max size threshold below which malloc requests are considered to be
+ * small enough in order to use preallocated memory pools. You can tune
+ * this value according to your application behaviour and memory needs.
+ *
+ * Note: a size threshold of 512 guarantees that newly created dictionaries
+ * will be allocated from preallocated memory pools on 64-bit.
+ *
+ * The following invariants must hold:
+ * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512
+ * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT
+ *
+ * Although not required, for better performance and space efficiency,
+ * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.
+ */
+#define SMALL_REQUEST_THRESHOLD 512
+#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT)
+
+/*
+ * The system's VMM page size can be obtained on most unices with a
+ * getpagesize() call or deduced from various header files. To make
+ * things simpler, we assume that it is 4K, which is OK for most systems.
+ * It is probably better if this is the native page size, but it doesn't
+ * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page
+ * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation
+ * violation fault. 4K is apparently OK for all the platforms that python
+ * currently targets.
+ */
+#define SYSTEM_PAGE_SIZE (4 * 1024)
+#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1)
+
+/*
+ * Maximum amount of memory managed by the allocator for small requests.
+ */
+#ifdef WITH_MEMORY_LIMITS
+#ifndef SMALL_MEMORY_LIMIT
+#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */
+#endif
+#endif
+
+/*
+ * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
+ * on a page boundary. This is a reserved virtual address space for the
+ * current process (obtained through a malloc()/mmap() call). In no way this
+ * means that the memory arenas will be used entirely. A malloc(<Big>) is
+ * usually an address range reservation for <Big> bytes, unless all pages within
+ * this space are referenced subsequently. So malloc'ing big blocks and not
+ * using them does not mean "wasting memory". It's an addressable range
+ * wastage...
+ *
+ * Arenas are allocated with mmap() on systems supporting anonymous memory
+ * mappings to reduce heap fragmentation.
+ */
+#define ARENA_SIZE (256 << 10) /* 256KB */
+
+#ifdef WITH_MEMORY_LIMITS
+#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE)
+#endif
+
+/*
+ * Size of the pools used for small blocks. Should be a power of 2,
+ * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
+ */
+#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */
+#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK
+
+/*
+ * -- End of tunable settings section --
+ */
+
+/*==========================================================================*/
+
+/*
+ * Locking
+ *
+ * To reduce lock contention, it would probably be better to refine the
+ * crude function locking with per size class locking. I'm not positive
+ * however, whether it's worth switching to such locking policy because
+ * of the performance penalty it might introduce.
+ *
+ * The following macros describe the simplest (should also be the fastest)
+ * lock object on a particular platform and the init/fini/lock/unlock
+ * operations on it. The locks defined here are not expected to be recursive
+ * because it is assumed that they will always be called in the order:
+ * INIT, [LOCK, UNLOCK]*, FINI.
+ */
+
+/*
+ * Python's threads are serialized, so object malloc locking is disabled.
+ */
+#define SIMPLELOCK_DECL(lock) /* simple lock declaration */
+#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */
+#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */
+#define SIMPLELOCK_LOCK(lock) /* acquire released lock */
+#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */
+
+/* When you say memory, my mind reasons in terms of (pointers to) blocks */
+typedef uint8_t block;
+
+/* Pool for small blocks. */
+struct pool_header {
+ union { block *_padding;
+ uint count; } ref; /* number of allocated blocks */
+ block *freeblock; /* pool's free list head */
+ struct pool_header *nextpool; /* next pool of this size class */
+ struct pool_header *prevpool; /* previous pool "" */
+ uint arenaindex; /* index into arenas of base adr */
+ uint szidx; /* block size class index */
+ uint nextoffset; /* bytes to virgin block */
+ uint maxnextoffset; /* largest valid nextoffset */
+};
+
+typedef struct pool_header *poolp;
+
+/* Record keeping for arenas. */
+struct arena_object {
+ /* The address of the arena, as returned by malloc. Note that 0
+ * will never be returned by a successful malloc, and is used
+ * here to mark an arena_object that doesn't correspond to an
+ * allocated arena.
+ */
+ uintptr_t address;
+
+ /* Pool-aligned pointer to the next pool to be carved off. */
+ block* pool_address;
+
+ /* The number of available pools in the arena: free pools + never-
+ * allocated pools.
+ */
+ uint nfreepools;
+
+ /* The total number of pools in the arena, whether or not available. */
+ uint ntotalpools;
+
+ /* Singly-linked list of available pools. */
+ struct pool_header* freepools;
+
+ /* Whenever this arena_object is not associated with an allocated
+ * arena, the nextarena member is used to link all unassociated
+ * arena_objects in the singly-linked `unused_arena_objects` list.
+ * The prevarena member is unused in this case.
+ *
+ * When this arena_object is associated with an allocated arena
+ * with at least one available pool, both members are used in the
+ * doubly-linked `usable_arenas` list, which is maintained in
+ * increasing order of `nfreepools` values.
+ *
+ * Else this arena_object is associated with an allocated arena
+ * all of whose pools are in use. `nextarena` and `prevarena`
+ * are both meaningless in this case.
+ */
+ struct arena_object* nextarena;
+ struct arena_object* prevarena;
+};
+
+#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT)
+
+#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */
+
+/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */
+#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE))
+
+/* Return total number of blocks in pool of size index I, as a uint. */
+#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I))
+
+/*==========================================================================*/
+
+/*
+ * This malloc lock
+ */
+SIMPLELOCK_DECL(_malloc_lock)
+#define LOCK() SIMPLELOCK_LOCK(_malloc_lock)
+#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock)
+#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock)
+#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock)
+
+/*
+ * Pool table -- headed, circular, doubly-linked lists of partially used pools.
+
+This is involved. For an index i, usedpools[i+i] is the header for a list of
+all partially used pools holding small blocks with "size class idx" i. So
+usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size
+16, and so on: index 2*i <-> blocks of size (i+1)<<ALIGNMENT_SHIFT.
+
+Pools are carved off an arena's highwater mark (an arena_object's pool_address
+member) as needed. Once carved off, a pool is in one of three states forever
+after:
+
+used == partially used, neither empty nor full
+ At least one block in the pool is currently allocated, and at least one
+ block in the pool is not currently allocated (note this implies a pool
+ has room for at least two blocks).
+ This is a pool's initial state, as a pool is created only when malloc
+ needs space.
+ The pool holds blocks of a fixed size, and is in the circular list headed
+ at usedpools[i] (see above). It's linked to the other used pools of the
+ same size class via the pool_header's nextpool and prevpool members.
+ If all but one block is currently allocated, a malloc can cause a
+ transition to the full state. If all but one block is not currently
+ allocated, a free can cause a transition to the empty state.
+
+full == all the pool's blocks are currently allocated
+ On transition to full, a pool is unlinked from its usedpools[] list.
+ It's not linked to from anything then anymore, and its nextpool and
+ prevpool members are meaningless until it transitions back to used.
+ A free of a block in a full pool puts the pool back in the used state.
+ Then it's linked in at the front of the appropriate usedpools[] list, so
+ that the next allocation for its size class will reuse the freed block.
+
+empty == all the pool's blocks are currently available for allocation
+ On transition to empty, a pool is unlinked from its usedpools[] list,
+ and linked to the front of its arena_object's singly-linked freepools list,
+ via its nextpool member. The prevpool member has no meaning in this case.
+ Empty pools have no inherent size class: the next time a malloc finds
+ an empty list in usedpools[], it takes the first pool off of freepools.
+ If the size class needed happens to be the same as the size class the pool
+ last had, some pool initialization can be skipped.
+
+
+Block Management
+
+Blocks within pools are again carved out as needed. pool->freeblock points to
+the start of a singly-linked list of free blocks within the pool. When a
+block is freed, it's inserted at the front of its pool's freeblock list. Note
+that the available blocks in a pool are *not* linked all together when a pool
+is initialized. Instead only "the first two" (lowest addresses) blocks are
+set up, returning the first such block, and setting pool->freeblock to a
+one-block list holding the second such block. This is consistent with that
+pymalloc strives at all levels (arena, pool, and block) never to touch a piece
+of memory until it's actually needed.
+
+So long as a pool is in the used state, we're certain there *is* a block
+available for allocating, and pool->freeblock is not NULL. If pool->freeblock
+points to the end of the free list before we've carved the entire pool into
+blocks, that means we simply haven't yet gotten to one of the higher-address
+blocks. The offset from the pool_header to the start of "the next" virgin
+block is stored in the pool_header nextoffset member, and the largest value
+of nextoffset that makes sense is stored in the maxnextoffset member when a
+pool is initialized. All the blocks in a pool have been passed out at least
+once when and only when nextoffset > maxnextoffset.
+
+
+Major obscurity: While the usedpools vector is declared to have poolp
+entries, it doesn't really. It really contains two pointers per (conceptual)
+poolp entry, the nextpool and prevpool members of a pool_header. The
+excruciating initialization code below fools C so that
+
+ usedpool[i+i]
+
+"acts like" a genuine poolp, but only so long as you only reference its
+nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is
+compensating for that a pool_header's nextpool and prevpool members
+immediately follow a pool_header's first two members:
+
+ union { block *_padding;
+ uint count; } ref;
+ block *freeblock;
+
+each of which consume sizeof(block *) bytes. So what usedpools[i+i] really
+contains is a fudged-up pointer p such that *if* C believes it's a poolp
+pointer, then p->nextpool and p->prevpool are both p (meaning that the headed
+circular list is empty).
+
+It's unclear why the usedpools setup is so convoluted. It could be to
+minimize the amount of cache required to hold this heavily-referenced table
+(which only *needs* the two interpool pointer members of a pool_header). OTOH,
+referencing code has to remember to "double the index" and doing so isn't
+free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying
+on that C doesn't insert any padding anywhere in a pool_header at or before
+the prevpool member.
+**************************************************************************** */
+
+#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *)))
+#define PT(x) PTA(x), PTA(x)
+
+static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = {
+ PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7)
+#if NB_SMALL_SIZE_CLASSES > 8
+ , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15)
+#if NB_SMALL_SIZE_CLASSES > 16
+ , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23)
+#if NB_SMALL_SIZE_CLASSES > 24
+ , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31)
+#if NB_SMALL_SIZE_CLASSES > 32
+ , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39)
+#if NB_SMALL_SIZE_CLASSES > 40
+ , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47)
+#if NB_SMALL_SIZE_CLASSES > 48
+ , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55)
+#if NB_SMALL_SIZE_CLASSES > 56
+ , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63)
+#if NB_SMALL_SIZE_CLASSES > 64
+#error "NB_SMALL_SIZE_CLASSES should be less than 64"
+#endif /* NB_SMALL_SIZE_CLASSES > 64 */
+#endif /* NB_SMALL_SIZE_CLASSES > 56 */
+#endif /* NB_SMALL_SIZE_CLASSES > 48 */
+#endif /* NB_SMALL_SIZE_CLASSES > 40 */
+#endif /* NB_SMALL_SIZE_CLASSES > 32 */
+#endif /* NB_SMALL_SIZE_CLASSES > 24 */
+#endif /* NB_SMALL_SIZE_CLASSES > 16 */
+#endif /* NB_SMALL_SIZE_CLASSES > 8 */
+};
+
+/*==========================================================================
+Arena management.
+
+`arenas` is a vector of arena_objects. It contains maxarenas entries, some of
+which may not be currently used (== they're arena_objects that aren't
+currently associated with an allocated arena). Note that arenas proper are
+separately malloc'ed.
+
+Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5,
+we do try to free() arenas, and use some mild heuristic strategies to increase
+the likelihood that arenas eventually can be freed.
+
+unused_arena_objects
+
+ This is a singly-linked list of the arena_objects that are currently not
+ being used (no arena is associated with them). Objects are taken off the
+ head of the list in new_arena(), and are pushed on the head of the list in
+ PyObject_Free() when the arena is empty. Key invariant: an arena_object
+ is on this list if and only if its .address member is 0.
+
+usable_arenas
+
+ This is a doubly-linked list of the arena_objects associated with arenas
+ that have pools available. These pools are either waiting to be reused,
+ or have not been used before. The list is sorted to have the most-
+ allocated arenas first (ascending order based on the nfreepools member).
+ This means that the next allocation will come from a heavily used arena,
+ which gives the nearly empty arenas a chance to be returned to the system.
+ In my unscientific tests this dramatically improved the number of arenas
+ that could be freed.
+
+Note that an arena_object associated with an arena all of whose pools are
+currently in use isn't on either list.
+*/
+
+/* Array of objects used to track chunks of memory (arenas). */
+static struct arena_object* arenas = NULL;
+/* Number of slots currently allocated in the `arenas` vector. */
+static uint maxarenas = 0;
+
+/* The head of the singly-linked, NULL-terminated list of available
+ * arena_objects.
+ */
+static struct arena_object* unused_arena_objects = NULL;
+
+/* The head of the doubly-linked, NULL-terminated at each end, list of
+ * arena_objects associated with arenas that have pools available.
+ */
+static struct arena_object* usable_arenas = NULL;
+
+/* How many arena_objects do we initially allocate?
+ * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
+ * `arenas` vector.
+ */
+#define INITIAL_ARENA_OBJECTS 16
+
+/* Number of arenas allocated that haven't been free()'d. */
+static size_t narenas_currently_allocated = 0;
+
+/* Total number of times malloc() called to allocate an arena. */
+static size_t ntimes_arena_allocated = 0;
+/* High water mark (max value ever seen) for narenas_currently_allocated. */
+static size_t narenas_highwater = 0;
+
+static Py_ssize_t _Py_AllocatedBlocks = 0;
+
Py_ssize_t
_Py_GetAllocatedBlocks(void)
{
- return _PyRuntime.mem.num_allocated_blocks;
+ return _Py_AllocatedBlocks;
}
if (debug_stats)
_PyObject_DebugMallocStats(stderr);
- if (_PyRuntime.mem.unused_arena_objects == NULL) {
+ if (unused_arena_objects == NULL) {
uint i;
uint numarenas;
size_t nbytes;
/* Double the number of arena objects on each allocation.
* Note that it's possible for `numarenas` to overflow.
*/
- numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS;
- if (numarenas <= _PyRuntime.mem.maxarenas)
+ numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;
+ if (numarenas <= maxarenas)
return NULL; /* overflow */
#if SIZEOF_SIZE_T <= SIZEOF_INT
- if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas))
+ if (numarenas > SIZE_MAX / sizeof(*arenas))
return NULL; /* overflow */
#endif
- nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas);
- arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes);
+ nbytes = numarenas * sizeof(*arenas);
+ arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes);
if (arenaobj == NULL)
return NULL;
- _PyRuntime.mem.arenas = arenaobj;
+ arenas = arenaobj;
/* We might need to fix pointers that were copied. However,
* new_arena only gets called when all the pages in the
* into the old array. Thus, we don't have to worry about
* invalid pointers. Just to be sure, some asserts:
*/
- assert(_PyRuntime.mem.usable_arenas == NULL);
- assert(_PyRuntime.mem.unused_arena_objects == NULL);
+ assert(usable_arenas == NULL);
+ assert(unused_arena_objects == NULL);
/* Put the new arenas on the unused_arena_objects list. */
- for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) {
- _PyRuntime.mem.arenas[i].address = 0; /* mark as unassociated */
- _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ?
- &_PyRuntime.mem.arenas[i+1] : NULL;
+ for (i = maxarenas; i < numarenas; ++i) {
+ arenas[i].address = 0; /* mark as unassociated */
+ arenas[i].nextarena = i < numarenas - 1 ?
+ &arenas[i+1] : NULL;
}
/* Update globals. */
- _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas];
- _PyRuntime.mem.maxarenas = numarenas;
+ unused_arena_objects = &arenas[maxarenas];
+ maxarenas = numarenas;
}
/* Take the next available arena object off the head of the list. */
- assert(_PyRuntime.mem.unused_arena_objects != NULL);
- arenaobj = _PyRuntime.mem.unused_arena_objects;
- _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena;
+ assert(unused_arena_objects != NULL);
+ arenaobj = unused_arena_objects;
+ unused_arena_objects = arenaobj->nextarena;
assert(arenaobj->address == 0);
- address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE);
+ address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
if (address == NULL) {
/* The allocation failed: return NULL after putting the
* arenaobj back.
*/
- arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects;
- _PyRuntime.mem.unused_arena_objects = arenaobj;
+ arenaobj->nextarena = unused_arena_objects;
+ unused_arena_objects = arenaobj;
return NULL;
}
arenaobj->address = (uintptr_t)address;
- ++_PyRuntime.mem.narenas_currently_allocated;
- ++_PyRuntime.mem.ntimes_arena_allocated;
- if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater)
- _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated;
+ ++narenas_currently_allocated;
+ ++ntimes_arena_allocated;
+ if (narenas_currently_allocated > narenas_highwater)
+ narenas_highwater = narenas_currently_allocated;
arenaobj->freepools = NULL;
/* pool_address <- first pool-aligned address in the arena
nfreepools <- number of whole pools that fit after alignment */
- arenaobj->pool_address = (pyblock*)arenaobj->address;
+ arenaobj->pool_address = (block*)arenaobj->address;
arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE;
assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE);
excess = (uint)(arenaobj->address & POOL_SIZE_MASK);
// the GIL. The following dance forces the compiler to read pool->arenaindex
// only once.
uint arenaindex = *((volatile uint *)&pool->arenaindex);
- return arenaindex < _PyRuntime.mem.maxarenas &&
- (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE &&
- _PyRuntime.mem.arenas[arenaindex].address != 0;
+ return arenaindex < maxarenas &&
+ (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE &&
+ arenas[arenaindex].address != 0;
}
static int
pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
{
- pyblock *bp;
+ block *bp;
poolp pool;
poolp next;
uint size;
* Most frequent paths first
*/
size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
- pool = _PyRuntime.mem.usedpools[size + size];
+ pool = usedpools[size + size];
if (pool != pool->nextpool) {
/*
* There is a used pool for this size class.
++pool->ref.count;
bp = pool->freeblock;
assert(bp != NULL);
- if ((pool->freeblock = *(pyblock **)bp) != NULL) {
+ if ((pool->freeblock = *(block **)bp) != NULL) {
goto success;
}
*/
if (pool->nextoffset <= pool->maxnextoffset) {
/* There is room for another block. */
- pool->freeblock = (pyblock*)pool +
+ pool->freeblock = (block*)pool +
pool->nextoffset;
pool->nextoffset += INDEX2SIZE(size);
- *(pyblock **)(pool->freeblock) = NULL;
+ *(block **)(pool->freeblock) = NULL;
goto success;
}
/* There isn't a pool of the right size class immediately
* available: use a free pool.
*/
- if (_PyRuntime.mem.usable_arenas == NULL) {
+ if (usable_arenas == NULL) {
/* No arena has a free pool: allocate a new arena. */
#ifdef WITH_MEMORY_LIMITS
- if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) {
+ if (narenas_currently_allocated >= MAX_ARENAS) {
goto failed;
}
#endif
- _PyRuntime.mem.usable_arenas = new_arena();
- if (_PyRuntime.mem.usable_arenas == NULL) {
+ usable_arenas = new_arena();
+ if (usable_arenas == NULL) {
goto failed;
}
- _PyRuntime.mem.usable_arenas->nextarena =
- _PyRuntime.mem.usable_arenas->prevarena = NULL;
+ usable_arenas->nextarena =
+ usable_arenas->prevarena = NULL;
}
- assert(_PyRuntime.mem.usable_arenas->address != 0);
+ assert(usable_arenas->address != 0);
/* Try to get a cached free pool. */
- pool = _PyRuntime.mem.usable_arenas->freepools;
+ pool = usable_arenas->freepools;
if (pool != NULL) {
/* Unlink from cached pools. */
- _PyRuntime.mem.usable_arenas->freepools = pool->nextpool;
+ usable_arenas->freepools = pool->nextpool;
/* This arena already had the smallest nfreepools
* value, so decreasing nfreepools doesn't change
* become wholly allocated, we need to remove its
* arena_object from usable_arenas.
*/
- --_PyRuntime.mem.usable_arenas->nfreepools;
- if (_PyRuntime.mem.usable_arenas->nfreepools == 0) {
+ --usable_arenas->nfreepools;
+ if (usable_arenas->nfreepools == 0) {
/* Wholly allocated: remove. */
- assert(_PyRuntime.mem.usable_arenas->freepools == NULL);
- assert(_PyRuntime.mem.usable_arenas->nextarena == NULL ||
- _PyRuntime.mem.usable_arenas->nextarena->prevarena ==
- _PyRuntime.mem.usable_arenas);
-
- _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena;
- if (_PyRuntime.mem.usable_arenas != NULL) {
- _PyRuntime.mem.usable_arenas->prevarena = NULL;
- assert(_PyRuntime.mem.usable_arenas->address != 0);
+ assert(usable_arenas->freepools == NULL);
+ assert(usable_arenas->nextarena == NULL ||
+ usable_arenas->nextarena->prevarena ==
+ usable_arenas);
+
+ usable_arenas = usable_arenas->nextarena;
+ if (usable_arenas != NULL) {
+ usable_arenas->prevarena = NULL;
+ assert(usable_arenas->address != 0);
}
}
else {
* off all the arena's pools for the first
* time.
*/
- assert(_PyRuntime.mem.usable_arenas->freepools != NULL ||
- _PyRuntime.mem.usable_arenas->pool_address <=
- (pyblock*)_PyRuntime.mem.usable_arenas->address +
+ assert(usable_arenas->freepools != NULL ||
+ usable_arenas->pool_address <=
+ (block*)usable_arenas->address +
ARENA_SIZE - POOL_SIZE);
}
init_pool:
/* Frontlink to used pools. */
- next = _PyRuntime.mem.usedpools[size + size]; /* == prev */
+ next = usedpools[size + size]; /* == prev */
pool->nextpool = next;
pool->prevpool = next;
next->nextpool = pool;
*/
bp = pool->freeblock;
assert(bp != NULL);
- pool->freeblock = *(pyblock **)bp;
+ pool->freeblock = *(block **)bp;
goto success;
}
/*
*/
pool->szidx = size;
size = INDEX2SIZE(size);
- bp = (pyblock *)pool + POOL_OVERHEAD;
+ bp = (block *)pool + POOL_OVERHEAD;
pool->nextoffset = POOL_OVERHEAD + (size << 1);
pool->maxnextoffset = POOL_SIZE - size;
pool->freeblock = bp + size;
- *(pyblock **)(pool->freeblock) = NULL;
+ *(block **)(pool->freeblock) = NULL;
goto success;
}
/* Carve off a new pool. */
- assert(_PyRuntime.mem.usable_arenas->nfreepools > 0);
- assert(_PyRuntime.mem.usable_arenas->freepools == NULL);
- pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address;
- assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address +
+ assert(usable_arenas->nfreepools > 0);
+ assert(usable_arenas->freepools == NULL);
+ pool = (poolp)usable_arenas->pool_address;
+ assert((block*)pool <= (block*)usable_arenas->address +
ARENA_SIZE - POOL_SIZE);
- pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas);
- assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas);
+ pool->arenaindex = (uint)(usable_arenas - arenas);
+ assert(&arenas[pool->arenaindex] == usable_arenas);
pool->szidx = DUMMY_SIZE_IDX;
- _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE;
- --_PyRuntime.mem.usable_arenas->nfreepools;
+ usable_arenas->pool_address += POOL_SIZE;
+ --usable_arenas->nfreepools;
- if (_PyRuntime.mem.usable_arenas->nfreepools == 0) {
- assert(_PyRuntime.mem.usable_arenas->nextarena == NULL ||
- _PyRuntime.mem.usable_arenas->nextarena->prevarena ==
- _PyRuntime.mem.usable_arenas);
+ if (usable_arenas->nfreepools == 0) {
+ assert(usable_arenas->nextarena == NULL ||
+ usable_arenas->nextarena->prevarena ==
+ usable_arenas);
/* Unlink the arena: it is completely allocated. */
- _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena;
- if (_PyRuntime.mem.usable_arenas != NULL) {
- _PyRuntime.mem.usable_arenas->prevarena = NULL;
- assert(_PyRuntime.mem.usable_arenas->address != 0);
+ usable_arenas = usable_arenas->nextarena;
+ if (usable_arenas != NULL) {
+ usable_arenas->prevarena = NULL;
+ assert(usable_arenas->address != 0);
}
}
{
void* ptr;
if (pymalloc_alloc(ctx, &ptr, nbytes)) {
- _PyRuntime.mem.num_allocated_blocks++;
+ _Py_AllocatedBlocks++;
return ptr;
}
ptr = PyMem_RawMalloc(nbytes);
if (ptr != NULL) {
- _PyRuntime.mem.num_allocated_blocks++;
+ _Py_AllocatedBlocks++;
}
return ptr;
}
if (pymalloc_alloc(ctx, &ptr, nbytes)) {
memset(ptr, 0, nbytes);
- _PyRuntime.mem.num_allocated_blocks++;
+ _Py_AllocatedBlocks++;
return ptr;
}
ptr = PyMem_RawCalloc(nelem, elsize);
if (ptr != NULL) {
- _PyRuntime.mem.num_allocated_blocks++;
+ _Py_AllocatedBlocks++;
}
return ptr;
}
pymalloc_free(void *ctx, void *p)
{
poolp pool;
- pyblock *lastfree;
+ block *lastfree;
poolp next, prev;
uint size;
* list in any case).
*/
assert(pool->ref.count > 0); /* else it was empty */
- *(pyblock **)p = lastfree = pool->freeblock;
- pool->freeblock = (pyblock *)p;
+ *(block **)p = lastfree = pool->freeblock;
+ pool->freeblock = (block *)p;
if (!lastfree) {
/* Pool was full, so doesn't currently live in any list:
* link it to the front of the appropriate usedpools[] list.
--pool->ref.count;
assert(pool->ref.count > 0); /* else the pool is empty */
size = pool->szidx;
- next = _PyRuntime.mem.usedpools[size + size];
+ next = usedpools[size + size];
prev = next->prevpool;
/* insert pool before next: prev <-> pool <-> next */
/* Link the pool to freepools. This is a singly-linked
* list, and pool->prevpool isn't used there.
*/
- ao = &_PyRuntime.mem.arenas[pool->arenaindex];
+ ao = &arenas[pool->arenaindex];
pool->nextpool = ao->freepools;
ao->freepools = pool;
nf = ++ao->nfreepools;
* usable_arenas pointer.
*/
if (ao->prevarena == NULL) {
- _PyRuntime.mem.usable_arenas = ao->nextarena;
- assert(_PyRuntime.mem.usable_arenas == NULL ||
- _PyRuntime.mem.usable_arenas->address != 0);
+ usable_arenas = ao->nextarena;
+ assert(usable_arenas == NULL ||
+ usable_arenas->address != 0);
}
else {
assert(ao->prevarena->nextarena == ao);
/* Record that this arena_object slot is
* available to be reused.
*/
- ao->nextarena = _PyRuntime.mem.unused_arena_objects;
- _PyRuntime.mem.unused_arena_objects = ao;
+ ao->nextarena = unused_arena_objects;
+ unused_arena_objects = ao;
/* Free the entire arena. */
- _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx,
+ _PyObject_Arena.free(_PyObject_Arena.ctx,
(void *)ao->address, ARENA_SIZE);
ao->address = 0; /* mark unassociated */
- --_PyRuntime.mem.narenas_currently_allocated;
+ --narenas_currently_allocated;
goto success;
}
* ao->nfreepools was 0 before, ao isn't
* currently on the usable_arenas list.
*/
- ao->nextarena = _PyRuntime.mem.usable_arenas;
+ ao->nextarena = usable_arenas;
ao->prevarena = NULL;
- if (_PyRuntime.mem.usable_arenas)
- _PyRuntime.mem.usable_arenas->prevarena = ao;
- _PyRuntime.mem.usable_arenas = ao;
- assert(_PyRuntime.mem.usable_arenas->address != 0);
+ if (usable_arenas)
+ usable_arenas->prevarena = ao;
+ usable_arenas = ao;
+ assert(usable_arenas->address != 0);
goto success;
}
}
else {
/* ao is at the head of the list */
- assert(_PyRuntime.mem.usable_arenas == ao);
- _PyRuntime.mem.usable_arenas = ao->nextarena;
+ assert(usable_arenas == ao);
+ usable_arenas = ao->nextarena;
}
ao->nextarena->prevarena = ao->prevarena;
assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools);
assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools);
assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao);
- assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL)
+ assert((usable_arenas == ao && ao->prevarena == NULL)
|| ao->prevarena->nextarena == ao);
goto success;
return;
}
- _PyRuntime.mem.num_allocated_blocks--;
+ _Py_AllocatedBlocks--;
if (!pymalloc_free(ctx, p)) {
/* pymalloc didn't allocate this address */
PyMem_RawFree(p);
#define DEADBYTE 0xDB /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */
+static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
+
/* serialno is always incremented via calling this routine. The point is
* to supply a single place to set a breakpoint.
*/
static void
bumpserialno(void)
{
- ++_PyRuntime.mem.serialno;
+ ++serialno;
}
#define SST SIZEOF_SIZE_T
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
- write_size_t(tail + SST, _PyRuntime.mem.serialno);
+ write_size_t(tail + SST, serialno);
return data;
}
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
size_t total; /* 2 * SST + nbytes + 2 * SST */
size_t original_nbytes;
- size_t serialno;
+ size_t block_serialno;
#define ERASED_SIZE 64
uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */
total = nbytes + 4*SST;
tail = data + original_nbytes;
- serialno = read_size_t(tail + SST);
+ block_serialno = read_size_t(tail + SST);
/* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
*/
else {
head = r;
bumpserialno();
- serialno = _PyRuntime.mem.serialno;
+ block_serialno = serialno;
}
write_size_t(head, nbytes);
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
- write_size_t(tail + SST, serialno);
+ write_size_t(tail + SST, block_serialno);
/* Restore saved bytes. */
if (original_nbytes <= sizeof(save)) {
* to march over all the arenas. If we're lucky, most of the memory
* will be living in full pools -- would be a shame to miss them.
*/
- for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) {
+ for (i = 0; i < maxarenas; ++i) {
uint j;
- uintptr_t base = _PyRuntime.mem.arenas[i].address;
+ uintptr_t base = arenas[i].address;
/* Skip arenas which are not allocated. */
- if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL)
+ if (arenas[i].address == (uintptr_t)NULL)
continue;
narenas += 1;
- numfreepools += _PyRuntime.mem.arenas[i].nfreepools;
+ numfreepools += arenas[i].nfreepools;
/* round up to pool alignment */
if (base & (uintptr_t)POOL_SIZE_MASK) {
}
/* visit every pool in the arena */
- assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address);
- for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address;
+ assert(base <= (uintptr_t) arenas[i].pool_address);
+ for (j = 0; base < (uintptr_t) arenas[i].pool_address;
++j, base += POOL_SIZE) {
poolp p = (poolp)base;
const uint sz = p->szidx;
if (p->ref.count == 0) {
/* currently unused */
#ifdef Py_DEBUG
- assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools));
+ assert(pool_is_in_list(p, arenas[i].freepools));
#endif
continue;
}
numfreeblocks[sz] += freeblocks;
#ifdef Py_DEBUG
if (freeblocks > 0)
- assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz]));
+ assert(pool_is_in_list(p, usedpools[sz + sz]));
#endif
}
}
- assert(narenas == _PyRuntime.mem.narenas_currently_allocated);
+ assert(narenas == narenas_currently_allocated);
fputc('\n', out);
fputs("class size num pools blocks in use avail blocks\n"
}
fputc('\n', out);
if (_PyMem_DebugEnabled())
- (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno);
- (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated);
- (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas);
- (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater);
+ (void)printone(out, "# times object malloc called", serialno);
+ (void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
+ (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
+ (void)printone(out, "# arenas highwater mark", narenas_highwater);
(void)printone(out, "# arenas allocated current", narenas);
PyOS_snprintf(buf, sizeof(buf),
projects / thirdparty / Python / cpython.git / commitdiff
