char data[0];
};
-/** List of free memory blocks */
-static LIST_HEAD ( free_blocks );
-
-/** Total amount of free memory */
-size_t freemem;
-
-/** Total amount of used memory */
-size_t usedmem;
-
-/** Maximum amount of used memory */
-size_t maxusedmem;
-
/**
* Heap size
*
*/
#define HEAP_SIZE ( 512 * 1024 )
-/** The heap itself */
-static char heap[HEAP_SIZE];
+/** The heap area */
+static char heap_area[HEAP_SIZE];
/**
* Mark all blocks in free list as defined
*
+ * @v heap Heap
*/
-static inline void valgrind_make_blocks_defined ( void ) {
+static inline void valgrind_make_blocks_defined ( struct heap *heap ) {
struct memory_block *block;
/* Do nothing unless running under Valgrind */
*/
/* Mark block list itself as defined */
- VALGRIND_MAKE_MEM_DEFINED ( &free_blocks, sizeof ( free_blocks ) );
+ VALGRIND_MAKE_MEM_DEFINED ( &heap->blocks, sizeof ( heap->blocks ) );
/* Mark areas accessed by list_check() as defined */
- VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.prev->next,
- sizeof ( free_blocks.prev->next ) );
- VALGRIND_MAKE_MEM_DEFINED ( free_blocks.next,
- sizeof ( *free_blocks.next ) );
- VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.next->next->prev,
- sizeof ( free_blocks.next->next->prev ) );
+ VALGRIND_MAKE_MEM_DEFINED ( &heap->blocks.prev->next,
+ sizeof ( heap->blocks.prev->next ) );
+ VALGRIND_MAKE_MEM_DEFINED ( heap->blocks.next,
+ sizeof ( *heap->blocks.next ) );
+ VALGRIND_MAKE_MEM_DEFINED ( &heap->blocks.next->next->prev,
+ sizeof ( heap->blocks.next->next->prev ) );
/* Mark each block in list as defined */
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
/* Mark block as defined */
VALGRIND_MAKE_MEM_DEFINED ( block, sizeof ( *block ) );
/**
* Mark all blocks in free list as inaccessible
*
+ * @v heap Heap
*/
-static inline void valgrind_make_blocks_noaccess ( void ) {
+static inline void valgrind_make_blocks_noaccess ( struct heap *heap ) {
struct memory_block *block;
struct memory_block *prev = NULL;
*/
/* Mark each block in list as inaccessible */
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
/* Mark previous block (if any) as inaccessible. (Current
* block will be accessed by list_check().)
* accessing the first list item. Temporarily mark
* this area as defined.
*/
- VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.next->prev,
- sizeof ( free_blocks.next->prev ) );
+ VALGRIND_MAKE_MEM_DEFINED ( &heap->blocks.next->prev,
+ sizeof ( heap->blocks.next->prev ));
}
/* Mark last block (if any) as inaccessible */
if ( prev )
/* Mark as inaccessible the area that was temporarily marked
* as defined to avoid errors from list_check().
*/
- VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks.next->prev,
- sizeof ( free_blocks.next->prev ) );
+ VALGRIND_MAKE_MEM_NOACCESS ( &heap->blocks.next->prev,
+ sizeof ( heap->blocks.next->prev ) );
/* Mark block list itself as inaccessible */
- VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks, sizeof ( free_blocks ) );
+ VALGRIND_MAKE_MEM_NOACCESS ( &heap->blocks, sizeof ( heap->blocks ) );
}
/**
* Check integrity of the blocks in the free list
*
+ * @v heap Heap
*/
-static inline void check_blocks ( void ) {
+static inline void check_blocks ( struct heap *heap ) {
struct memory_block *block;
struct memory_block *prev = NULL;
if ( ! ASSERTING )
return;
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
/* Check alignment */
assert ( ( virt_to_phys ( block ) &
/**
* Discard some cached data
*
+ * @v size Failed allocation size
* @ret discarded Number of cached items discarded
*/
-static unsigned int discard_cache ( void ) {
+static unsigned int discard_cache ( size_t size __unused ) {
struct cache_discarder *discarder;
unsigned int discarded;
unsigned int discarded;
do {
- discarded = discard_cache();
+ discarded = discard_cache ( 0 );
} while ( discarded );
}
/**
* Allocate a memory block
*
+ * @v heap Heap
* @v size Requested size
* @v align Physical alignment
* @v offset Offset from physical alignment
*
* @c align must be a power of two. @c size may not be zero.
*/
-void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
+static void * heap_alloc_block ( struct heap *heap, size_t size, size_t align,
+ size_t offset ) {
struct memory_block *block;
size_t actual_offset;
size_t align_mask;
size_t post_size;
struct memory_block *pre;
struct memory_block *post;
- unsigned int discarded;
+ unsigned int grown;
void *ptr;
/* Sanity checks */
assert ( size != 0 );
assert ( ( align == 0 ) || ( ( align & ( align - 1 ) ) == 0 ) );
- valgrind_make_blocks_defined();
- check_blocks();
+ valgrind_make_blocks_defined ( heap );
+ check_blocks ( heap );
/* Calculate offset of memory block */
actual_offset = ( offset & ~( MEMBLOCK_ALIGN - 1 ) );
/* Calculate alignment mask */
align_mask = ( ( align - 1 ) | ( MEMBLOCK_ALIGN - 1 ) );
- DBGC2 ( &heap, "HEAP allocating %#zx (aligned %#zx+%zx)\n",
+ DBGC2 ( heap, "HEAP allocating %#zx (aligned %#zx+%#zx)\n",
size, align, offset );
while ( 1 ) {
/* Search through blocks for the first one with enough space */
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
pre_size = ( ( actual_offset - virt_to_phys ( block ) )
& align_mask );
if ( ( block->size < pre_size ) ||
pre = block;
block = ( ( ( void * ) pre ) + pre_size );
post = ( ( ( void * ) block ) + actual_size );
- DBGC2 ( &heap, "HEAP splitting [%p,%p) -> [%p,%p) "
+ DBGC2 ( heap, "HEAP splitting [%p,%p) -> [%p,%p) "
"+ [%p,%p)\n", pre,
( ( ( void * ) pre ) + pre->size ), pre, block,
post, ( ( ( void * ) pre ) + pre->size ) );
assert ( pre_size >= MEMBLOCK_ALIGN );
}
/* Update memory usage statistics */
- freemem -= actual_size;
- usedmem += actual_size;
- if ( usedmem > maxusedmem )
- maxusedmem = usedmem;
+ heap->freemem -= actual_size;
+ heap->usedmem += actual_size;
+ if ( heap->usedmem > heap->maxusedmem )
+ heap->maxusedmem = heap->usedmem;
/* Return allocated block */
ptr = ( ( ( void * ) block ) + offset - actual_offset );
- DBGC2 ( &heap, "HEAP allocated [%p,%p) within "
+ DBGC2 ( heap, "HEAP allocated [%p,%p) within "
"[%p,%p)\n", ptr, ( ptr + size ), block,
( ( ( void * ) block ) + actual_size ) );
VALGRIND_MAKE_MEM_UNDEFINED ( ptr, size );
goto done;
}
- /* Try discarding some cached data to free up memory */
- DBGC ( &heap, "HEAP attempting discard for %#zx (aligned "
+ /* Attempt to grow heap to satisfy allocation */
+ DBGC ( heap, "HEAP attempting to grow for %#zx (aligned "
"%#zx+%zx), used %zdkB\n", size, align, offset,
- ( usedmem >> 10 ) );
- valgrind_make_blocks_noaccess();
- discarded = discard_cache();
- valgrind_make_blocks_defined();
- check_blocks();
- if ( ! discarded ) {
- /* Nothing available to discard */
- DBGC ( &heap, "HEAP failed to allocate %#zx (aligned "
+ ( heap->usedmem >> 10 ) );
+ valgrind_make_blocks_noaccess ( heap );
+ grown = ( heap->grow ? heap->grow ( actual_size ) : 0 );
+ valgrind_make_blocks_defined ( heap );
+ check_blocks ( heap );
+ if ( ! grown ) {
+ /* Heap did not grow: fail allocation */
+ DBGC ( heap, "HEAP failed to allocate %#zx (aligned "
"%#zx)\n", size, align );
ptr = NULL;
goto done;
}
done:
- check_blocks();
- valgrind_make_blocks_noaccess();
+ check_blocks ( heap );
+ valgrind_make_blocks_noaccess ( heap );
return ptr;
}
/**
* Free a memory block
*
- * @v ptr Memory allocated by alloc_memblock(), or NULL
+ * @v heap Heap
+ * @v ptr Memory allocated by heap_alloc_block(), or NULL
* @v size Size of the memory
*
* If @c ptr is NULL, no action is taken.
*/
-void free_memblock ( void *ptr, size_t size ) {
+static void heap_free_block ( struct heap *heap, void *ptr, size_t size ) {
struct memory_block *freeing;
struct memory_block *block;
struct memory_block *tmp;
VALGRIND_MAKE_MEM_NOACCESS ( ptr, size );
/* Sanity checks */
- valgrind_make_blocks_defined();
- check_blocks();
+ valgrind_make_blocks_defined ( heap );
+ check_blocks ( heap );
- /* Round up to match actual block that alloc_memblock() would
+ /* Round up to match actual block that heap_alloc_block() would
* have allocated.
*/
assert ( size != 0 );
freeing = ( ptr - sub_offset );
actual_size = ( ( size + sub_offset + MEMBLOCK_ALIGN - 1 ) &
~( MEMBLOCK_ALIGN - 1 ) );
- DBGC2 ( &heap, "HEAP freeing [%p,%p) within [%p,%p)\n",
+ DBGC2 ( heap, "HEAP freeing [%p,%p) within [%p,%p)\n",
ptr, ( ptr + size ), freeing,
( ( ( void * ) freeing ) + actual_size ) );
VALGRIND_MAKE_MEM_UNDEFINED ( freeing, sizeof ( *freeing ) );
/* Check that this block does not overlap the free list */
if ( ASSERTING ) {
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
if ( ( ( ( void * ) block ) <
( ( void * ) freeing + actual_size ) ) &&
( ( void * ) freeing <
( ( void * ) block + block->size ) ) ) {
assert ( 0 );
- DBGC ( &heap, "HEAP double free of [%p,%p) "
+ DBGC ( heap, "HEAP double free of [%p,%p) "
"overlapping [%p,%p) detected from %p\n",
freeing,
( ( ( void * ) freeing ) + size ), block,
/* Insert/merge into free list */
freeing->size = actual_size;
- list_for_each_entry_safe ( block, tmp, &free_blocks, list ) {
+ list_for_each_entry_safe ( block, tmp, &heap->blocks, list ) {
/* Calculate gaps before and after the "freeing" block */
gap_before = ( ( ( void * ) freeing ) -
( ( ( void * ) block ) + block->size ) );
( ( ( void * ) freeing ) + freeing->size ) );
/* Merge with immediately preceding block, if possible */
if ( gap_before == 0 ) {
- DBGC2 ( &heap, "HEAP merging [%p,%p) + [%p,%p) -> "
+ DBGC2 ( heap, "HEAP merging [%p,%p) + [%p,%p) -> "
"[%p,%p)\n", block,
( ( ( void * ) block ) + block->size ), freeing,
( ( ( void * ) freeing ) + freeing->size ),
* possible, merge the following block into the "freeing"
* block.
*/
- DBGC2 ( &heap, "HEAP freed [%p,%p)\n",
+ DBGC2 ( heap, "HEAP freed [%p,%p)\n",
freeing, ( ( ( void * ) freeing ) + freeing->size ) );
list_add_tail ( &freeing->list, &block->list );
if ( gap_after == 0 ) {
- DBGC2 ( &heap, "HEAP merging [%p,%p) + [%p,%p) -> [%p,%p)\n",
+ DBGC2 ( heap, "HEAP merging [%p,%p) + [%p,%p) -> [%p,%p)\n",
freeing, ( ( ( void * ) freeing ) + freeing->size ),
block, ( ( ( void * ) block ) + block->size ), freeing,
( ( ( void * ) block ) + block->size ) );
}
/* Update memory usage statistics */
- freemem += actual_size;
- usedmem -= actual_size;
+ heap->freemem += actual_size;
+ heap->usedmem -= actual_size;
+
+ /* Allow heap to shrink */
+ if ( heap->shrink && heap->shrink ( freeing, freeing->size ) ) {
+ list_del ( &freeing->list );
+ heap->freemem -= freeing->size;
+ VALGRIND_MAKE_MEM_UNDEFINED ( freeing, freeing->size );
+ }
- check_blocks();
- valgrind_make_blocks_noaccess();
+ /* Sanity checks */
+ check_blocks ( heap );
+ valgrind_make_blocks_noaccess ( heap );
}
/**
* Reallocate memory
*
- * @v old_ptr Memory previously allocated by malloc(), or NULL
+ * @v heap Heap
+ * @v old_ptr Memory previously allocated by heap_realloc(), or NULL
* @v new_size Requested size
* @ret new_ptr Allocated memory, or NULL
*
* If allocation fails the previously allocated block is left
* untouched and NULL is returned.
*
- * Calling realloc() with a new size of zero is a valid way to free a
- * memory block.
+ * Calling heap_realloc() with a new size of zero is a valid way to
+ * free a memory block.
*/
-void * realloc ( void *old_ptr, size_t new_size ) {
+void * heap_realloc ( struct heap *heap, void *old_ptr, size_t new_size ) {
struct autosized_block *old_block;
struct autosized_block *new_block;
size_t old_total_size;
size_t new_total_size;
size_t old_size;
+ size_t offset = offsetof ( struct autosized_block, data );
void *new_ptr = NOWHERE;
/* Allocate new memory if necessary. If allocation fails,
* return without touching the old block.
*/
if ( new_size ) {
- new_total_size = ( new_size +
- offsetof ( struct autosized_block, data ) );
+ new_total_size = ( new_size + offset );
if ( new_total_size < new_size )
return NULL;
- new_block = alloc_memblock ( new_total_size, 1, 0 );
+ new_block = heap_alloc_block ( heap, new_total_size, 1, 0 );
if ( ! new_block )
return NULL;
new_block->size = new_total_size;
sizeof ( old_block->size ) );
old_total_size = old_block->size;
assert ( old_total_size != 0 );
- old_size = ( old_total_size -
- offsetof ( struct autosized_block, data ) );
+ old_size = ( old_total_size - offset );
memcpy ( new_ptr, old_ptr,
( ( old_size < new_size ) ? old_size : new_size ) );
VALGRIND_FREELIKE_BLOCK ( old_ptr, 0 );
- free_memblock ( old_block, old_total_size );
+ heap_free_block ( heap, old_block, old_total_size );
}
if ( ASSERTED ) {
- DBGC ( &heap, "HEAP detected possible memory corruption "
+ DBGC ( heap, "HEAP detected possible memory corruption "
"from %p\n", __builtin_return_address ( 0 ) );
}
return new_ptr;
}
+/** The global heap */
+static struct heap heap = {
+ .blocks = LIST_HEAD_INIT ( heap.blocks ),
+ .grow = discard_cache,
+};
+
+/**
+ * Reallocate memory
+ *
+ * @v old_ptr Memory previously allocated by malloc(), or NULL
+ * @v new_size Requested size
+ * @ret new_ptr Allocated memory, or NULL
+ */
+void * realloc ( void *old_ptr, size_t new_size ) {
+
+ return heap_realloc ( &heap, old_ptr, new_size );
+}
+
/**
* Allocate memory
*
return data;
}
+/**
+ * Allocate memory with specified physical alignment and offset
+ *
+ * @v size Requested size
+ * @v align Physical alignment
+ * @v offset Offset from physical alignment
+ * @ret ptr Memory, or NULL
+ *
+ * @c align must be a power of two. @c size may not be zero.
+ */
+void * malloc_phys_offset ( size_t size, size_t phys_align, size_t offset ) {
+ void * ptr;
+
+ ptr = heap_alloc_block ( &heap, size, phys_align, offset );
+ if ( ptr && size ) {
+ assert ( ( phys_align == 0 ) ||
+ ( ( ( virt_to_phys ( ptr ) ^ offset ) &
+ ( phys_align - 1 ) ) == 0 ) );
+ VALGRIND_MALLOCLIKE_BLOCK ( ptr, size, 0, 0 );
+ }
+ return ptr;
+}
+
+/**
+ * Allocate memory with specified physical alignment
+ *
+ * @v size Requested size
+ * @v align Physical alignment
+ * @ret ptr Memory, or NULL
+ *
+ * @c align must be a power of two. @c size may not be zero.
+ */
+void * malloc_phys ( size_t size, size_t phys_align ) {
+
+ return malloc_phys_offset ( size, phys_align, 0 );
+}
+
+/**
+ * Free memory allocated with malloc_phys()
+ *
+ * @v ptr Memory allocated by malloc_phys(), or NULL
+ * @v size Size of memory, as passed to malloc_phys()
+ *
+ * Memory allocated with malloc_phys() can only be freed with
+ * free_phys(); it cannot be freed with the standard free().
+ *
+ * If @c ptr is NULL, no action is taken.
+ */
+void free_phys ( void *ptr, size_t size ) {
+
+ VALGRIND_FREELIKE_BLOCK ( ptr, 0 );
+ heap_free_block ( &heap, ptr, size );
+}
+
/**
* Add memory to allocation pool
*
+ * @v heap Heap
* @v start Start address
* @v len Length of memory
*
- * Adds a block of memory to the allocation pool. This is a one-way
- * operation; there is no way to reclaim this memory.
+ * Adds a block of memory to the allocation pool.
*/
-static void mpopulate ( void *start, size_t len ) {
+void heap_populate ( struct heap *heap, void *start, size_t len ) {
size_t skip;
/* Align start of block */
return;
/* Add to allocation pool */
- free_memblock ( start, len );
+ heap_free_block ( heap, start, len );
/* Fix up memory usage statistics */
- usedmem += len;
+ heap->usedmem += len;
}
/**
*
*/
static void init_heap ( void ) {
- VALGRIND_MAKE_MEM_NOACCESS ( heap, sizeof ( heap ) );
- VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks, sizeof ( free_blocks ) );
- mpopulate ( heap, sizeof ( heap ) );
+ VALGRIND_MAKE_MEM_NOACCESS ( heap_area, sizeof ( heap_area ) );
+ VALGRIND_MAKE_MEM_NOACCESS ( &heap.blocks, sizeof ( heap.blocks ) );
+ heap_populate ( &heap, heap_area, sizeof ( heap_area ) );
}
/** Memory allocator initialisation function */
*/
static void shutdown_cache ( int booting __unused ) {
discard_all_cache();
- DBGC ( &heap, "HEAP maximum usage %zdkB\n", ( maxusedmem >> 10 ) );
+ DBGC ( &heap, "HEAP maximum usage %zdkB\n",
+ ( heap.maxusedmem >> 10 ) );
}
/** Memory allocator shutdown function */
* Dump free block list (for debugging)
*
*/
-void mdumpfree ( void ) {
+void heap_dump ( struct heap *heap ) {
struct memory_block *block;
dbg_printf ( "HEAP free block list:\n" );
- list_for_each_entry ( block, &free_blocks, list ) {
+ list_for_each_entry ( block, &heap->blocks, list ) {
dbg_printf ( "...[%p,%p] (size %#zx)\n", block,
( ( ( void * ) block ) + block->size ),
block->size );
projects / thirdparty / ipxe.git / commitdiff
