1 /* Malloc implementation for multiple threads without lock contention.
2 Copyright (C) 1996-2017 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Wolfram Gloger <wg@malloc.de>
5 and Doug Lea <dl@cs.oswego.edu>, 2001.
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public License as
9 published by the Free Software Foundation; either version 2.1 of the
10 License, or (at your option) any later version.
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; see the file COPYING.LIB. If
19 not, see <http://www.gnu.org/licenses/>. */
22 This is a version (aka ptmalloc2) of malloc/free/realloc written by
23 Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
25 There have been substantial changes made after the integration into
26 glibc in all parts of the code. Do not look for much commonality
27 with the ptmalloc2 version.
29 * Version ptmalloc2-20011215
31 VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
35 In order to compile this implementation, a Makefile is provided with
36 the ptmalloc2 distribution, which has pre-defined targets for some
37 popular systems (e.g. "make posix" for Posix threads). All that is
38 typically required with regard to compiler flags is the selection of
39 the thread package via defining one out of USE_PTHREADS, USE_THR or
40 USE_SPROC. Check the thread-m.h file for what effects this has.
41 Many/most systems will additionally require USE_TSD_DATA_HACK to be
42 defined, so this is the default for "make posix".
44 * Why use this malloc?
46 This is not the fastest, most space-conserving, most portable, or
47 most tunable malloc ever written. However it is among the fastest
48 while also being among the most space-conserving, portable and tunable.
49 Consistent balance across these factors results in a good general-purpose
50 allocator for malloc-intensive programs.
52 The main properties of the algorithms are:
53 * For large (>= 512 bytes) requests, it is a pure best-fit allocator,
54 with ties normally decided via FIFO (i.e. least recently used).
55 * For small (<= 64 bytes by default) requests, it is a caching
56 allocator, that maintains pools of quickly recycled chunks.
57 * In between, and for combinations of large and small requests, it does
58 the best it can trying to meet both goals at once.
59 * For very large requests (>= 128KB by default), it relies on system
60 memory mapping facilities, if supported.
62 For a longer but slightly out of date high-level description, see
63 http://gee.cs.oswego.edu/dl/html/malloc.html
65 You may already by default be using a C library containing a malloc
66 that is based on some version of this malloc (for example in
67 linux). You might still want to use the one in this file in order to
68 customize settings or to avoid overheads associated with library
71 * Contents, described in more detail in "description of public routines" below.
73 Standard (ANSI/SVID/...) functions:
75 calloc(size_t n_elements, size_t element_size);
77 realloc(void* p, size_t n);
78 memalign(size_t alignment, size_t n);
81 mallopt(int parameter_number, int parameter_value)
84 independent_calloc(size_t n_elements, size_t size, void* chunks[]);
85 independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
87 malloc_trim(size_t pad);
88 malloc_usable_size(void* p);
93 Supported pointer representation: 4 or 8 bytes
94 Supported size_t representation: 4 or 8 bytes
95 Note that size_t is allowed to be 4 bytes even if pointers are 8.
96 You can adjust this by defining INTERNAL_SIZE_T
98 Alignment: 2 * sizeof(size_t) (default)
99 (i.e., 8 byte alignment with 4byte size_t). This suffices for
100 nearly all current machines and C compilers. However, you can
101 define MALLOC_ALIGNMENT to be wider than this if necessary.
103 Minimum overhead per allocated chunk: 4 or 8 bytes
104 Each malloced chunk has a hidden word of overhead holding size
105 and status information.
107 Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
108 8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
110 When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
111 ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
112 needed; 4 (8) for a trailing size field and 8 (16) bytes for
113 free list pointers. Thus, the minimum allocatable size is
116 Even a request for zero bytes (i.e., malloc(0)) returns a
117 pointer to something of the minimum allocatable size.
119 The maximum overhead wastage (i.e., number of extra bytes
120 allocated than were requested in malloc) is less than or equal
121 to the minimum size, except for requests >= mmap_threshold that
122 are serviced via mmap(), where the worst case wastage is 2 *
123 sizeof(size_t) bytes plus the remainder from a system page (the
124 minimal mmap unit); typically 4096 or 8192 bytes.
126 Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
127 8-byte size_t: 2^64 minus about two pages
129 It is assumed that (possibly signed) size_t values suffice to
130 represent chunk sizes. `Possibly signed' is due to the fact
131 that `size_t' may be defined on a system as either a signed or
132 an unsigned type. The ISO C standard says that it must be
133 unsigned, but a few systems are known not to adhere to this.
134 Additionally, even when size_t is unsigned, sbrk (which is by
135 default used to obtain memory from system) accepts signed
136 arguments, and may not be able to handle size_t-wide arguments
137 with negative sign bit. Generally, values that would
138 appear as negative after accounting for overhead and alignment
139 are supported only via mmap(), which does not have this
142 Requests for sizes outside the allowed range will perform an optional
143 failure action and then return null. (Requests may also
144 also fail because a system is out of memory.)
146 Thread-safety: thread-safe
148 Compliance: I believe it is compliant with the 1997 Single Unix Specification
149 Also SVID/XPG, ANSI C, and probably others as well.
151 * Synopsis of compile-time options:
153 People have reported using previous versions of this malloc on all
154 versions of Unix, sometimes by tweaking some of the defines
155 below. It has been tested most extensively on Solaris and Linux.
156 People also report using it in stand-alone embedded systems.
158 The implementation is in straight, hand-tuned ANSI C. It is not
159 at all modular. (Sorry!) It uses a lot of macros. To be at all
160 usable, this code should be compiled using an optimizing compiler
161 (for example gcc -O3) that can simplify expressions and control
162 paths. (FAQ: some macros import variables as arguments rather than
163 declare locals because people reported that some debuggers
164 otherwise get confused.)
168 Compilation Environment options:
172 Changing default word sizes:
174 INTERNAL_SIZE_T size_t
176 Configuration and functionality options:
178 USE_PUBLIC_MALLOC_WRAPPERS NOT defined
179 USE_MALLOC_LOCK NOT defined
180 MALLOC_DEBUG NOT defined
181 REALLOC_ZERO_BYTES_FREES 1
184 Options for customizing MORECORE:
188 MORECORE_CONTIGUOUS 1
189 MORECORE_CANNOT_TRIM NOT defined
191 MMAP_AS_MORECORE_SIZE (1024 * 1024)
193 Tuning options that are also dynamically changeable via mallopt:
195 DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
196 DEFAULT_TRIM_THRESHOLD 128 * 1024
198 DEFAULT_MMAP_THRESHOLD 128 * 1024
199 DEFAULT_MMAP_MAX 65536
201 There are several other #defined constants and macros that you
202 probably don't want to touch unless you are extending or adapting malloc. */
205 void* is the pointer type that malloc should say it returns
212 #include <stddef.h> /* for size_t */
213 #include <stdlib.h> /* for getenv(), abort() */
214 #include <unistd.h> /* for __libc_enable_secure */
218 #include <bits/wordsize.h>
219 #include <sys/sysinfo.h>
221 #include <ldsodefs.h>
224 #include <stdio.h> /* needed for malloc_stats */
227 #include <shlib-compat.h>
232 /* For va_arg, va_start, va_end. */
235 /* For MIN, MAX, powerof2. */
236 #include <sys/param.h>
238 /* For ALIGN_UP et. al. */
239 #include <libc-pointer-arith.h>
241 /* For DIAG_PUSH/POP_NEEDS_COMMENT et al. */
242 #include <libc-diag.h>
244 #include <malloc/malloc-internal.h>
249 Because freed chunks may be overwritten with bookkeeping fields, this
250 malloc will often die when freed memory is overwritten by user
251 programs. This can be very effective (albeit in an annoying way)
252 in helping track down dangling pointers.
254 If you compile with -DMALLOC_DEBUG, a number of assertion checks are
255 enabled that will catch more memory errors. You probably won't be
256 able to make much sense of the actual assertion errors, but they
257 should help you locate incorrectly overwritten memory. The checking
258 is fairly extensive, and will slow down execution
259 noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
260 will attempt to check every non-mmapped allocated and free chunk in
261 the course of computing the summmaries. (By nature, mmapped regions
262 cannot be checked very much automatically.)
264 Setting MALLOC_DEBUG may also be helpful if you are trying to modify
265 this code. The assertions in the check routines spell out in more
266 detail the assumptions and invariants underlying the algorithms.
268 Setting MALLOC_DEBUG does NOT provide an automated mechanism for
269 checking that all accesses to malloced memory stay within their
270 bounds. However, there are several add-ons and adaptations of this
271 or other mallocs available that do this.
275 #define MALLOC_DEBUG 0
279 # define assert(expr) ((void) 0)
281 # define assert(expr) \
284 : __malloc_assert (#expr, __FILE__, __LINE__, __func__))
286 extern const char *__progname
;
289 __malloc_assert (const char *assertion
, const char *file
, unsigned int line
,
290 const char *function
)
292 (void) __fxprintf (NULL
, "%s%s%s:%u: %s%sAssertion `%s' failed.\n",
293 __progname
, __progname
[0] ? ": " : "",
295 function
? function
: "", function
? ": " : "",
303 /* We want 64 entries. This is an arbitrary limit, which tunables can reduce. */
304 # define TCACHE_MAX_BINS 64
305 # define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1)
307 /* Only used to pre-fill the tunables. */
308 # define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
310 /* When "x" is from chunksize(). */
311 # define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT)
312 /* When "x" is a user-provided size. */
313 # define usize2tidx(x) csize2tidx (request2size (x))
315 /* With rounding and alignment, the bins are...
316 idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit)
317 idx 1 bytes 25..40 or 13..20
318 idx 2 bytes 41..56 or 21..28
321 /* This is another arbitrary limit, which tunables can change. Each
322 tcache bin will hold at most this number of chunks. */
323 # define TCACHE_FILL_COUNT 7
328 REALLOC_ZERO_BYTES_FREES should be set if a call to
329 realloc with zero bytes should be the same as a call to free.
330 This is required by the C standard. Otherwise, since this malloc
331 returns a unique pointer for malloc(0), so does realloc(p, 0).
334 #ifndef REALLOC_ZERO_BYTES_FREES
335 #define REALLOC_ZERO_BYTES_FREES 1
339 TRIM_FASTBINS controls whether free() of a very small chunk can
340 immediately lead to trimming. Setting to true (1) can reduce memory
341 footprint, but will almost always slow down programs that use a lot
344 Define this only if you are willing to give up some speed to more
345 aggressively reduce system-level memory footprint when releasing
346 memory in programs that use many small chunks. You can get
347 essentially the same effect by setting MXFAST to 0, but this can
348 lead to even greater slowdowns in programs using many small chunks.
349 TRIM_FASTBINS is an in-between compile-time option, that disables
350 only those chunks bordering topmost memory from being placed in
354 #ifndef TRIM_FASTBINS
355 #define TRIM_FASTBINS 0
359 /* Definition for getting more memory from the OS. */
360 #define MORECORE (*__morecore)
361 #define MORECORE_FAILURE 0
362 void * __default_morecore (ptrdiff_t);
363 void *(*__morecore
)(ptrdiff_t) = __default_morecore
;
369 MORECORE-related declarations. By default, rely on sbrk
374 MORECORE is the name of the routine to call to obtain more memory
375 from the system. See below for general guidance on writing
376 alternative MORECORE functions, as well as a version for WIN32 and a
377 sample version for pre-OSX macos.
381 #define MORECORE sbrk
385 MORECORE_FAILURE is the value returned upon failure of MORECORE
386 as well as mmap. Since it cannot be an otherwise valid memory address,
387 and must reflect values of standard sys calls, you probably ought not
391 #ifndef MORECORE_FAILURE
392 #define MORECORE_FAILURE (-1)
396 If MORECORE_CONTIGUOUS is true, take advantage of fact that
397 consecutive calls to MORECORE with positive arguments always return
398 contiguous increasing addresses. This is true of unix sbrk. Even
399 if not defined, when regions happen to be contiguous, malloc will
400 permit allocations spanning regions obtained from different
401 calls. But defining this when applicable enables some stronger
402 consistency checks and space efficiencies.
405 #ifndef MORECORE_CONTIGUOUS
406 #define MORECORE_CONTIGUOUS 1
410 Define MORECORE_CANNOT_TRIM if your version of MORECORE
411 cannot release space back to the system when given negative
412 arguments. This is generally necessary only if you are using
413 a hand-crafted MORECORE function that cannot handle negative arguments.
416 /* #define MORECORE_CANNOT_TRIM */
418 /* MORECORE_CLEARS (default 1)
419 The degree to which the routine mapped to MORECORE zeroes out
420 memory: never (0), only for newly allocated space (1) or always
421 (2). The distinction between (1) and (2) is necessary because on
422 some systems, if the application first decrements and then
423 increments the break value, the contents of the reallocated space
427 #ifndef MORECORE_CLEARS
428 # define MORECORE_CLEARS 1
433 MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
434 sbrk fails, and mmap is used as a backup. The value must be a
435 multiple of page size. This backup strategy generally applies only
436 when systems have "holes" in address space, so sbrk cannot perform
437 contiguous expansion, but there is still space available on system.
438 On systems for which this is known to be useful (i.e. most linux
439 kernels), this occurs only when programs allocate huge amounts of
440 memory. Between this, and the fact that mmap regions tend to be
441 limited, the size should be large, to avoid too many mmap calls and
442 thus avoid running out of kernel resources. */
444 #ifndef MMAP_AS_MORECORE_SIZE
445 #define MMAP_AS_MORECORE_SIZE (1024 * 1024)
449 Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
454 #define HAVE_MREMAP 0
457 /* We may need to support __malloc_initialize_hook for backwards
460 #if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_24)
461 # define HAVE_MALLOC_INIT_HOOK 1
463 # define HAVE_MALLOC_INIT_HOOK 0
468 This version of malloc supports the standard SVID/XPG mallinfo
469 routine that returns a struct containing usage properties and
470 statistics. It should work on any SVID/XPG compliant system that has
471 a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
472 install such a thing yourself, cut out the preliminary declarations
473 as described above and below and save them in a malloc.h file. But
474 there's no compelling reason to bother to do this.)
476 The main declaration needed is the mallinfo struct that is returned
477 (by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
478 bunch of fields that are not even meaningful in this version of
479 malloc. These fields are are instead filled by mallinfo() with
480 other numbers that might be of interest.
484 /* ---------- description of public routines ------------ */
488 Returns a pointer to a newly allocated chunk of at least n bytes, or null
489 if no space is available. Additionally, on failure, errno is
490 set to ENOMEM on ANSI C systems.
492 If n is zero, malloc returns a minumum-sized chunk. (The minimum
493 size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
494 systems.) On most systems, size_t is an unsigned type, so calls
495 with negative arguments are interpreted as requests for huge amounts
496 of space, which will often fail. The maximum supported value of n
497 differs across systems, but is in all cases less than the maximum
498 representable value of a size_t.
500 void* __libc_malloc(size_t);
501 libc_hidden_proto (__libc_malloc
)
505 Releases the chunk of memory pointed to by p, that had been previously
506 allocated using malloc or a related routine such as realloc.
507 It has no effect if p is null. It can have arbitrary (i.e., bad!)
508 effects if p has already been freed.
510 Unless disabled (using mallopt), freeing very large spaces will
511 when possible, automatically trigger operations that give
512 back unused memory to the system, thus reducing program footprint.
514 void __libc_free(void*);
515 libc_hidden_proto (__libc_free
)
518 calloc(size_t n_elements, size_t element_size);
519 Returns a pointer to n_elements * element_size bytes, with all locations
522 void* __libc_calloc(size_t, size_t);
525 realloc(void* p, size_t n)
526 Returns a pointer to a chunk of size n that contains the same data
527 as does chunk p up to the minimum of (n, p's size) bytes, or null
528 if no space is available.
530 The returned pointer may or may not be the same as p. The algorithm
531 prefers extending p when possible, otherwise it employs the
532 equivalent of a malloc-copy-free sequence.
534 If p is null, realloc is equivalent to malloc.
536 If space is not available, realloc returns null, errno is set (if on
537 ANSI) and p is NOT freed.
539 if n is for fewer bytes than already held by p, the newly unused
540 space is lopped off and freed if possible. Unless the #define
541 REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
542 zero (re)allocates a minimum-sized chunk.
544 Large chunks that were internally obtained via mmap will always be
545 grown using malloc-copy-free sequences unless the system supports
546 MREMAP (currently only linux).
548 The old unix realloc convention of allowing the last-free'd chunk
549 to be used as an argument to realloc is not supported.
551 void* __libc_realloc(void*, size_t);
552 libc_hidden_proto (__libc_realloc
)
555 memalign(size_t alignment, size_t n);
556 Returns a pointer to a newly allocated chunk of n bytes, aligned
557 in accord with the alignment argument.
559 The alignment argument should be a power of two. If the argument is
560 not a power of two, the nearest greater power is used.
561 8-byte alignment is guaranteed by normal malloc calls, so don't
562 bother calling memalign with an argument of 8 or less.
564 Overreliance on memalign is a sure way to fragment space.
566 void* __libc_memalign(size_t, size_t);
567 libc_hidden_proto (__libc_memalign
)
571 Equivalent to memalign(pagesize, n), where pagesize is the page
572 size of the system. If the pagesize is unknown, 4096 is used.
574 void* __libc_valloc(size_t);
579 mallopt(int parameter_number, int parameter_value)
580 Sets tunable parameters The format is to provide a
581 (parameter-number, parameter-value) pair. mallopt then sets the
582 corresponding parameter to the argument value if it can (i.e., so
583 long as the value is meaningful), and returns 1 if successful else
584 0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
585 normally defined in malloc.h. Only one of these (M_MXFAST) is used
586 in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
587 so setting them has no effect. But this malloc also supports four
588 other options in mallopt. See below for details. Briefly, supported
589 parameters are as follows (listed defaults are for "typical"
592 Symbol param # default allowed param values
593 M_MXFAST 1 64 0-80 (0 disables fastbins)
594 M_TRIM_THRESHOLD -1 128*1024 any (-1U disables trimming)
596 M_MMAP_THRESHOLD -3 128*1024 any (or 0 if no MMAP support)
597 M_MMAP_MAX -4 65536 any (0 disables use of mmap)
599 int __libc_mallopt(int, int);
600 libc_hidden_proto (__libc_mallopt
)
605 Returns (by copy) a struct containing various summary statistics:
607 arena: current total non-mmapped bytes allocated from system
608 ordblks: the number of free chunks
609 smblks: the number of fastbin blocks (i.e., small chunks that
610 have been freed but not use resused or consolidated)
611 hblks: current number of mmapped regions
612 hblkhd: total bytes held in mmapped regions
614 fsmblks: total bytes held in fastbin blocks
615 uordblks: current total allocated space (normal or mmapped)
616 fordblks: total free space
617 keepcost: the maximum number of bytes that could ideally be released
618 back to system via malloc_trim. ("ideally" means that
619 it ignores page restrictions etc.)
621 Because these fields are ints, but internal bookkeeping may
622 be kept as longs, the reported values may wrap around zero and
625 struct mallinfo
__libc_mallinfo(void);
630 Equivalent to valloc(minimum-page-that-holds(n)), that is,
631 round up n to nearest pagesize.
633 void* __libc_pvalloc(size_t);
636 malloc_trim(size_t pad);
638 If possible, gives memory back to the system (via negative
639 arguments to sbrk) if there is unused memory at the `high' end of
640 the malloc pool. You can call this after freeing large blocks of
641 memory to potentially reduce the system-level memory requirements
642 of a program. However, it cannot guarantee to reduce memory. Under
643 some allocation patterns, some large free blocks of memory will be
644 locked between two used chunks, so they cannot be given back to
647 The `pad' argument to malloc_trim represents the amount of free
648 trailing space to leave untrimmed. If this argument is zero,
649 only the minimum amount of memory to maintain internal data
650 structures will be left (one page or less). Non-zero arguments
651 can be supplied to maintain enough trailing space to service
652 future expected allocations without having to re-obtain memory
655 Malloc_trim returns 1 if it actually released any memory, else 0.
656 On systems that do not support "negative sbrks", it will always
659 int __malloc_trim(size_t);
662 malloc_usable_size(void* p);
664 Returns the number of bytes you can actually use in
665 an allocated chunk, which may be more than you requested (although
666 often not) due to alignment and minimum size constraints.
667 You can use this many bytes without worrying about
668 overwriting other allocated objects. This is not a particularly great
669 programming practice. malloc_usable_size can be more useful in
670 debugging and assertions, for example:
673 assert(malloc_usable_size(p) >= 256);
676 size_t __malloc_usable_size(void*);
680 Prints on stderr the amount of space obtained from the system (both
681 via sbrk and mmap), the maximum amount (which may be more than
682 current if malloc_trim and/or munmap got called), and the current
683 number of bytes allocated via malloc (or realloc, etc) but not yet
684 freed. Note that this is the number of bytes allocated, not the
685 number requested. It will be larger than the number requested
686 because of alignment and bookkeeping overhead. Because it includes
687 alignment wastage as being in use, this figure may be greater than
688 zero even when no user-level chunks are allocated.
690 The reported current and maximum system memory can be inaccurate if
691 a program makes other calls to system memory allocation functions
692 (normally sbrk) outside of malloc.
694 malloc_stats prints only the most commonly interesting statistics.
695 More information can be obtained by calling mallinfo.
698 void __malloc_stats(void);
701 malloc_get_state(void);
703 Returns the state of all malloc variables in an opaque data
706 void* __malloc_get_state(void);
709 malloc_set_state(void* state);
711 Restore the state of all malloc variables from data obtained with
714 int __malloc_set_state(void*);
717 posix_memalign(void **memptr, size_t alignment, size_t size);
719 POSIX wrapper like memalign(), checking for validity of size.
721 int __posix_memalign(void **, size_t, size_t);
723 /* mallopt tuning options */
726 M_MXFAST is the maximum request size used for "fastbins", special bins
727 that hold returned chunks without consolidating their spaces. This
728 enables future requests for chunks of the same size to be handled
729 very quickly, but can increase fragmentation, and thus increase the
730 overall memory footprint of a program.
732 This malloc manages fastbins very conservatively yet still
733 efficiently, so fragmentation is rarely a problem for values less
734 than or equal to the default. The maximum supported value of MXFAST
735 is 80. You wouldn't want it any higher than this anyway. Fastbins
736 are designed especially for use with many small structs, objects or
737 strings -- the default handles structs/objects/arrays with sizes up
738 to 8 4byte fields, or small strings representing words, tokens,
739 etc. Using fastbins for larger objects normally worsens
740 fragmentation without improving speed.
742 M_MXFAST is set in REQUEST size units. It is internally used in
743 chunksize units, which adds padding and alignment. You can reduce
744 M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
745 algorithm to be a closer approximation of fifo-best-fit in all cases,
746 not just for larger requests, but will generally cause it to be
751 /* M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h */
756 #ifndef DEFAULT_MXFAST
757 #define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
762 M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
763 to keep before releasing via malloc_trim in free().
765 Automatic trimming is mainly useful in long-lived programs.
766 Because trimming via sbrk can be slow on some systems, and can
767 sometimes be wasteful (in cases where programs immediately
768 afterward allocate more large chunks) the value should be high
769 enough so that your overall system performance would improve by
770 releasing this much memory.
772 The trim threshold and the mmap control parameters (see below)
773 can be traded off with one another. Trimming and mmapping are
774 two different ways of releasing unused memory back to the
775 system. Between these two, it is often possible to keep
776 system-level demands of a long-lived program down to a bare
777 minimum. For example, in one test suite of sessions measuring
778 the XF86 X server on Linux, using a trim threshold of 128K and a
779 mmap threshold of 192K led to near-minimal long term resource
782 If you are using this malloc in a long-lived program, it should
783 pay to experiment with these values. As a rough guide, you
784 might set to a value close to the average size of a process
785 (program) running on your system. Releasing this much memory
786 would allow such a process to run in memory. Generally, it's
787 worth it to tune for trimming rather tham memory mapping when a
788 program undergoes phases where several large chunks are
789 allocated and released in ways that can reuse each other's
790 storage, perhaps mixed with phases where there are no such
791 chunks at all. And in well-behaved long-lived programs,
792 controlling release of large blocks via trimming versus mapping
795 However, in most programs, these parameters serve mainly as
796 protection against the system-level effects of carrying around
797 massive amounts of unneeded memory. Since frequent calls to
798 sbrk, mmap, and munmap otherwise degrade performance, the default
799 parameters are set to relatively high values that serve only as
802 The trim value It must be greater than page size to have any useful
803 effect. To disable trimming completely, you can set to
806 Trim settings interact with fastbin (MXFAST) settings: Unless
807 TRIM_FASTBINS is defined, automatic trimming never takes place upon
808 freeing a chunk with size less than or equal to MXFAST. Trimming is
809 instead delayed until subsequent freeing of larger chunks. However,
810 you can still force an attempted trim by calling malloc_trim.
812 Also, trimming is not generally possible in cases where
813 the main arena is obtained via mmap.
815 Note that the trick some people use of mallocing a huge space and
816 then freeing it at program startup, in an attempt to reserve system
817 memory, doesn't have the intended effect under automatic trimming,
818 since that memory will immediately be returned to the system.
821 #define M_TRIM_THRESHOLD -1
823 #ifndef DEFAULT_TRIM_THRESHOLD
824 #define DEFAULT_TRIM_THRESHOLD (128 * 1024)
828 M_TOP_PAD is the amount of extra `padding' space to allocate or
829 retain whenever sbrk is called. It is used in two ways internally:
831 * When sbrk is called to extend the top of the arena to satisfy
832 a new malloc request, this much padding is added to the sbrk
835 * When malloc_trim is called automatically from free(),
836 it is used as the `pad' argument.
838 In both cases, the actual amount of padding is rounded
839 so that the end of the arena is always a system page boundary.
841 The main reason for using padding is to avoid calling sbrk so
842 often. Having even a small pad greatly reduces the likelihood
843 that nearly every malloc request during program start-up (or
844 after trimming) will invoke sbrk, which needlessly wastes
847 Automatic rounding-up to page-size units is normally sufficient
848 to avoid measurable overhead, so the default is 0. However, in
849 systems where sbrk is relatively slow, it can pay to increase
850 this value, at the expense of carrying around more memory than
856 #ifndef DEFAULT_TOP_PAD
857 #define DEFAULT_TOP_PAD (0)
861 MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
862 adjusted MMAP_THRESHOLD.
865 #ifndef DEFAULT_MMAP_THRESHOLD_MIN
866 #define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
869 #ifndef DEFAULT_MMAP_THRESHOLD_MAX
870 /* For 32-bit platforms we cannot increase the maximum mmap
871 threshold much because it is also the minimum value for the
872 maximum heap size and its alignment. Going above 512k (i.e., 1M
873 for new heaps) wastes too much address space. */
874 # if __WORDSIZE == 32
875 # define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
877 # define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
882 M_MMAP_THRESHOLD is the request size threshold for using mmap()
883 to service a request. Requests of at least this size that cannot
884 be allocated using already-existing space will be serviced via mmap.
885 (If enough normal freed space already exists it is used instead.)
887 Using mmap segregates relatively large chunks of memory so that
888 they can be individually obtained and released from the host
889 system. A request serviced through mmap is never reused by any
890 other request (at least not directly; the system may just so
891 happen to remap successive requests to the same locations).
893 Segregating space in this way has the benefits that:
895 1. Mmapped space can ALWAYS be individually released back
896 to the system, which helps keep the system level memory
897 demands of a long-lived program low.
898 2. Mapped memory can never become `locked' between
899 other chunks, as can happen with normally allocated chunks, which
900 means that even trimming via malloc_trim would not release them.
901 3. On some systems with "holes" in address spaces, mmap can obtain
902 memory that sbrk cannot.
904 However, it has the disadvantages that:
906 1. The space cannot be reclaimed, consolidated, and then
907 used to service later requests, as happens with normal chunks.
908 2. It can lead to more wastage because of mmap page alignment
910 3. It causes malloc performance to be more dependent on host
911 system memory management support routines which may vary in
912 implementation quality and may impose arbitrary
913 limitations. Generally, servicing a request via normal
914 malloc steps is faster than going through a system's mmap.
916 The advantages of mmap nearly always outweigh disadvantages for
917 "large" chunks, but the value of "large" varies across systems. The
918 default is an empirically derived value that works well in most
923 The above was written in 2001. Since then the world has changed a lot.
924 Memory got bigger. Applications got bigger. The virtual address space
925 layout in 32 bit linux changed.
927 In the new situation, brk() and mmap space is shared and there are no
928 artificial limits on brk size imposed by the kernel. What is more,
929 applications have started using transient allocations larger than the
930 128Kb as was imagined in 2001.
932 The price for mmap is also high now; each time glibc mmaps from the
933 kernel, the kernel is forced to zero out the memory it gives to the
934 application. Zeroing memory is expensive and eats a lot of cache and
935 memory bandwidth. This has nothing to do with the efficiency of the
936 virtual memory system, by doing mmap the kernel just has no choice but
939 In 2001, the kernel had a maximum size for brk() which was about 800
940 megabytes on 32 bit x86, at that point brk() would hit the first
941 mmaped shared libaries and couldn't expand anymore. With current 2.6
942 kernels, the VA space layout is different and brk() and mmap
943 both can span the entire heap at will.
945 Rather than using a static threshold for the brk/mmap tradeoff,
946 we are now using a simple dynamic one. The goal is still to avoid
947 fragmentation. The old goals we kept are
948 1) try to get the long lived large allocations to use mmap()
949 2) really large allocations should always use mmap()
950 and we're adding now:
951 3) transient allocations should use brk() to avoid forcing the kernel
952 having to zero memory over and over again
954 The implementation works with a sliding threshold, which is by default
955 limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
956 out at 128Kb as per the 2001 default.
958 This allows us to satisfy requirement 1) under the assumption that long
959 lived allocations are made early in the process' lifespan, before it has
960 started doing dynamic allocations of the same size (which will
961 increase the threshold).
963 The upperbound on the threshold satisfies requirement 2)
965 The threshold goes up in value when the application frees memory that was
966 allocated with the mmap allocator. The idea is that once the application
967 starts freeing memory of a certain size, it's highly probable that this is
968 a size the application uses for transient allocations. This estimator
969 is there to satisfy the new third requirement.
973 #define M_MMAP_THRESHOLD -3
975 #ifndef DEFAULT_MMAP_THRESHOLD
976 #define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
980 M_MMAP_MAX is the maximum number of requests to simultaneously
981 service using mmap. This parameter exists because
982 some systems have a limited number of internal tables for
983 use by mmap, and using more than a few of them may degrade
986 The default is set to a value that serves only as a safeguard.
987 Setting to 0 disables use of mmap for servicing large requests.
990 #define M_MMAP_MAX -4
992 #ifndef DEFAULT_MMAP_MAX
993 #define DEFAULT_MMAP_MAX (65536)
998 #ifndef RETURN_ADDRESS
999 #define RETURN_ADDRESS(X_) (NULL)
1002 /* On some platforms we can compile internal, not exported functions better.
1003 Let the environment provide a macro and define it to be empty if it
1004 is not available. */
1005 #ifndef internal_function
1006 # define internal_function
1009 /* Forward declarations. */
1010 struct malloc_chunk
;
1011 typedef struct malloc_chunk
* mchunkptr
;
1013 /* Internal routines. */
1015 static void* _int_malloc(mstate
, size_t);
1016 static void _int_free(mstate
, mchunkptr
, int);
1017 static void* _int_realloc(mstate
, mchunkptr
, INTERNAL_SIZE_T
,
1019 static void* _int_memalign(mstate
, size_t, size_t);
1020 static void* _mid_memalign(size_t, size_t, void *);
1022 static void malloc_printerr(const char *str
) __attribute__ ((noreturn
));
1024 static void* internal_function
mem2mem_check(void *p
, size_t sz
);
1025 static int internal_function
top_check(void);
1026 static void internal_function
munmap_chunk(mchunkptr p
);
1028 static mchunkptr internal_function
mremap_chunk(mchunkptr p
, size_t new_size
);
1031 static void* malloc_check(size_t sz
, const void *caller
);
1032 static void free_check(void* mem
, const void *caller
);
1033 static void* realloc_check(void* oldmem
, size_t bytes
,
1034 const void *caller
);
1035 static void* memalign_check(size_t alignment
, size_t bytes
,
1036 const void *caller
);
1038 /* ------------------ MMAP support ------------------ */
1042 #include <sys/mman.h>
1044 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1045 # define MAP_ANONYMOUS MAP_ANON
1048 #ifndef MAP_NORESERVE
1049 # define MAP_NORESERVE 0
1052 #define MMAP(addr, size, prot, flags) \
1053 __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
1057 ----------------------- Chunk representations -----------------------
1062 This struct declaration is misleading (but accurate and necessary).
1063 It declares a "view" into memory allowing access to necessary
1064 fields at known offsets from a given base. See explanation below.
1067 struct malloc_chunk
{
1069 INTERNAL_SIZE_T mchunk_prev_size
; /* Size of previous chunk (if free). */
1070 INTERNAL_SIZE_T mchunk_size
; /* Size in bytes, including overhead. */
1072 struct malloc_chunk
* fd
; /* double links -- used only if free. */
1073 struct malloc_chunk
* bk
;
1075 /* Only used for large blocks: pointer to next larger size. */
1076 struct malloc_chunk
* fd_nextsize
; /* double links -- used only if free. */
1077 struct malloc_chunk
* bk_nextsize
;
1082 malloc_chunk details:
1084 (The following includes lightly edited explanations by Colin Plumb.)
1086 Chunks of memory are maintained using a `boundary tag' method as
1087 described in e.g., Knuth or Standish. (See the paper by Paul
1088 Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1089 survey of such techniques.) Sizes of free chunks are stored both
1090 in the front of each chunk and at the end. This makes
1091 consolidating fragmented chunks into bigger chunks very fast. The
1092 size fields also hold bits representing whether chunks are free or
1095 An allocated chunk looks like this:
1098 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1099 | Size of previous chunk, if unallocated (P clear) |
1100 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1101 | Size of chunk, in bytes |A|M|P|
1102 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1103 | User data starts here... .
1105 . (malloc_usable_size() bytes) .
1107 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1108 | (size of chunk, but used for application data) |
1109 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1110 | Size of next chunk, in bytes |A|0|1|
1111 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1113 Where "chunk" is the front of the chunk for the purpose of most of
1114 the malloc code, but "mem" is the pointer that is returned to the
1115 user. "Nextchunk" is the beginning of the next contiguous chunk.
1117 Chunks always begin on even word boundaries, so the mem portion
1118 (which is returned to the user) is also on an even word boundary, and
1119 thus at least double-word aligned.
1121 Free chunks are stored in circular doubly-linked lists, and look like this:
1123 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1124 | Size of previous chunk, if unallocated (P clear) |
1125 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1126 `head:' | Size of chunk, in bytes |A|0|P|
1127 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1128 | Forward pointer to next chunk in list |
1129 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1130 | Back pointer to previous chunk in list |
1131 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1132 | Unused space (may be 0 bytes long) .
1135 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1136 `foot:' | Size of chunk, in bytes |
1137 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1138 | Size of next chunk, in bytes |A|0|0|
1139 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1141 The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1142 chunk size (which is always a multiple of two words), is an in-use
1143 bit for the *previous* chunk. If that bit is *clear*, then the
1144 word before the current chunk size contains the previous chunk
1145 size, and can be used to find the front of the previous chunk.
1146 The very first chunk allocated always has this bit set,
1147 preventing access to non-existent (or non-owned) memory. If
1148 prev_inuse is set for any given chunk, then you CANNOT determine
1149 the size of the previous chunk, and might even get a memory
1150 addressing fault when trying to do so.
1152 The A (NON_MAIN_ARENA) bit is cleared for chunks on the initial,
1153 main arena, described by the main_arena variable. When additional
1154 threads are spawned, each thread receives its own arena (up to a
1155 configurable limit, after which arenas are reused for multiple
1156 threads), and the chunks in these arenas have the A bit set. To
1157 find the arena for a chunk on such a non-main arena, heap_for_ptr
1158 performs a bit mask operation and indirection through the ar_ptr
1159 member of the per-heap header heap_info (see arena.c).
1161 Note that the `foot' of the current chunk is actually represented
1162 as the prev_size of the NEXT chunk. This makes it easier to
1163 deal with alignments etc but can be very confusing when trying
1164 to extend or adapt this code.
1166 The three exceptions to all this are:
1168 1. The special chunk `top' doesn't bother using the
1169 trailing size field since there is no next contiguous chunk
1170 that would have to index off it. After initialization, `top'
1171 is forced to always exist. If it would become less than
1172 MINSIZE bytes long, it is replenished.
1174 2. Chunks allocated via mmap, which have the second-lowest-order
1175 bit M (IS_MMAPPED) set in their size fields. Because they are
1176 allocated one-by-one, each must contain its own trailing size
1177 field. If the M bit is set, the other bits are ignored
1178 (because mmapped chunks are neither in an arena, nor adjacent
1179 to a freed chunk). The M bit is also used for chunks which
1180 originally came from a dumped heap via malloc_set_state in
1183 3. Chunks in fastbins are treated as allocated chunks from the
1184 point of view of the chunk allocator. They are consolidated
1185 with their neighbors only in bulk, in malloc_consolidate.
1189 ---------- Size and alignment checks and conversions ----------
1192 /* conversion from malloc headers to user pointers, and back */
1194 #define chunk2mem(p) ((void*)((char*)(p) + 2*SIZE_SZ))
1195 #define mem2chunk(mem) ((mchunkptr)((char*)(mem) - 2*SIZE_SZ))
1197 /* The smallest possible chunk */
1198 #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1200 /* The smallest size we can malloc is an aligned minimal chunk */
1203 (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1205 /* Check if m has acceptable alignment */
1207 #define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1209 #define misaligned_chunk(p) \
1210 ((uintptr_t)(MALLOC_ALIGNMENT == 2 * SIZE_SZ ? (p) : chunk2mem (p)) \
1211 & MALLOC_ALIGN_MASK)
1215 Check if a request is so large that it would wrap around zero when
1216 padded and aligned. To simplify some other code, the bound is made
1217 low enough so that adding MINSIZE will also not wrap around zero.
1220 #define REQUEST_OUT_OF_RANGE(req) \
1221 ((unsigned long) (req) >= \
1222 (unsigned long) (INTERNAL_SIZE_T) (-2 * MINSIZE))
1224 /* pad request bytes into a usable size -- internal version */
1226 #define request2size(req) \
1227 (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1229 ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1231 /* Same, except also perform argument check */
1233 #define checked_request2size(req, sz) \
1234 if (REQUEST_OUT_OF_RANGE (req)) { \
1235 __set_errno (ENOMEM); \
1238 (sz) = request2size (req);
1241 --------------- Physical chunk operations ---------------
1245 /* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
1246 #define PREV_INUSE 0x1
1248 /* extract inuse bit of previous chunk */
1249 #define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
1252 /* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
1253 #define IS_MMAPPED 0x2
1255 /* check for mmap()'ed chunk */
1256 #define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
1259 /* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
1260 from a non-main arena. This is only set immediately before handing
1261 the chunk to the user, if necessary. */
1262 #define NON_MAIN_ARENA 0x4
1264 /* Check for chunk from main arena. */
1265 #define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
1267 /* Mark a chunk as not being on the main arena. */
1268 #define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
1272 Bits to mask off when extracting size
1274 Note: IS_MMAPPED is intentionally not masked off from size field in
1275 macros for which mmapped chunks should never be seen. This should
1276 cause helpful core dumps to occur if it is tried by accident by
1277 people extending or adapting this malloc.
1279 #define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
1281 /* Get size, ignoring use bits */
1282 #define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
1284 /* Like chunksize, but do not mask SIZE_BITS. */
1285 #define chunksize_nomask(p) ((p)->mchunk_size)
1287 /* Ptr to next physical malloc_chunk. */
1288 #define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
1290 /* Size of the chunk below P. Only valid if prev_inuse (P). */
1291 #define prev_size(p) ((p)->mchunk_prev_size)
1293 /* Set the size of the chunk below P. Only valid if prev_inuse (P). */
1294 #define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz))
1296 /* Ptr to previous physical malloc_chunk. Only valid if prev_inuse (P). */
1297 #define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p)))
1299 /* Treat space at ptr + offset as a chunk */
1300 #define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s)))
1302 /* extract p's inuse bit */
1304 ((((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size) & PREV_INUSE)
1306 /* set/clear chunk as being inuse without otherwise disturbing */
1307 #define set_inuse(p) \
1308 ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size |= PREV_INUSE
1310 #define clear_inuse(p) \
1311 ((mchunkptr) (((char *) (p)) + chunksize (p)))->mchunk_size &= ~(PREV_INUSE)
1314 /* check/set/clear inuse bits in known places */
1315 #define inuse_bit_at_offset(p, s) \
1316 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE)
1318 #define set_inuse_bit_at_offset(p, s) \
1319 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size |= PREV_INUSE)
1321 #define clear_inuse_bit_at_offset(p, s) \
1322 (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE))
1325 /* Set size at head, without disturbing its use bit */
1326 #define set_head_size(p, s) ((p)->mchunk_size = (((p)->mchunk_size & SIZE_BITS) | (s)))
1328 /* Set size/use field */
1329 #define set_head(p, s) ((p)->mchunk_size = (s))
1331 /* Set size at footer (only when chunk is not in use) */
1332 #define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s))
1335 #pragma GCC poison mchunk_size
1336 #pragma GCC poison mchunk_prev_size
1339 -------------------- Internal data structures --------------------
1341 All internal state is held in an instance of malloc_state defined
1342 below. There are no other static variables, except in two optional
1344 * If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1345 * If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1348 Beware of lots of tricks that minimize the total bookkeeping space
1349 requirements. The result is a little over 1K bytes (for 4byte
1350 pointers and size_t.)
1356 An array of bin headers for free chunks. Each bin is doubly
1357 linked. The bins are approximately proportionally (log) spaced.
1358 There are a lot of these bins (128). This may look excessive, but
1359 works very well in practice. Most bins hold sizes that are
1360 unusual as malloc request sizes, but are more usual for fragments
1361 and consolidated sets of chunks, which is what these bins hold, so
1362 they can be found quickly. All procedures maintain the invariant
1363 that no consolidated chunk physically borders another one, so each
1364 chunk in a list is known to be preceeded and followed by either
1365 inuse chunks or the ends of memory.
1367 Chunks in bins are kept in size order, with ties going to the
1368 approximately least recently used chunk. Ordering isn't needed
1369 for the small bins, which all contain the same-sized chunks, but
1370 facilitates best-fit allocation for larger chunks. These lists
1371 are just sequential. Keeping them in order almost never requires
1372 enough traversal to warrant using fancier ordered data
1375 Chunks of the same size are linked with the most
1376 recently freed at the front, and allocations are taken from the
1377 back. This results in LRU (FIFO) allocation order, which tends
1378 to give each chunk an equal opportunity to be consolidated with
1379 adjacent freed chunks, resulting in larger free chunks and less
1382 To simplify use in double-linked lists, each bin header acts
1383 as a malloc_chunk. This avoids special-casing for headers.
1384 But to conserve space and improve locality, we allocate
1385 only the fd/bk pointers of bins, and then use repositioning tricks
1386 to treat these as the fields of a malloc_chunk*.
1389 typedef struct malloc_chunk
*mbinptr
;
1391 /* addressing -- note that bin_at(0) does not exist */
1392 #define bin_at(m, i) \
1393 (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2])) \
1394 - offsetof (struct malloc_chunk, fd))
1396 /* analog of ++bin */
1397 #define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1)))
1399 /* Reminders about list directionality within bins */
1400 #define first(b) ((b)->fd)
1401 #define last(b) ((b)->bk)
1403 /* Take a chunk off a bin list */
1404 #define unlink(AV, P, BK, FD) { \
1405 if (__builtin_expect (chunksize(P) != prev_size (next_chunk(P)), 0)) \
1406 malloc_printerr ("corrupted size vs. prev_size"); \
1409 if (__builtin_expect (FD->bk != P || BK->fd != P, 0)) \
1410 malloc_printerr ("corrupted double-linked list"); \
1414 if (!in_smallbin_range (chunksize_nomask (P)) \
1415 && __builtin_expect (P->fd_nextsize != NULL, 0)) { \
1416 if (__builtin_expect (P->fd_nextsize->bk_nextsize != P, 0) \
1417 || __builtin_expect (P->bk_nextsize->fd_nextsize != P, 0)) \
1418 malloc_printerr ("corrupted double-linked list (not small)"); \
1419 if (FD->fd_nextsize == NULL) { \
1420 if (P->fd_nextsize == P) \
1421 FD->fd_nextsize = FD->bk_nextsize = FD; \
1423 FD->fd_nextsize = P->fd_nextsize; \
1424 FD->bk_nextsize = P->bk_nextsize; \
1425 P->fd_nextsize->bk_nextsize = FD; \
1426 P->bk_nextsize->fd_nextsize = FD; \
1429 P->fd_nextsize->bk_nextsize = P->bk_nextsize; \
1430 P->bk_nextsize->fd_nextsize = P->fd_nextsize; \
1439 Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1440 8 bytes apart. Larger bins are approximately logarithmically spaced:
1446 4 bins of size 32768
1447 2 bins of size 262144
1448 1 bin of size what's left
1450 There is actually a little bit of slop in the numbers in bin_index
1451 for the sake of speed. This makes no difference elsewhere.
1453 The bins top out around 1MB because we expect to service large
1456 Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1457 a valid chunk size the small bins are bumped up one.
1461 #define NSMALLBINS 64
1462 #define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1463 #define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > 2 * SIZE_SZ)
1464 #define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1466 #define in_smallbin_range(sz) \
1467 ((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE)
1469 #define smallbin_index(sz) \
1470 ((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\
1471 + SMALLBIN_CORRECTION)
1473 #define largebin_index_32(sz) \
1474 (((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\
1475 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1476 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1477 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1478 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1481 #define largebin_index_32_big(sz) \
1482 (((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\
1483 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1484 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1485 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1486 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1489 // XXX It remains to be seen whether it is good to keep the widths of
1490 // XXX the buckets the same or whether it should be scaled by a factor
1491 // XXX of two as well.
1492 #define largebin_index_64(sz) \
1493 (((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\
1494 ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\
1495 ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\
1496 ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\
1497 ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\
1500 #define largebin_index(sz) \
1501 (SIZE_SZ == 8 ? largebin_index_64 (sz) \
1502 : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1503 : largebin_index_32 (sz))
1505 #define bin_index(sz) \
1506 ((in_smallbin_range (sz)) ? smallbin_index (sz) : largebin_index (sz))
1512 All remainders from chunk splits, as well as all returned chunks,
1513 are first placed in the "unsorted" bin. They are then placed
1514 in regular bins after malloc gives them ONE chance to be used before
1515 binning. So, basically, the unsorted_chunks list acts as a queue,
1516 with chunks being placed on it in free (and malloc_consolidate),
1517 and taken off (to be either used or placed in bins) in malloc.
1519 The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1520 does not have to be taken into account in size comparisons.
1523 /* The otherwise unindexable 1-bin is used to hold unsorted chunks. */
1524 #define unsorted_chunks(M) (bin_at (M, 1))
1529 The top-most available chunk (i.e., the one bordering the end of
1530 available memory) is treated specially. It is never included in
1531 any bin, is used only if no other chunk is available, and is
1532 released back to the system if it is very large (see
1533 M_TRIM_THRESHOLD). Because top initially
1534 points to its own bin with initial zero size, thus forcing
1535 extension on the first malloc request, we avoid having any special
1536 code in malloc to check whether it even exists yet. But we still
1537 need to do so when getting memory from system, so we make
1538 initial_top treat the bin as a legal but unusable chunk during the
1539 interval between initialization and the first call to
1540 sysmalloc. (This is somewhat delicate, since it relies on
1541 the 2 preceding words to be zero during this interval as well.)
1544 /* Conveniently, the unsorted bin can be used as dummy top on first call */
1545 #define initial_top(M) (unsorted_chunks (M))
1550 To help compensate for the large number of bins, a one-level index
1551 structure is used for bin-by-bin searching. `binmap' is a
1552 bitvector recording whether bins are definitely empty so they can
1553 be skipped over during during traversals. The bits are NOT always
1554 cleared as soon as bins are empty, but instead only
1555 when they are noticed to be empty during traversal in malloc.
1558 /* Conservatively use 32 bits per map word, even if on 64bit system */
1559 #define BINMAPSHIFT 5
1560 #define BITSPERMAP (1U << BINMAPSHIFT)
1561 #define BINMAPSIZE (NBINS / BITSPERMAP)
1563 #define idx2block(i) ((i) >> BINMAPSHIFT)
1564 #define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1))))
1566 #define mark_bin(m, i) ((m)->binmap[idx2block (i)] |= idx2bit (i))
1567 #define unmark_bin(m, i) ((m)->binmap[idx2block (i)] &= ~(idx2bit (i)))
1568 #define get_binmap(m, i) ((m)->binmap[idx2block (i)] & idx2bit (i))
1573 An array of lists holding recently freed small chunks. Fastbins
1574 are not doubly linked. It is faster to single-link them, and
1575 since chunks are never removed from the middles of these lists,
1576 double linking is not necessary. Also, unlike regular bins, they
1577 are not even processed in FIFO order (they use faster LIFO) since
1578 ordering doesn't much matter in the transient contexts in which
1579 fastbins are normally used.
1581 Chunks in fastbins keep their inuse bit set, so they cannot
1582 be consolidated with other free chunks. malloc_consolidate
1583 releases all chunks in fastbins and consolidates them with
1587 typedef struct malloc_chunk
*mfastbinptr
;
1588 #define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1590 /* offset 2 to use otherwise unindexable first 2 bins */
1591 #define fastbin_index(sz) \
1592 ((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1595 /* The maximum fastbin request size we support */
1596 #define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1598 #define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1)
1601 FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1602 that triggers automatic consolidation of possibly-surrounding
1603 fastbin chunks. This is a heuristic, so the exact value should not
1604 matter too much. It is defined at half the default trim threshold as a
1605 compromise heuristic to only attempt consolidation if it is likely
1606 to lead to trimming. However, it is not dynamically tunable, since
1607 consolidation reduces fragmentation surrounding large chunks even
1608 if trimming is not used.
1611 #define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1614 Since the lowest 2 bits in max_fast don't matter in size comparisons,
1615 they are used as flags.
1619 FASTCHUNKS_BIT held in max_fast indicates that there are probably
1620 some fastbin chunks. It is set true on entering a chunk into any
1621 fastbin, and cleared only in malloc_consolidate.
1623 The truth value is inverted so that have_fastchunks will be true
1624 upon startup (since statics are zero-filled), simplifying
1625 initialization checks.
1628 #define FASTCHUNKS_BIT (1U)
1630 #define have_fastchunks(M) (((M)->flags & FASTCHUNKS_BIT) == 0)
1631 #define clear_fastchunks(M) catomic_or (&(M)->flags, FASTCHUNKS_BIT)
1632 #define set_fastchunks(M) catomic_and (&(M)->flags, ~FASTCHUNKS_BIT)
1635 NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1636 regions. Otherwise, contiguity is exploited in merging together,
1637 when possible, results from consecutive MORECORE calls.
1639 The initial value comes from MORECORE_CONTIGUOUS, but is
1640 changed dynamically if mmap is ever used as an sbrk substitute.
1643 #define NONCONTIGUOUS_BIT (2U)
1645 #define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1646 #define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1647 #define set_noncontiguous(M) ((M)->flags |= NONCONTIGUOUS_BIT)
1648 #define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1650 /* Maximum size of memory handled in fastbins. */
1651 static INTERNAL_SIZE_T global_max_fast
;
1654 Set value of max_fast.
1655 Use impossibly small value if 0.
1656 Precondition: there are no existing fastbin chunks.
1657 Setting the value clears fastchunk bit but preserves noncontiguous bit.
1660 #define set_max_fast(s) \
1661 global_max_fast = (((s) == 0) \
1662 ? SMALLBIN_WIDTH : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1664 static inline INTERNAL_SIZE_T
1667 /* Tell the GCC optimizers that global_max_fast is never larger
1668 than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in
1669 _int_malloc after constant propagation of the size parameter.
1670 (The code never executes because malloc preserves the
1671 global_max_fast invariant, but the optimizers may not recognize
1673 if (global_max_fast
> MAX_FAST_SIZE
)
1674 __builtin_unreachable ();
1675 return global_max_fast
;
1679 ----------- Internal state representation and initialization -----------
1684 /* Serialize access. */
1685 __libc_lock_define (, mutex
);
1687 /* Flags (formerly in max_fast). */
1691 mfastbinptr fastbinsY
[NFASTBINS
];
1693 /* Base of the topmost chunk -- not otherwise kept in a bin */
1696 /* The remainder from the most recent split of a small request */
1697 mchunkptr last_remainder
;
1699 /* Normal bins packed as described above */
1700 mchunkptr bins
[NBINS
* 2 - 2];
1702 /* Bitmap of bins */
1703 unsigned int binmap
[BINMAPSIZE
];
1706 struct malloc_state
*next
;
1708 /* Linked list for free arenas. Access to this field is serialized
1709 by free_list_lock in arena.c. */
1710 struct malloc_state
*next_free
;
1712 /* Number of threads attached to this arena. 0 if the arena is on
1713 the free list. Access to this field is serialized by
1714 free_list_lock in arena.c. */
1715 INTERNAL_SIZE_T attached_threads
;
1717 /* Memory allocated from the system in this arena. */
1718 INTERNAL_SIZE_T system_mem
;
1719 INTERNAL_SIZE_T max_system_mem
;
1724 /* Tunable parameters */
1725 unsigned long trim_threshold
;
1726 INTERNAL_SIZE_T top_pad
;
1727 INTERNAL_SIZE_T mmap_threshold
;
1728 INTERNAL_SIZE_T arena_test
;
1729 INTERNAL_SIZE_T arena_max
;
1731 /* Memory map support */
1735 /* the mmap_threshold is dynamic, until the user sets
1736 it manually, at which point we need to disable any
1737 dynamic behavior. */
1738 int no_dyn_threshold
;
1741 INTERNAL_SIZE_T mmapped_mem
;
1742 INTERNAL_SIZE_T max_mmapped_mem
;
1744 /* First address handed out by MORECORE/sbrk. */
1748 /* Maximum number of buckets to use. */
1750 size_t tcache_max_bytes
;
1751 /* Maximum number of chunks in each bucket. */
1752 size_t tcache_count
;
1753 /* Maximum number of chunks to remove from the unsorted list, which
1754 aren't used to prefill the cache. */
1755 size_t tcache_unsorted_limit
;
1759 /* There are several instances of this struct ("arenas") in this
1760 malloc. If you are adapting this malloc in a way that does NOT use
1761 a static or mmapped malloc_state, you MUST explicitly zero-fill it
1762 before using. This malloc relies on the property that malloc_state
1763 is initialized to all zeroes (as is true of C statics). */
1765 static struct malloc_state main_arena
=
1767 .mutex
= _LIBC_LOCK_INITIALIZER
,
1768 .next
= &main_arena
,
1769 .attached_threads
= 1
1772 /* These variables are used for undumping support. Chunked are marked
1773 as using mmap, but we leave them alone if they fall into this
1774 range. NB: The chunk size for these chunks only includes the
1775 initial size field (of SIZE_SZ bytes), there is no trailing size
1776 field (unlike with regular mmapped chunks). */
1777 static mchunkptr dumped_main_arena_start
; /* Inclusive. */
1778 static mchunkptr dumped_main_arena_end
; /* Exclusive. */
1780 /* True if the pointer falls into the dumped arena. Use this after
1781 chunk_is_mmapped indicates a chunk is mmapped. */
1782 #define DUMPED_MAIN_ARENA_CHUNK(p) \
1783 ((p) >= dumped_main_arena_start && (p) < dumped_main_arena_end)
1785 /* There is only one instance of the malloc parameters. */
1787 static struct malloc_par mp_
=
1789 .top_pad
= DEFAULT_TOP_PAD
,
1790 .n_mmaps_max
= DEFAULT_MMAP_MAX
,
1791 .mmap_threshold
= DEFAULT_MMAP_THRESHOLD
,
1792 .trim_threshold
= DEFAULT_TRIM_THRESHOLD
,
1793 #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8))
1794 .arena_test
= NARENAS_FROM_NCORES (1)
1797 .tcache_count
= TCACHE_FILL_COUNT
,
1798 .tcache_bins
= TCACHE_MAX_BINS
,
1799 .tcache_max_bytes
= tidx2usize (TCACHE_MAX_BINS
-1),
1800 .tcache_unsorted_limit
= 0 /* No limit. */
1805 Initialize a malloc_state struct.
1807 This is called only from within malloc_consolidate, which needs
1808 be called in the same contexts anyway. It is never called directly
1809 outside of malloc_consolidate because some optimizing compilers try
1810 to inline it at all call points, which turns out not to be an
1811 optimization at all. (Inlining it in malloc_consolidate is fine though.)
1815 malloc_init_state (mstate av
)
1820 /* Establish circular links for normal bins */
1821 for (i
= 1; i
< NBINS
; ++i
)
1823 bin
= bin_at (av
, i
);
1824 bin
->fd
= bin
->bk
= bin
;
1827 #if MORECORE_CONTIGUOUS
1828 if (av
!= &main_arena
)
1830 set_noncontiguous (av
);
1831 if (av
== &main_arena
)
1832 set_max_fast (DEFAULT_MXFAST
);
1833 av
->flags
|= FASTCHUNKS_BIT
;
1835 av
->top
= initial_top (av
);
1839 Other internal utilities operating on mstates
1842 static void *sysmalloc (INTERNAL_SIZE_T
, mstate
);
1843 static int systrim (size_t, mstate
);
1844 static void malloc_consolidate (mstate
);
1847 /* -------------- Early definitions for debugging hooks ---------------- */
1849 /* Define and initialize the hook variables. These weak definitions must
1850 appear before any use of the variables in a function (arena.c uses one). */
1851 #ifndef weak_variable
1852 /* In GNU libc we want the hook variables to be weak definitions to
1853 avoid a problem with Emacs. */
1854 # define weak_variable weak_function
1857 /* Forward declarations. */
1858 static void *malloc_hook_ini (size_t sz
,
1859 const void *caller
) __THROW
;
1860 static void *realloc_hook_ini (void *ptr
, size_t sz
,
1861 const void *caller
) __THROW
;
1862 static void *memalign_hook_ini (size_t alignment
, size_t sz
,
1863 const void *caller
) __THROW
;
1865 #if HAVE_MALLOC_INIT_HOOK
1866 void weak_variable (*__malloc_initialize_hook
) (void) = NULL
;
1867 compat_symbol (libc
, __malloc_initialize_hook
,
1868 __malloc_initialize_hook
, GLIBC_2_0
);
1871 void weak_variable (*__free_hook
) (void *__ptr
,
1872 const void *) = NULL
;
1873 void *weak_variable (*__malloc_hook
)
1874 (size_t __size
, const void *) = malloc_hook_ini
;
1875 void *weak_variable (*__realloc_hook
)
1876 (void *__ptr
, size_t __size
, const void *)
1878 void *weak_variable (*__memalign_hook
)
1879 (size_t __alignment
, size_t __size
, const void *)
1880 = memalign_hook_ini
;
1881 void weak_variable (*__after_morecore_hook
) (void) = NULL
;
1884 /* ------------------ Testing support ----------------------------------*/
1886 static int perturb_byte
;
1889 alloc_perturb (char *p
, size_t n
)
1891 if (__glibc_unlikely (perturb_byte
))
1892 memset (p
, perturb_byte
^ 0xff, n
);
1896 free_perturb (char *p
, size_t n
)
1898 if (__glibc_unlikely (perturb_byte
))
1899 memset (p
, perturb_byte
, n
);
1904 #include <stap-probe.h>
1906 /* ------------------- Support for multiple arenas -------------------- */
1912 These routines make a number of assertions about the states
1913 of data structures that should be true at all times. If any
1914 are not true, it's very likely that a user program has somehow
1915 trashed memory. (It's also possible that there is a coding error
1916 in malloc. In which case, please report it!)
1921 # define check_chunk(A, P)
1922 # define check_free_chunk(A, P)
1923 # define check_inuse_chunk(A, P)
1924 # define check_remalloced_chunk(A, P, N)
1925 # define check_malloced_chunk(A, P, N)
1926 # define check_malloc_state(A)
1930 # define check_chunk(A, P) do_check_chunk (A, P)
1931 # define check_free_chunk(A, P) do_check_free_chunk (A, P)
1932 # define check_inuse_chunk(A, P) do_check_inuse_chunk (A, P)
1933 # define check_remalloced_chunk(A, P, N) do_check_remalloced_chunk (A, P, N)
1934 # define check_malloced_chunk(A, P, N) do_check_malloced_chunk (A, P, N)
1935 # define check_malloc_state(A) do_check_malloc_state (A)
1938 Properties of all chunks
1942 do_check_chunk (mstate av
, mchunkptr p
)
1944 unsigned long sz
= chunksize (p
);
1945 /* min and max possible addresses assuming contiguous allocation */
1946 char *max_address
= (char *) (av
->top
) + chunksize (av
->top
);
1947 char *min_address
= max_address
- av
->system_mem
;
1949 if (!chunk_is_mmapped (p
))
1951 /* Has legal address ... */
1954 if (contiguous (av
))
1956 assert (((char *) p
) >= min_address
);
1957 assert (((char *) p
+ sz
) <= ((char *) (av
->top
)));
1962 /* top size is always at least MINSIZE */
1963 assert ((unsigned long) (sz
) >= MINSIZE
);
1964 /* top predecessor always marked inuse */
1965 assert (prev_inuse (p
));
1968 else if (!DUMPED_MAIN_ARENA_CHUNK (p
))
1970 /* address is outside main heap */
1971 if (contiguous (av
) && av
->top
!= initial_top (av
))
1973 assert (((char *) p
) < min_address
|| ((char *) p
) >= max_address
);
1975 /* chunk is page-aligned */
1976 assert (((prev_size (p
) + sz
) & (GLRO (dl_pagesize
) - 1)) == 0);
1977 /* mem is aligned */
1978 assert (aligned_OK (chunk2mem (p
)));
1983 Properties of free chunks
1987 do_check_free_chunk (mstate av
, mchunkptr p
)
1989 INTERNAL_SIZE_T sz
= p
->size
& ~(PREV_INUSE
| NON_MAIN_ARENA
);
1990 mchunkptr next
= chunk_at_offset (p
, sz
);
1992 do_check_chunk (av
, p
);
1994 /* Chunk must claim to be free ... */
1995 assert (!inuse (p
));
1996 assert (!chunk_is_mmapped (p
));
1998 /* Unless a special marker, must have OK fields */
1999 if ((unsigned long) (sz
) >= MINSIZE
)
2001 assert ((sz
& MALLOC_ALIGN_MASK
) == 0);
2002 assert (aligned_OK (chunk2mem (p
)));
2003 /* ... matching footer field */
2004 assert (prev_size (p
) == sz
);
2005 /* ... and is fully consolidated */
2006 assert (prev_inuse (p
));
2007 assert (next
== av
->top
|| inuse (next
));
2009 /* ... and has minimally sane links */
2010 assert (p
->fd
->bk
== p
);
2011 assert (p
->bk
->fd
== p
);
2013 else /* markers are always of size SIZE_SZ */
2014 assert (sz
== SIZE_SZ
);
2018 Properties of inuse chunks
2022 do_check_inuse_chunk (mstate av
, mchunkptr p
)
2026 do_check_chunk (av
, p
);
2028 if (chunk_is_mmapped (p
))
2029 return; /* mmapped chunks have no next/prev */
2031 /* Check whether it claims to be in use ... */
2034 next
= next_chunk (p
);
2036 /* ... and is surrounded by OK chunks.
2037 Since more things can be checked with free chunks than inuse ones,
2038 if an inuse chunk borders them and debug is on, it's worth doing them.
2040 if (!prev_inuse (p
))
2042 /* Note that we cannot even look at prev unless it is not inuse */
2043 mchunkptr prv
= prev_chunk (p
);
2044 assert (next_chunk (prv
) == p
);
2045 do_check_free_chunk (av
, prv
);
2048 if (next
== av
->top
)
2050 assert (prev_inuse (next
));
2051 assert (chunksize (next
) >= MINSIZE
);
2053 else if (!inuse (next
))
2054 do_check_free_chunk (av
, next
);
2058 Properties of chunks recycled from fastbins
2062 do_check_remalloced_chunk (mstate av
, mchunkptr p
, INTERNAL_SIZE_T s
)
2064 INTERNAL_SIZE_T sz
= p
->size
& ~(PREV_INUSE
| NON_MAIN_ARENA
);
2066 if (!chunk_is_mmapped (p
))
2068 assert (av
== arena_for_chunk (p
));
2069 if (chunk_main_arena (p
))
2070 assert (av
== &main_arena
);
2072 assert (av
!= &main_arena
);
2075 do_check_inuse_chunk (av
, p
);
2077 /* Legal size ... */
2078 assert ((sz
& MALLOC_ALIGN_MASK
) == 0);
2079 assert ((unsigned long) (sz
) >= MINSIZE
);
2080 /* ... and alignment */
2081 assert (aligned_OK (chunk2mem (p
)));
2082 /* chunk is less than MINSIZE more than request */
2083 assert ((long) (sz
) - (long) (s
) >= 0);
2084 assert ((long) (sz
) - (long) (s
+ MINSIZE
) < 0);
2088 Properties of nonrecycled chunks at the point they are malloced
2092 do_check_malloced_chunk (mstate av
, mchunkptr p
, INTERNAL_SIZE_T s
)
2094 /* same as recycled case ... */
2095 do_check_remalloced_chunk (av
, p
, s
);
2098 ... plus, must obey implementation invariant that prev_inuse is
2099 always true of any allocated chunk; i.e., that each allocated
2100 chunk borders either a previously allocated and still in-use
2101 chunk, or the base of its memory arena. This is ensured
2102 by making all allocations from the `lowest' part of any found
2103 chunk. This does not necessarily hold however for chunks
2104 recycled via fastbins.
2107 assert (prev_inuse (p
));
2112 Properties of malloc_state.
2114 This may be useful for debugging malloc, as well as detecting user
2115 programmer errors that somehow write into malloc_state.
2117 If you are extending or experimenting with this malloc, you can
2118 probably figure out how to hack this routine to print out or
2119 display chunk addresses, sizes, bins, and other instrumentation.
2123 do_check_malloc_state (mstate av
)
2130 INTERNAL_SIZE_T size
;
2131 unsigned long total
= 0;
2134 /* internal size_t must be no wider than pointer type */
2135 assert (sizeof (INTERNAL_SIZE_T
) <= sizeof (char *));
2137 /* alignment is a power of 2 */
2138 assert ((MALLOC_ALIGNMENT
& (MALLOC_ALIGNMENT
- 1)) == 0);
2140 /* cannot run remaining checks until fully initialized */
2141 if (av
->top
== 0 || av
->top
== initial_top (av
))
2144 /* pagesize is a power of 2 */
2145 assert (powerof2(GLRO (dl_pagesize
)));
2147 /* A contiguous main_arena is consistent with sbrk_base. */
2148 if (av
== &main_arena
&& contiguous (av
))
2149 assert ((char *) mp_
.sbrk_base
+ av
->system_mem
==
2150 (char *) av
->top
+ chunksize (av
->top
));
2152 /* properties of fastbins */
2154 /* max_fast is in allowed range */
2155 assert ((get_max_fast () & ~1) <= request2size (MAX_FAST_SIZE
));
2157 max_fast_bin
= fastbin_index (get_max_fast ());
2159 for (i
= 0; i
< NFASTBINS
; ++i
)
2161 p
= fastbin (av
, i
);
2163 /* The following test can only be performed for the main arena.
2164 While mallopt calls malloc_consolidate to get rid of all fast
2165 bins (especially those larger than the new maximum) this does
2166 only happen for the main arena. Trying to do this for any
2167 other arena would mean those arenas have to be locked and
2168 malloc_consolidate be called for them. This is excessive. And
2169 even if this is acceptable to somebody it still cannot solve
2170 the problem completely since if the arena is locked a
2171 concurrent malloc call might create a new arena which then
2172 could use the newly invalid fast bins. */
2174 /* all bins past max_fast are empty */
2175 if (av
== &main_arena
&& i
> max_fast_bin
)
2180 /* each chunk claims to be inuse */
2181 do_check_inuse_chunk (av
, p
);
2182 total
+= chunksize (p
);
2183 /* chunk belongs in this bin */
2184 assert (fastbin_index (chunksize (p
)) == i
);
2190 assert (have_fastchunks (av
));
2191 else if (!have_fastchunks (av
))
2192 assert (total
== 0);
2194 /* check normal bins */
2195 for (i
= 1; i
< NBINS
; ++i
)
2199 /* binmap is accurate (except for bin 1 == unsorted_chunks) */
2202 unsigned int binbit
= get_binmap (av
, i
);
2203 int empty
= last (b
) == b
;
2210 for (p
= last (b
); p
!= b
; p
= p
->bk
)
2212 /* each chunk claims to be free */
2213 do_check_free_chunk (av
, p
);
2214 size
= chunksize (p
);
2218 /* chunk belongs in bin */
2219 idx
= bin_index (size
);
2221 /* lists are sorted */
2222 assert (p
->bk
== b
||
2223 (unsigned long) chunksize (p
->bk
) >= (unsigned long) chunksize (p
));
2225 if (!in_smallbin_range (size
))
2227 if (p
->fd_nextsize
!= NULL
)
2229 if (p
->fd_nextsize
== p
)
2230 assert (p
->bk_nextsize
== p
);
2233 if (p
->fd_nextsize
== first (b
))
2234 assert (chunksize (p
) < chunksize (p
->fd_nextsize
));
2236 assert (chunksize (p
) > chunksize (p
->fd_nextsize
));
2239 assert (chunksize (p
) > chunksize (p
->bk_nextsize
));
2241 assert (chunksize (p
) < chunksize (p
->bk_nextsize
));
2245 assert (p
->bk_nextsize
== NULL
);
2248 else if (!in_smallbin_range (size
))
2249 assert (p
->fd_nextsize
== NULL
&& p
->bk_nextsize
== NULL
);
2250 /* chunk is followed by a legal chain of inuse chunks */
2251 for (q
= next_chunk (p
);
2252 (q
!= av
->top
&& inuse (q
) &&
2253 (unsigned long) (chunksize (q
)) >= MINSIZE
);
2255 do_check_inuse_chunk (av
, q
);
2259 /* top chunk is OK */
2260 check_chunk (av
, av
->top
);
2265 /* ----------------- Support for debugging hooks -------------------- */
2269 /* ----------- Routines dealing with system allocation -------------- */
2272 sysmalloc handles malloc cases requiring more memory from the system.
2273 On entry, it is assumed that av->top does not have enough
2274 space to service request for nb bytes, thus requiring that av->top
2275 be extended or replaced.
2279 sysmalloc (INTERNAL_SIZE_T nb
, mstate av
)
2281 mchunkptr old_top
; /* incoming value of av->top */
2282 INTERNAL_SIZE_T old_size
; /* its size */
2283 char *old_end
; /* its end address */
2285 long size
; /* arg to first MORECORE or mmap call */
2286 char *brk
; /* return value from MORECORE */
2288 long correction
; /* arg to 2nd MORECORE call */
2289 char *snd_brk
; /* 2nd return val */
2291 INTERNAL_SIZE_T front_misalign
; /* unusable bytes at front of new space */
2292 INTERNAL_SIZE_T end_misalign
; /* partial page left at end of new space */
2293 char *aligned_brk
; /* aligned offset into brk */
2295 mchunkptr p
; /* the allocated/returned chunk */
2296 mchunkptr remainder
; /* remainder from allocation */
2297 unsigned long remainder_size
; /* its size */
2300 size_t pagesize
= GLRO (dl_pagesize
);
2301 bool tried_mmap
= false;
2305 If have mmap, and the request size meets the mmap threshold, and
2306 the system supports mmap, and there are few enough currently
2307 allocated mmapped regions, try to directly map this request
2308 rather than expanding top.
2312 || ((unsigned long) (nb
) >= (unsigned long) (mp_
.mmap_threshold
)
2313 && (mp_
.n_mmaps
< mp_
.n_mmaps_max
)))
2315 char *mm
; /* return value from mmap call*/
2319 Round up size to nearest page. For mmapped chunks, the overhead
2320 is one SIZE_SZ unit larger than for normal chunks, because there
2321 is no following chunk whose prev_size field could be used.
2323 See the front_misalign handling below, for glibc there is no
2324 need for further alignments unless we have have high alignment.
2326 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2327 size
= ALIGN_UP (nb
+ SIZE_SZ
, pagesize
);
2329 size
= ALIGN_UP (nb
+ SIZE_SZ
+ MALLOC_ALIGN_MASK
, pagesize
);
2332 /* Don't try if size wraps around 0 */
2333 if ((unsigned long) (size
) > (unsigned long) (nb
))
2335 mm
= (char *) (MMAP (0, size
, PROT_READ
| PROT_WRITE
, 0));
2337 if (mm
!= MAP_FAILED
)
2340 The offset to the start of the mmapped region is stored
2341 in the prev_size field of the chunk. This allows us to adjust
2342 returned start address to meet alignment requirements here
2343 and in memalign(), and still be able to compute proper
2344 address argument for later munmap in free() and realloc().
2347 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2349 /* For glibc, chunk2mem increases the address by 2*SIZE_SZ and
2350 MALLOC_ALIGN_MASK is 2*SIZE_SZ-1. Each mmap'ed area is page
2351 aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. */
2352 assert (((INTERNAL_SIZE_T
) chunk2mem (mm
) & MALLOC_ALIGN_MASK
) == 0);
2356 front_misalign
= (INTERNAL_SIZE_T
) chunk2mem (mm
) & MALLOC_ALIGN_MASK
;
2357 if (front_misalign
> 0)
2359 correction
= MALLOC_ALIGNMENT
- front_misalign
;
2360 p
= (mchunkptr
) (mm
+ correction
);
2361 set_prev_size (p
, correction
);
2362 set_head (p
, (size
- correction
) | IS_MMAPPED
);
2367 set_prev_size (p
, 0);
2368 set_head (p
, size
| IS_MMAPPED
);
2371 /* update statistics */
2373 int new = atomic_exchange_and_add (&mp_
.n_mmaps
, 1) + 1;
2374 atomic_max (&mp_
.max_n_mmaps
, new);
2377 sum
= atomic_exchange_and_add (&mp_
.mmapped_mem
, size
) + size
;
2378 atomic_max (&mp_
.max_mmapped_mem
, sum
);
2380 check_chunk (av
, p
);
2382 return chunk2mem (p
);
2387 /* There are no usable arenas and mmap also failed. */
2391 /* Record incoming configuration of top */
2394 old_size
= chunksize (old_top
);
2395 old_end
= (char *) (chunk_at_offset (old_top
, old_size
));
2397 brk
= snd_brk
= (char *) (MORECORE_FAILURE
);
2400 If not the first time through, we require old_size to be
2401 at least MINSIZE and to have prev_inuse set.
2404 assert ((old_top
== initial_top (av
) && old_size
== 0) ||
2405 ((unsigned long) (old_size
) >= MINSIZE
&&
2406 prev_inuse (old_top
) &&
2407 ((unsigned long) old_end
& (pagesize
- 1)) == 0));
2409 /* Precondition: not enough current space to satisfy nb request */
2410 assert ((unsigned long) (old_size
) < (unsigned long) (nb
+ MINSIZE
));
2413 if (av
!= &main_arena
)
2415 heap_info
*old_heap
, *heap
;
2416 size_t old_heap_size
;
2418 /* First try to extend the current heap. */
2419 old_heap
= heap_for_ptr (old_top
);
2420 old_heap_size
= old_heap
->size
;
2421 if ((long) (MINSIZE
+ nb
- old_size
) > 0
2422 && grow_heap (old_heap
, MINSIZE
+ nb
- old_size
) == 0)
2424 av
->system_mem
+= old_heap
->size
- old_heap_size
;
2425 set_head (old_top
, (((char *) old_heap
+ old_heap
->size
) - (char *) old_top
)
2428 else if ((heap
= new_heap (nb
+ (MINSIZE
+ sizeof (*heap
)), mp_
.top_pad
)))
2430 /* Use a newly allocated heap. */
2432 heap
->prev
= old_heap
;
2433 av
->system_mem
+= heap
->size
;
2434 /* Set up the new top. */
2435 top (av
) = chunk_at_offset (heap
, sizeof (*heap
));
2436 set_head (top (av
), (heap
->size
- sizeof (*heap
)) | PREV_INUSE
);
2438 /* Setup fencepost and free the old top chunk with a multiple of
2439 MALLOC_ALIGNMENT in size. */
2440 /* The fencepost takes at least MINSIZE bytes, because it might
2441 become the top chunk again later. Note that a footer is set
2442 up, too, although the chunk is marked in use. */
2443 old_size
= (old_size
- MINSIZE
) & ~MALLOC_ALIGN_MASK
;
2444 set_head (chunk_at_offset (old_top
, old_size
+ 2 * SIZE_SZ
), 0 | PREV_INUSE
);
2445 if (old_size
>= MINSIZE
)
2447 set_head (chunk_at_offset (old_top
, old_size
), (2 * SIZE_SZ
) | PREV_INUSE
);
2448 set_foot (chunk_at_offset (old_top
, old_size
), (2 * SIZE_SZ
));
2449 set_head (old_top
, old_size
| PREV_INUSE
| NON_MAIN_ARENA
);
2450 _int_free (av
, old_top
, 1);
2454 set_head (old_top
, (old_size
+ 2 * SIZE_SZ
) | PREV_INUSE
);
2455 set_foot (old_top
, (old_size
+ 2 * SIZE_SZ
));
2458 else if (!tried_mmap
)
2459 /* We can at least try to use to mmap memory. */
2462 else /* av == main_arena */
2465 { /* Request enough space for nb + pad + overhead */
2466 size
= nb
+ mp_
.top_pad
+ MINSIZE
;
2469 If contiguous, we can subtract out existing space that we hope to
2470 combine with new space. We add it back later only if
2471 we don't actually get contiguous space.
2474 if (contiguous (av
))
2478 Round to a multiple of page size.
2479 If MORECORE is not contiguous, this ensures that we only call it
2480 with whole-page arguments. And if MORECORE is contiguous and
2481 this is not first time through, this preserves page-alignment of
2482 previous calls. Otherwise, we correct to page-align below.
2485 size
= ALIGN_UP (size
, pagesize
);
2488 Don't try to call MORECORE if argument is so big as to appear
2489 negative. Note that since mmap takes size_t arg, it may succeed
2490 below even if we cannot call MORECORE.
2495 brk
= (char *) (MORECORE (size
));
2496 LIBC_PROBE (memory_sbrk_more
, 2, brk
, size
);
2499 if (brk
!= (char *) (MORECORE_FAILURE
))
2501 /* Call the `morecore' hook if necessary. */
2502 void (*hook
) (void) = atomic_forced_read (__after_morecore_hook
);
2503 if (__builtin_expect (hook
!= NULL
, 0))
2509 If have mmap, try using it as a backup when MORECORE fails or
2510 cannot be used. This is worth doing on systems that have "holes" in
2511 address space, so sbrk cannot extend to give contiguous space, but
2512 space is available elsewhere. Note that we ignore mmap max count
2513 and threshold limits, since the space will not be used as a
2514 segregated mmap region.
2517 /* Cannot merge with old top, so add its size back in */
2518 if (contiguous (av
))
2519 size
= ALIGN_UP (size
+ old_size
, pagesize
);
2521 /* If we are relying on mmap as backup, then use larger units */
2522 if ((unsigned long) (size
) < (unsigned long) (MMAP_AS_MORECORE_SIZE
))
2523 size
= MMAP_AS_MORECORE_SIZE
;
2525 /* Don't try if size wraps around 0 */
2526 if ((unsigned long) (size
) > (unsigned long) (nb
))
2528 char *mbrk
= (char *) (MMAP (0, size
, PROT_READ
| PROT_WRITE
, 0));
2530 if (mbrk
!= MAP_FAILED
)
2532 /* We do not need, and cannot use, another sbrk call to find end */
2534 snd_brk
= brk
+ size
;
2537 Record that we no longer have a contiguous sbrk region.
2538 After the first time mmap is used as backup, we do not
2539 ever rely on contiguous space since this could incorrectly
2542 set_noncontiguous (av
);
2547 if (brk
!= (char *) (MORECORE_FAILURE
))
2549 if (mp_
.sbrk_base
== 0)
2550 mp_
.sbrk_base
= brk
;
2551 av
->system_mem
+= size
;
2554 If MORECORE extends previous space, we can likewise extend top size.
2557 if (brk
== old_end
&& snd_brk
== (char *) (MORECORE_FAILURE
))
2558 set_head (old_top
, (size
+ old_size
) | PREV_INUSE
);
2560 else if (contiguous (av
) && old_size
&& brk
< old_end
)
2561 /* Oops! Someone else killed our space.. Can't touch anything. */
2562 malloc_printerr ("break adjusted to free malloc space");
2565 Otherwise, make adjustments:
2567 * If the first time through or noncontiguous, we need to call sbrk
2568 just to find out where the end of memory lies.
2570 * We need to ensure that all returned chunks from malloc will meet
2573 * If there was an intervening foreign sbrk, we need to adjust sbrk
2574 request size to account for fact that we will not be able to
2575 combine new space with existing space in old_top.
2577 * Almost all systems internally allocate whole pages at a time, in
2578 which case we might as well use the whole last page of request.
2579 So we allocate enough more memory to hit a page boundary now,
2580 which in turn causes future contiguous calls to page-align.
2590 /* handle contiguous cases */
2591 if (contiguous (av
))
2593 /* Count foreign sbrk as system_mem. */
2595 av
->system_mem
+= brk
- old_end
;
2597 /* Guarantee alignment of first new chunk made from this space */
2599 front_misalign
= (INTERNAL_SIZE_T
) chunk2mem (brk
) & MALLOC_ALIGN_MASK
;
2600 if (front_misalign
> 0)
2603 Skip over some bytes to arrive at an aligned position.
2604 We don't need to specially mark these wasted front bytes.
2605 They will never be accessed anyway because
2606 prev_inuse of av->top (and any chunk created from its start)
2607 is always true after initialization.
2610 correction
= MALLOC_ALIGNMENT
- front_misalign
;
2611 aligned_brk
+= correction
;
2615 If this isn't adjacent to existing space, then we will not
2616 be able to merge with old_top space, so must add to 2nd request.
2619 correction
+= old_size
;
2621 /* Extend the end address to hit a page boundary */
2622 end_misalign
= (INTERNAL_SIZE_T
) (brk
+ size
+ correction
);
2623 correction
+= (ALIGN_UP (end_misalign
, pagesize
)) - end_misalign
;
2625 assert (correction
>= 0);
2626 snd_brk
= (char *) (MORECORE (correction
));
2629 If can't allocate correction, try to at least find out current
2630 brk. It might be enough to proceed without failing.
2632 Note that if second sbrk did NOT fail, we assume that space
2633 is contiguous with first sbrk. This is a safe assumption unless
2634 program is multithreaded but doesn't use locks and a foreign sbrk
2635 occurred between our first and second calls.
2638 if (snd_brk
== (char *) (MORECORE_FAILURE
))
2641 snd_brk
= (char *) (MORECORE (0));
2645 /* Call the `morecore' hook if necessary. */
2646 void (*hook
) (void) = atomic_forced_read (__after_morecore_hook
);
2647 if (__builtin_expect (hook
!= NULL
, 0))
2652 /* handle non-contiguous cases */
2655 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2656 /* MORECORE/mmap must correctly align */
2657 assert (((unsigned long) chunk2mem (brk
) & MALLOC_ALIGN_MASK
) == 0);
2660 front_misalign
= (INTERNAL_SIZE_T
) chunk2mem (brk
) & MALLOC_ALIGN_MASK
;
2661 if (front_misalign
> 0)
2664 Skip over some bytes to arrive at an aligned position.
2665 We don't need to specially mark these wasted front bytes.
2666 They will never be accessed anyway because
2667 prev_inuse of av->top (and any chunk created from its start)
2668 is always true after initialization.
2671 aligned_brk
+= MALLOC_ALIGNMENT
- front_misalign
;
2675 /* Find out current end of memory */
2676 if (snd_brk
== (char *) (MORECORE_FAILURE
))
2678 snd_brk
= (char *) (MORECORE (0));
2682 /* Adjust top based on results of second sbrk */
2683 if (snd_brk
!= (char *) (MORECORE_FAILURE
))
2685 av
->top
= (mchunkptr
) aligned_brk
;
2686 set_head (av
->top
, (snd_brk
- aligned_brk
+ correction
) | PREV_INUSE
);
2687 av
->system_mem
+= correction
;
2690 If not the first time through, we either have a
2691 gap due to foreign sbrk or a non-contiguous region. Insert a
2692 double fencepost at old_top to prevent consolidation with space
2693 we don't own. These fenceposts are artificial chunks that are
2694 marked as inuse and are in any case too small to use. We need
2695 two to make sizes and alignments work out.
2701 Shrink old_top to insert fenceposts, keeping size a
2702 multiple of MALLOC_ALIGNMENT. We know there is at least
2703 enough space in old_top to do this.
2705 old_size
= (old_size
- 4 * SIZE_SZ
) & ~MALLOC_ALIGN_MASK
;
2706 set_head (old_top
, old_size
| PREV_INUSE
);
2709 Note that the following assignments completely overwrite
2710 old_top when old_size was previously MINSIZE. This is
2711 intentional. We need the fencepost, even if old_top otherwise gets
2714 set_head (chunk_at_offset (old_top
, old_size
),
2715 (2 * SIZE_SZ
) | PREV_INUSE
);
2716 set_head (chunk_at_offset (old_top
, old_size
+ 2 * SIZE_SZ
),
2717 (2 * SIZE_SZ
) | PREV_INUSE
);
2719 /* If possible, release the rest. */
2720 if (old_size
>= MINSIZE
)
2722 _int_free (av
, old_top
, 1);
2728 } /* if (av != &main_arena) */
2730 if ((unsigned long) av
->system_mem
> (unsigned long) (av
->max_system_mem
))
2731 av
->max_system_mem
= av
->system_mem
;
2732 check_malloc_state (av
);
2734 /* finally, do the allocation */
2736 size
= chunksize (p
);
2738 /* check that one of the above allocation paths succeeded */
2739 if ((unsigned long) (size
) >= (unsigned long) (nb
+ MINSIZE
))
2741 remainder_size
= size
- nb
;
2742 remainder
= chunk_at_offset (p
, nb
);
2743 av
->top
= remainder
;
2744 set_head (p
, nb
| PREV_INUSE
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
2745 set_head (remainder
, remainder_size
| PREV_INUSE
);
2746 check_malloced_chunk (av
, p
, nb
);
2747 return chunk2mem (p
);
2750 /* catch all failure paths */
2751 __set_errno (ENOMEM
);
2757 systrim is an inverse of sorts to sysmalloc. It gives memory back
2758 to the system (via negative arguments to sbrk) if there is unused
2759 memory at the `high' end of the malloc pool. It is called
2760 automatically by free() when top space exceeds the trim
2761 threshold. It is also called by the public malloc_trim routine. It
2762 returns 1 if it actually released any memory, else 0.
2766 systrim (size_t pad
, mstate av
)
2768 long top_size
; /* Amount of top-most memory */
2769 long extra
; /* Amount to release */
2770 long released
; /* Amount actually released */
2771 char *current_brk
; /* address returned by pre-check sbrk call */
2772 char *new_brk
; /* address returned by post-check sbrk call */
2776 pagesize
= GLRO (dl_pagesize
);
2777 top_size
= chunksize (av
->top
);
2779 top_area
= top_size
- MINSIZE
- 1;
2780 if (top_area
<= pad
)
2783 /* Release in pagesize units and round down to the nearest page. */
2784 extra
= ALIGN_DOWN(top_area
- pad
, pagesize
);
2790 Only proceed if end of memory is where we last set it.
2791 This avoids problems if there were foreign sbrk calls.
2793 current_brk
= (char *) (MORECORE (0));
2794 if (current_brk
== (char *) (av
->top
) + top_size
)
2797 Attempt to release memory. We ignore MORECORE return value,
2798 and instead call again to find out where new end of memory is.
2799 This avoids problems if first call releases less than we asked,
2800 of if failure somehow altered brk value. (We could still
2801 encounter problems if it altered brk in some very bad way,
2802 but the only thing we can do is adjust anyway, which will cause
2803 some downstream failure.)
2807 /* Call the `morecore' hook if necessary. */
2808 void (*hook
) (void) = atomic_forced_read (__after_morecore_hook
);
2809 if (__builtin_expect (hook
!= NULL
, 0))
2811 new_brk
= (char *) (MORECORE (0));
2813 LIBC_PROBE (memory_sbrk_less
, 2, new_brk
, extra
);
2815 if (new_brk
!= (char *) MORECORE_FAILURE
)
2817 released
= (long) (current_brk
- new_brk
);
2821 /* Success. Adjust top. */
2822 av
->system_mem
-= released
;
2823 set_head (av
->top
, (top_size
- released
) | PREV_INUSE
);
2824 check_malloc_state (av
);
2834 munmap_chunk (mchunkptr p
)
2836 INTERNAL_SIZE_T size
= chunksize (p
);
2838 assert (chunk_is_mmapped (p
));
2840 /* Do nothing if the chunk is a faked mmapped chunk in the dumped
2841 main arena. We never free this memory. */
2842 if (DUMPED_MAIN_ARENA_CHUNK (p
))
2845 uintptr_t block
= (uintptr_t) p
- prev_size (p
);
2846 size_t total_size
= prev_size (p
) + size
;
2847 /* Unfortunately we have to do the compilers job by hand here. Normally
2848 we would test BLOCK and TOTAL-SIZE separately for compliance with the
2849 page size. But gcc does not recognize the optimization possibility
2850 (in the moment at least) so we combine the two values into one before
2852 if (__builtin_expect (((block
| total_size
) & (GLRO (dl_pagesize
) - 1)) != 0, 0))
2853 malloc_printerr ("munmap_chunk(): invalid pointer");
2855 atomic_decrement (&mp_
.n_mmaps
);
2856 atomic_add (&mp_
.mmapped_mem
, -total_size
);
2858 /* If munmap failed the process virtual memory address space is in a
2859 bad shape. Just leave the block hanging around, the process will
2860 terminate shortly anyway since not much can be done. */
2861 __munmap ((char *) block
, total_size
);
2868 mremap_chunk (mchunkptr p
, size_t new_size
)
2870 size_t pagesize
= GLRO (dl_pagesize
);
2871 INTERNAL_SIZE_T offset
= prev_size (p
);
2872 INTERNAL_SIZE_T size
= chunksize (p
);
2875 assert (chunk_is_mmapped (p
));
2876 assert (((size
+ offset
) & (GLRO (dl_pagesize
) - 1)) == 0);
2878 /* Note the extra SIZE_SZ overhead as in mmap_chunk(). */
2879 new_size
= ALIGN_UP (new_size
+ offset
+ SIZE_SZ
, pagesize
);
2881 /* No need to remap if the number of pages does not change. */
2882 if (size
+ offset
== new_size
)
2885 cp
= (char *) __mremap ((char *) p
- offset
, size
+ offset
, new_size
,
2888 if (cp
== MAP_FAILED
)
2891 p
= (mchunkptr
) (cp
+ offset
);
2893 assert (aligned_OK (chunk2mem (p
)));
2895 assert (prev_size (p
) == offset
);
2896 set_head (p
, (new_size
- offset
) | IS_MMAPPED
);
2898 INTERNAL_SIZE_T
new;
2899 new = atomic_exchange_and_add (&mp_
.mmapped_mem
, new_size
- size
- offset
)
2900 + new_size
- size
- offset
;
2901 atomic_max (&mp_
.max_mmapped_mem
, new);
2904 #endif /* HAVE_MREMAP */
2906 /*------------------------ Public wrappers. --------------------------------*/
2910 /* We overlay this structure on the user-data portion of a chunk when
2911 the chunk is stored in the per-thread cache. */
2912 typedef struct tcache_entry
2914 struct tcache_entry
*next
;
2917 /* There is one of these for each thread, which contains the
2918 per-thread cache (hence "tcache_perthread_struct"). Keeping
2919 overall size low is mildly important. Note that COUNTS and ENTRIES
2920 are redundant (we could have just counted the linked list each
2921 time), this is for performance reasons. */
2922 typedef struct tcache_perthread_struct
2924 char counts
[TCACHE_MAX_BINS
];
2925 tcache_entry
*entries
[TCACHE_MAX_BINS
];
2926 } tcache_perthread_struct
;
2928 static __thread
char tcache_shutting_down
= 0;
2929 static __thread tcache_perthread_struct
*tcache
= NULL
;
2931 /* Caller must ensure that we know tc_idx is valid and there's room
2934 tcache_put (mchunkptr chunk
, size_t tc_idx
)
2936 tcache_entry
*e
= (tcache_entry
*) chunk2mem (chunk
);
2937 assert (tc_idx
< TCACHE_MAX_BINS
);
2938 e
->next
= tcache
->entries
[tc_idx
];
2939 tcache
->entries
[tc_idx
] = e
;
2940 ++(tcache
->counts
[tc_idx
]);
2943 /* Caller must ensure that we know tc_idx is valid and there's
2944 available chunks to remove. */
2946 tcache_get (size_t tc_idx
)
2948 tcache_entry
*e
= tcache
->entries
[tc_idx
];
2949 assert (tc_idx
< TCACHE_MAX_BINS
);
2950 assert (tcache
->entries
[tc_idx
] > 0);
2951 tcache
->entries
[tc_idx
] = e
->next
;
2952 --(tcache
->counts
[tc_idx
]);
2956 static void __attribute__ ((section ("__libc_thread_freeres_fn")))
2957 tcache_thread_freeres (void)
2960 tcache_perthread_struct
*tcache_tmp
= tcache
;
2967 for (i
= 0; i
< TCACHE_MAX_BINS
; ++i
)
2969 while (tcache_tmp
->entries
[i
])
2971 tcache_entry
*e
= tcache_tmp
->entries
[i
];
2972 tcache_tmp
->entries
[i
] = e
->next
;
2977 __libc_free (tcache_tmp
);
2979 tcache_shutting_down
= 1;
2981 text_set_element (__libc_thread_subfreeres
, tcache_thread_freeres
);
2988 const size_t bytes
= sizeof (tcache_perthread_struct
);
2990 if (tcache_shutting_down
)
2993 arena_get (ar_ptr
, bytes
);
2994 victim
= _int_malloc (ar_ptr
, bytes
);
2995 if (!victim
&& ar_ptr
!= NULL
)
2997 ar_ptr
= arena_get_retry (ar_ptr
, bytes
);
2998 victim
= _int_malloc (ar_ptr
, bytes
);
3003 __libc_lock_unlock (ar_ptr
->mutex
);
3005 /* In a low memory situation, we may not be able to allocate memory
3006 - in which case, we just keep trying later. However, we
3007 typically do this very early, so either there is sufficient
3008 memory, or there isn't enough memory to do non-trivial
3009 allocations anyway. */
3012 tcache
= (tcache_perthread_struct
*) victim
;
3013 memset (tcache
, 0, sizeof (tcache_perthread_struct
));
3018 #define MAYBE_INIT_TCACHE() \
3019 if (__glibc_unlikely (tcache == NULL)) \
3023 #define MAYBE_INIT_TCACHE()
3027 __libc_malloc (size_t bytes
)
3032 void *(*hook
) (size_t, const void *)
3033 = atomic_forced_read (__malloc_hook
);
3034 if (__builtin_expect (hook
!= NULL
, 0))
3035 return (*hook
)(bytes
, RETURN_ADDRESS (0));
3037 /* int_free also calls request2size, be careful to not pad twice. */
3038 size_t tbytes
= request2size (bytes
);
3039 size_t tc_idx
= csize2tidx (tbytes
);
3041 MAYBE_INIT_TCACHE ();
3043 DIAG_PUSH_NEEDS_COMMENT
;
3044 if (tc_idx
< mp_
.tcache_bins
3045 /*&& tc_idx < TCACHE_MAX_BINS*/ /* to appease gcc */
3047 && tcache
->entries
[tc_idx
] != NULL
)
3049 return tcache_get (tc_idx
);
3051 DIAG_POP_NEEDS_COMMENT
;
3054 arena_get (ar_ptr
, bytes
);
3056 victim
= _int_malloc (ar_ptr
, bytes
);
3057 /* Retry with another arena only if we were able to find a usable arena
3059 if (!victim
&& ar_ptr
!= NULL
)
3061 LIBC_PROBE (memory_malloc_retry
, 1, bytes
);
3062 ar_ptr
= arena_get_retry (ar_ptr
, bytes
);
3063 victim
= _int_malloc (ar_ptr
, bytes
);
3067 __libc_lock_unlock (ar_ptr
->mutex
);
3069 assert (!victim
|| chunk_is_mmapped (mem2chunk (victim
)) ||
3070 ar_ptr
== arena_for_chunk (mem2chunk (victim
)));
3073 libc_hidden_def (__libc_malloc
)
3076 __libc_free (void *mem
)
3079 mchunkptr p
; /* chunk corresponding to mem */
3081 void (*hook
) (void *, const void *)
3082 = atomic_forced_read (__free_hook
);
3083 if (__builtin_expect (hook
!= NULL
, 0))
3085 (*hook
)(mem
, RETURN_ADDRESS (0));
3089 if (mem
== 0) /* free(0) has no effect */
3092 p
= mem2chunk (mem
);
3094 if (chunk_is_mmapped (p
)) /* release mmapped memory. */
3096 /* See if the dynamic brk/mmap threshold needs adjusting.
3097 Dumped fake mmapped chunks do not affect the threshold. */
3098 if (!mp_
.no_dyn_threshold
3099 && chunksize_nomask (p
) > mp_
.mmap_threshold
3100 && chunksize_nomask (p
) <= DEFAULT_MMAP_THRESHOLD_MAX
3101 && !DUMPED_MAIN_ARENA_CHUNK (p
))
3103 mp_
.mmap_threshold
= chunksize (p
);
3104 mp_
.trim_threshold
= 2 * mp_
.mmap_threshold
;
3105 LIBC_PROBE (memory_mallopt_free_dyn_thresholds
, 2,
3106 mp_
.mmap_threshold
, mp_
.trim_threshold
);
3112 MAYBE_INIT_TCACHE ();
3114 ar_ptr
= arena_for_chunk (p
);
3115 _int_free (ar_ptr
, p
, 0);
3117 libc_hidden_def (__libc_free
)
3120 __libc_realloc (void *oldmem
, size_t bytes
)
3123 INTERNAL_SIZE_T nb
; /* padded request size */
3125 void *newp
; /* chunk to return */
3127 void *(*hook
) (void *, size_t, const void *) =
3128 atomic_forced_read (__realloc_hook
);
3129 if (__builtin_expect (hook
!= NULL
, 0))
3130 return (*hook
)(oldmem
, bytes
, RETURN_ADDRESS (0));
3132 #if REALLOC_ZERO_BYTES_FREES
3133 if (bytes
== 0 && oldmem
!= NULL
)
3135 __libc_free (oldmem
); return 0;
3139 /* realloc of null is supposed to be same as malloc */
3141 return __libc_malloc (bytes
);
3143 /* chunk corresponding to oldmem */
3144 const mchunkptr oldp
= mem2chunk (oldmem
);
3146 const INTERNAL_SIZE_T oldsize
= chunksize (oldp
);
3148 if (chunk_is_mmapped (oldp
))
3152 MAYBE_INIT_TCACHE ();
3153 ar_ptr
= arena_for_chunk (oldp
);
3156 /* Little security check which won't hurt performance: the allocator
3157 never wrapps around at the end of the address space. Therefore
3158 we can exclude some size values which might appear here by
3159 accident or by "design" from some intruder. We need to bypass
3160 this check for dumped fake mmap chunks from the old main arena
3161 because the new malloc may provide additional alignment. */
3162 if ((__builtin_expect ((uintptr_t) oldp
> (uintptr_t) -oldsize
, 0)
3163 || __builtin_expect (misaligned_chunk (oldp
), 0))
3164 && !DUMPED_MAIN_ARENA_CHUNK (oldp
))
3165 malloc_printerr ("realloc(): invalid pointer");
3167 checked_request2size (bytes
, nb
);
3169 if (chunk_is_mmapped (oldp
))
3171 /* If this is a faked mmapped chunk from the dumped main arena,
3172 always make a copy (and do not free the old chunk). */
3173 if (DUMPED_MAIN_ARENA_CHUNK (oldp
))
3175 /* Must alloc, copy, free. */
3176 void *newmem
= __libc_malloc (bytes
);
3179 /* Copy as many bytes as are available from the old chunk
3180 and fit into the new size. NB: The overhead for faked
3181 mmapped chunks is only SIZE_SZ, not 2 * SIZE_SZ as for
3182 regular mmapped chunks. */
3183 if (bytes
> oldsize
- SIZE_SZ
)
3184 bytes
= oldsize
- SIZE_SZ
;
3185 memcpy (newmem
, oldmem
, bytes
);
3192 newp
= mremap_chunk (oldp
, nb
);
3194 return chunk2mem (newp
);
3196 /* Note the extra SIZE_SZ overhead. */
3197 if (oldsize
- SIZE_SZ
>= nb
)
3198 return oldmem
; /* do nothing */
3200 /* Must alloc, copy, free. */
3201 newmem
= __libc_malloc (bytes
);
3203 return 0; /* propagate failure */
3205 memcpy (newmem
, oldmem
, oldsize
- 2 * SIZE_SZ
);
3206 munmap_chunk (oldp
);
3210 __libc_lock_lock (ar_ptr
->mutex
);
3212 newp
= _int_realloc (ar_ptr
, oldp
, oldsize
, nb
);
3214 __libc_lock_unlock (ar_ptr
->mutex
);
3215 assert (!newp
|| chunk_is_mmapped (mem2chunk (newp
)) ||
3216 ar_ptr
== arena_for_chunk (mem2chunk (newp
)));
3220 /* Try harder to allocate memory in other arenas. */
3221 LIBC_PROBE (memory_realloc_retry
, 2, bytes
, oldmem
);
3222 newp
= __libc_malloc (bytes
);
3225 memcpy (newp
, oldmem
, oldsize
- SIZE_SZ
);
3226 _int_free (ar_ptr
, oldp
, 0);
3232 libc_hidden_def (__libc_realloc
)
3235 __libc_memalign (size_t alignment
, size_t bytes
)
3237 void *address
= RETURN_ADDRESS (0);
3238 return _mid_memalign (alignment
, bytes
, address
);
3242 _mid_memalign (size_t alignment
, size_t bytes
, void *address
)
3247 void *(*hook
) (size_t, size_t, const void *) =
3248 atomic_forced_read (__memalign_hook
);
3249 if (__builtin_expect (hook
!= NULL
, 0))
3250 return (*hook
)(alignment
, bytes
, address
);
3252 /* If we need less alignment than we give anyway, just relay to malloc. */
3253 if (alignment
<= MALLOC_ALIGNMENT
)
3254 return __libc_malloc (bytes
);
3256 /* Otherwise, ensure that it is at least a minimum chunk size */
3257 if (alignment
< MINSIZE
)
3258 alignment
= MINSIZE
;
3260 /* If the alignment is greater than SIZE_MAX / 2 + 1 it cannot be a
3261 power of 2 and will cause overflow in the check below. */
3262 if (alignment
> SIZE_MAX
/ 2 + 1)
3264 __set_errno (EINVAL
);
3268 /* Check for overflow. */
3269 if (bytes
> SIZE_MAX
- alignment
- MINSIZE
)
3271 __set_errno (ENOMEM
);
3276 /* Make sure alignment is power of 2. */
3277 if (!powerof2 (alignment
))
3279 size_t a
= MALLOC_ALIGNMENT
* 2;
3280 while (a
< alignment
)
3285 arena_get (ar_ptr
, bytes
+ alignment
+ MINSIZE
);
3287 p
= _int_memalign (ar_ptr
, alignment
, bytes
);
3288 if (!p
&& ar_ptr
!= NULL
)
3290 LIBC_PROBE (memory_memalign_retry
, 2, bytes
, alignment
);
3291 ar_ptr
= arena_get_retry (ar_ptr
, bytes
);
3292 p
= _int_memalign (ar_ptr
, alignment
, bytes
);
3296 __libc_lock_unlock (ar_ptr
->mutex
);
3298 assert (!p
|| chunk_is_mmapped (mem2chunk (p
)) ||
3299 ar_ptr
== arena_for_chunk (mem2chunk (p
)));
3303 weak_alias (__libc_memalign
, aligned_alloc
)
3304 libc_hidden_def (__libc_memalign
)
3307 __libc_valloc (size_t bytes
)
3309 if (__malloc_initialized
< 0)
3312 void *address
= RETURN_ADDRESS (0);
3313 size_t pagesize
= GLRO (dl_pagesize
);
3314 return _mid_memalign (pagesize
, bytes
, address
);
3318 __libc_pvalloc (size_t bytes
)
3320 if (__malloc_initialized
< 0)
3323 void *address
= RETURN_ADDRESS (0);
3324 size_t pagesize
= GLRO (dl_pagesize
);
3325 size_t rounded_bytes
= ALIGN_UP (bytes
, pagesize
);
3327 /* Check for overflow. */
3328 if (bytes
> SIZE_MAX
- 2 * pagesize
- MINSIZE
)
3330 __set_errno (ENOMEM
);
3334 return _mid_memalign (pagesize
, rounded_bytes
, address
);
3338 __libc_calloc (size_t n
, size_t elem_size
)
3341 mchunkptr oldtop
, p
;
3342 INTERNAL_SIZE_T bytes
, sz
, csz
, oldtopsize
;
3344 unsigned long clearsize
;
3345 unsigned long nclears
;
3348 /* size_t is unsigned so the behavior on overflow is defined. */
3349 bytes
= n
* elem_size
;
3350 #define HALF_INTERNAL_SIZE_T \
3351 (((INTERNAL_SIZE_T) 1) << (8 * sizeof (INTERNAL_SIZE_T) / 2))
3352 if (__builtin_expect ((n
| elem_size
) >= HALF_INTERNAL_SIZE_T
, 0))
3354 if (elem_size
!= 0 && bytes
/ elem_size
!= n
)
3356 __set_errno (ENOMEM
);
3361 void *(*hook
) (size_t, const void *) =
3362 atomic_forced_read (__malloc_hook
);
3363 if (__builtin_expect (hook
!= NULL
, 0))
3366 mem
= (*hook
)(sz
, RETURN_ADDRESS (0));
3370 return memset (mem
, 0, sz
);
3375 MAYBE_INIT_TCACHE ();
3380 /* Check if we hand out the top chunk, in which case there may be no
3384 oldtopsize
= chunksize (top (av
));
3385 # if MORECORE_CLEARS < 2
3386 /* Only newly allocated memory is guaranteed to be cleared. */
3387 if (av
== &main_arena
&&
3388 oldtopsize
< mp_
.sbrk_base
+ av
->max_system_mem
- (char *) oldtop
)
3389 oldtopsize
= (mp_
.sbrk_base
+ av
->max_system_mem
- (char *) oldtop
);
3391 if (av
!= &main_arena
)
3393 heap_info
*heap
= heap_for_ptr (oldtop
);
3394 if (oldtopsize
< (char *) heap
+ heap
->mprotect_size
- (char *) oldtop
)
3395 oldtopsize
= (char *) heap
+ heap
->mprotect_size
- (char *) oldtop
;
3401 /* No usable arenas. */
3405 mem
= _int_malloc (av
, sz
);
3408 assert (!mem
|| chunk_is_mmapped (mem2chunk (mem
)) ||
3409 av
== arena_for_chunk (mem2chunk (mem
)));
3411 if (mem
== 0 && av
!= NULL
)
3413 LIBC_PROBE (memory_calloc_retry
, 1, sz
);
3414 av
= arena_get_retry (av
, sz
);
3415 mem
= _int_malloc (av
, sz
);
3419 __libc_lock_unlock (av
->mutex
);
3421 /* Allocation failed even after a retry. */
3425 p
= mem2chunk (mem
);
3427 /* Two optional cases in which clearing not necessary */
3428 if (chunk_is_mmapped (p
))
3430 if (__builtin_expect (perturb_byte
, 0))
3431 return memset (mem
, 0, sz
);
3436 csz
= chunksize (p
);
3439 if (perturb_byte
== 0 && (p
== oldtop
&& csz
> oldtopsize
))
3441 /* clear only the bytes from non-freshly-sbrked memory */
3446 /* Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that
3447 contents have an odd number of INTERNAL_SIZE_T-sized words;
3449 d
= (INTERNAL_SIZE_T
*) mem
;
3450 clearsize
= csz
- SIZE_SZ
;
3451 nclears
= clearsize
/ sizeof (INTERNAL_SIZE_T
);
3452 assert (nclears
>= 3);
3455 return memset (d
, 0, clearsize
);
3483 ------------------------------ malloc ------------------------------
3487 _int_malloc (mstate av
, size_t bytes
)
3489 INTERNAL_SIZE_T nb
; /* normalized request size */
3490 unsigned int idx
; /* associated bin index */
3491 mbinptr bin
; /* associated bin */
3493 mchunkptr victim
; /* inspected/selected chunk */
3494 INTERNAL_SIZE_T size
; /* its size */
3495 int victim_index
; /* its bin index */
3497 mchunkptr remainder
; /* remainder from a split */
3498 unsigned long remainder_size
; /* its size */
3500 unsigned int block
; /* bit map traverser */
3501 unsigned int bit
; /* bit map traverser */
3502 unsigned int map
; /* current word of binmap */
3504 mchunkptr fwd
; /* misc temp for linking */
3505 mchunkptr bck
; /* misc temp for linking */
3508 size_t tcache_unsorted_count
; /* count of unsorted chunks processed */
3512 Convert request size to internal form by adding SIZE_SZ bytes
3513 overhead plus possibly more to obtain necessary alignment and/or
3514 to obtain a size of at least MINSIZE, the smallest allocatable
3515 size. Also, checked_request2size traps (returning 0) request sizes
3516 that are so large that they wrap around zero when padded and
3520 checked_request2size (bytes
, nb
);
3522 /* There are no usable arenas. Fall back to sysmalloc to get a chunk from
3524 if (__glibc_unlikely (av
== NULL
))
3526 void *p
= sysmalloc (nb
, av
);
3528 alloc_perturb (p
, bytes
);
3533 If the size qualifies as a fastbin, first check corresponding bin.
3534 This code is safe to execute even if av is not yet initialized, so we
3535 can try it without checking, which saves some time on this fast path.
3538 #define REMOVE_FB(fb, victim, pp) \
3542 if (victim == NULL) \
3545 while ((pp = catomic_compare_and_exchange_val_acq (fb, victim->fd, victim)) \
3548 if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ()))
3550 idx
= fastbin_index (nb
);
3551 mfastbinptr
*fb
= &fastbin (av
, idx
);
3553 REMOVE_FB (fb
, victim
, pp
);
3556 if (__builtin_expect (fastbin_index (chunksize (victim
)) != idx
, 0))
3557 malloc_printerr ("malloc(): memory corruption (fast)");
3558 check_remalloced_chunk (av
, victim
, nb
);
3560 /* While we're here, if we see other chunks of the same size,
3561 stash them in the tcache. */
3562 size_t tc_idx
= csize2tidx (nb
);
3563 if (tcache
&& tc_idx
< mp_
.tcache_bins
)
3565 mchunkptr tc_victim
;
3567 /* While bin not empty and tcache not full, copy chunks over. */
3568 while (tcache
->counts
[tc_idx
] < mp_
.tcache_count
3569 && (pp
= *fb
) != NULL
)
3571 REMOVE_FB (fb
, tc_victim
, pp
);
3574 tcache_put (tc_victim
, tc_idx
);
3579 void *p
= chunk2mem (victim
);
3580 alloc_perturb (p
, bytes
);
3586 If a small request, check regular bin. Since these "smallbins"
3587 hold one size each, no searching within bins is necessary.
3588 (For a large request, we need to wait until unsorted chunks are
3589 processed to find best fit. But for small ones, fits are exact
3590 anyway, so we can check now, which is faster.)
3593 if (in_smallbin_range (nb
))
3595 idx
= smallbin_index (nb
);
3596 bin
= bin_at (av
, idx
);
3598 if ((victim
= last (bin
)) != bin
)
3600 if (victim
== 0) /* initialization check */
3601 malloc_consolidate (av
);
3605 if (__glibc_unlikely (bck
->fd
!= victim
))
3607 ("malloc(): smallbin double linked list corrupted");
3608 set_inuse_bit_at_offset (victim
, nb
);
3612 if (av
!= &main_arena
)
3613 set_non_main_arena (victim
);
3614 check_malloced_chunk (av
, victim
, nb
);
3616 /* While we're here, if we see other chunks of the same size,
3617 stash them in the tcache. */
3618 size_t tc_idx
= csize2tidx (nb
);
3619 if (tcache
&& tc_idx
< mp_
.tcache_bins
)
3621 mchunkptr tc_victim
;
3623 /* While bin not empty and tcache not full, copy chunks over. */
3624 while (tcache
->counts
[tc_idx
] < mp_
.tcache_count
3625 && (tc_victim
= last (bin
)) != bin
)
3629 bck
= tc_victim
->bk
;
3630 set_inuse_bit_at_offset (tc_victim
, nb
);
3631 if (av
!= &main_arena
)
3632 set_non_main_arena (tc_victim
);
3636 tcache_put (tc_victim
, tc_idx
);
3641 void *p
= chunk2mem (victim
);
3642 alloc_perturb (p
, bytes
);
3649 If this is a large request, consolidate fastbins before continuing.
3650 While it might look excessive to kill all fastbins before
3651 even seeing if there is space available, this avoids
3652 fragmentation problems normally associated with fastbins.
3653 Also, in practice, programs tend to have runs of either small or
3654 large requests, but less often mixtures, so consolidation is not
3655 invoked all that often in most programs. And the programs that
3656 it is called frequently in otherwise tend to fragment.
3661 idx
= largebin_index (nb
);
3662 if (have_fastchunks (av
))
3663 malloc_consolidate (av
);
3667 Process recently freed or remaindered chunks, taking one only if
3668 it is exact fit, or, if this a small request, the chunk is remainder from
3669 the most recent non-exact fit. Place other traversed chunks in
3670 bins. Note that this step is the only place in any routine where
3671 chunks are placed in bins.
3673 The outer loop here is needed because we might not realize until
3674 near the end of malloc that we should have consolidated, so must
3675 do so and retry. This happens at most once, and only when we would
3676 otherwise need to expand memory to service a "small" request.
3680 INTERNAL_SIZE_T tcache_nb
= 0;
3681 size_t tc_idx
= csize2tidx (nb
);
3682 if (tcache
&& tc_idx
< mp_
.tcache_bins
)
3684 int return_cached
= 0;
3686 tcache_unsorted_count
= 0;
3692 while ((victim
= unsorted_chunks (av
)->bk
) != unsorted_chunks (av
))
3695 if (__builtin_expect (chunksize_nomask (victim
) <= 2 * SIZE_SZ
, 0)
3696 || __builtin_expect (chunksize_nomask (victim
)
3697 > av
->system_mem
, 0))
3698 malloc_printerr ("malloc(): memory corruption");
3699 size
= chunksize (victim
);
3702 If a small request, try to use last remainder if it is the
3703 only chunk in unsorted bin. This helps promote locality for
3704 runs of consecutive small requests. This is the only
3705 exception to best-fit, and applies only when there is
3706 no exact fit for a small chunk.
3709 if (in_smallbin_range (nb
) &&
3710 bck
== unsorted_chunks (av
) &&
3711 victim
== av
->last_remainder
&&
3712 (unsigned long) (size
) > (unsigned long) (nb
+ MINSIZE
))
3714 /* split and reattach remainder */
3715 remainder_size
= size
- nb
;
3716 remainder
= chunk_at_offset (victim
, nb
);
3717 unsorted_chunks (av
)->bk
= unsorted_chunks (av
)->fd
= remainder
;
3718 av
->last_remainder
= remainder
;
3719 remainder
->bk
= remainder
->fd
= unsorted_chunks (av
);
3720 if (!in_smallbin_range (remainder_size
))
3722 remainder
->fd_nextsize
= NULL
;
3723 remainder
->bk_nextsize
= NULL
;
3726 set_head (victim
, nb
| PREV_INUSE
|
3727 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3728 set_head (remainder
, remainder_size
| PREV_INUSE
);
3729 set_foot (remainder
, remainder_size
);
3731 check_malloced_chunk (av
, victim
, nb
);
3732 void *p
= chunk2mem (victim
);
3733 alloc_perturb (p
, bytes
);
3737 /* remove from unsorted list */
3738 unsorted_chunks (av
)->bk
= bck
;
3739 bck
->fd
= unsorted_chunks (av
);
3741 /* Take now instead of binning if exact fit */
3745 set_inuse_bit_at_offset (victim
, size
);
3746 if (av
!= &main_arena
)
3747 set_non_main_arena (victim
);
3749 /* Fill cache first, return to user only if cache fills.
3750 We may return one of these chunks later. */
3752 && tcache
->counts
[tc_idx
] < mp_
.tcache_count
)
3754 tcache_put (victim
, tc_idx
);
3761 check_malloced_chunk (av
, victim
, nb
);
3762 void *p
= chunk2mem (victim
);
3763 alloc_perturb (p
, bytes
);
3770 /* place chunk in bin */
3772 if (in_smallbin_range (size
))
3774 victim_index
= smallbin_index (size
);
3775 bck
= bin_at (av
, victim_index
);
3780 victim_index
= largebin_index (size
);
3781 bck
= bin_at (av
, victim_index
);
3784 /* maintain large bins in sorted order */
3787 /* Or with inuse bit to speed comparisons */
3789 /* if smaller than smallest, bypass loop below */
3790 assert (chunk_main_arena (bck
->bk
));
3791 if ((unsigned long) (size
)
3792 < (unsigned long) chunksize_nomask (bck
->bk
))
3797 victim
->fd_nextsize
= fwd
->fd
;
3798 victim
->bk_nextsize
= fwd
->fd
->bk_nextsize
;
3799 fwd
->fd
->bk_nextsize
= victim
->bk_nextsize
->fd_nextsize
= victim
;
3803 assert (chunk_main_arena (fwd
));
3804 while ((unsigned long) size
< chunksize_nomask (fwd
))
3806 fwd
= fwd
->fd_nextsize
;
3807 assert (chunk_main_arena (fwd
));
3810 if ((unsigned long) size
3811 == (unsigned long) chunksize_nomask (fwd
))
3812 /* Always insert in the second position. */
3816 victim
->fd_nextsize
= fwd
;
3817 victim
->bk_nextsize
= fwd
->bk_nextsize
;
3818 fwd
->bk_nextsize
= victim
;
3819 victim
->bk_nextsize
->fd_nextsize
= victim
;
3825 victim
->fd_nextsize
= victim
->bk_nextsize
= victim
;
3828 mark_bin (av
, victim_index
);
3835 /* If we've processed as many chunks as we're allowed while
3836 filling the cache, return one of the cached ones. */
3837 ++tcache_unsorted_count
;
3839 && mp_
.tcache_unsorted_limit
> 0
3840 && tcache_unsorted_count
> mp_
.tcache_unsorted_limit
)
3842 return tcache_get (tc_idx
);
3846 #define MAX_ITERS 10000
3847 if (++iters
>= MAX_ITERS
)
3852 /* If all the small chunks we found ended up cached, return one now. */
3855 return tcache_get (tc_idx
);
3860 If a large request, scan through the chunks of current bin in
3861 sorted order to find smallest that fits. Use the skip list for this.
3864 if (!in_smallbin_range (nb
))
3866 bin
= bin_at (av
, idx
);
3868 /* skip scan if empty or largest chunk is too small */
3869 if ((victim
= first (bin
)) != bin
3870 && (unsigned long) chunksize_nomask (victim
)
3871 >= (unsigned long) (nb
))
3873 victim
= victim
->bk_nextsize
;
3874 while (((unsigned long) (size
= chunksize (victim
)) <
3875 (unsigned long) (nb
)))
3876 victim
= victim
->bk_nextsize
;
3878 /* Avoid removing the first entry for a size so that the skip
3879 list does not have to be rerouted. */
3880 if (victim
!= last (bin
)
3881 && chunksize_nomask (victim
)
3882 == chunksize_nomask (victim
->fd
))
3883 victim
= victim
->fd
;
3885 remainder_size
= size
- nb
;
3886 unlink (av
, victim
, bck
, fwd
);
3889 if (remainder_size
< MINSIZE
)
3891 set_inuse_bit_at_offset (victim
, size
);
3892 if (av
!= &main_arena
)
3893 set_non_main_arena (victim
);
3898 remainder
= chunk_at_offset (victim
, nb
);
3899 /* We cannot assume the unsorted list is empty and therefore
3900 have to perform a complete insert here. */
3901 bck
= unsorted_chunks (av
);
3903 if (__glibc_unlikely (fwd
->bk
!= bck
))
3904 malloc_printerr ("malloc(): corrupted unsorted chunks");
3905 remainder
->bk
= bck
;
3906 remainder
->fd
= fwd
;
3907 bck
->fd
= remainder
;
3908 fwd
->bk
= remainder
;
3909 if (!in_smallbin_range (remainder_size
))
3911 remainder
->fd_nextsize
= NULL
;
3912 remainder
->bk_nextsize
= NULL
;
3914 set_head (victim
, nb
| PREV_INUSE
|
3915 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3916 set_head (remainder
, remainder_size
| PREV_INUSE
);
3917 set_foot (remainder
, remainder_size
);
3919 check_malloced_chunk (av
, victim
, nb
);
3920 void *p
= chunk2mem (victim
);
3921 alloc_perturb (p
, bytes
);
3927 Search for a chunk by scanning bins, starting with next largest
3928 bin. This search is strictly by best-fit; i.e., the smallest
3929 (with ties going to approximately the least recently used) chunk
3930 that fits is selected.
3932 The bitmap avoids needing to check that most blocks are nonempty.
3933 The particular case of skipping all bins during warm-up phases
3934 when no chunks have been returned yet is faster than it might look.
3938 bin
= bin_at (av
, idx
);
3939 block
= idx2block (idx
);
3940 map
= av
->binmap
[block
];
3941 bit
= idx2bit (idx
);
3945 /* Skip rest of block if there are no more set bits in this block. */
3946 if (bit
> map
|| bit
== 0)
3950 if (++block
>= BINMAPSIZE
) /* out of bins */
3953 while ((map
= av
->binmap
[block
]) == 0);
3955 bin
= bin_at (av
, (block
<< BINMAPSHIFT
));
3959 /* Advance to bin with set bit. There must be one. */
3960 while ((bit
& map
) == 0)
3962 bin
= next_bin (bin
);
3967 /* Inspect the bin. It is likely to be non-empty */
3968 victim
= last (bin
);
3970 /* If a false alarm (empty bin), clear the bit. */
3973 av
->binmap
[block
] = map
&= ~bit
; /* Write through */
3974 bin
= next_bin (bin
);
3980 size
= chunksize (victim
);
3982 /* We know the first chunk in this bin is big enough to use. */
3983 assert ((unsigned long) (size
) >= (unsigned long) (nb
));
3985 remainder_size
= size
- nb
;
3988 unlink (av
, victim
, bck
, fwd
);
3991 if (remainder_size
< MINSIZE
)
3993 set_inuse_bit_at_offset (victim
, size
);
3994 if (av
!= &main_arena
)
3995 set_non_main_arena (victim
);
4001 remainder
= chunk_at_offset (victim
, nb
);
4003 /* We cannot assume the unsorted list is empty and therefore
4004 have to perform a complete insert here. */
4005 bck
= unsorted_chunks (av
);
4007 if (__glibc_unlikely (fwd
->bk
!= bck
))
4008 malloc_printerr ("malloc(): corrupted unsorted chunks 2");
4009 remainder
->bk
= bck
;
4010 remainder
->fd
= fwd
;
4011 bck
->fd
= remainder
;
4012 fwd
->bk
= remainder
;
4014 /* advertise as last remainder */
4015 if (in_smallbin_range (nb
))
4016 av
->last_remainder
= remainder
;
4017 if (!in_smallbin_range (remainder_size
))
4019 remainder
->fd_nextsize
= NULL
;
4020 remainder
->bk_nextsize
= NULL
;
4022 set_head (victim
, nb
| PREV_INUSE
|
4023 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4024 set_head (remainder
, remainder_size
| PREV_INUSE
);
4025 set_foot (remainder
, remainder_size
);
4027 check_malloced_chunk (av
, victim
, nb
);
4028 void *p
= chunk2mem (victim
);
4029 alloc_perturb (p
, bytes
);
4036 If large enough, split off the chunk bordering the end of memory
4037 (held in av->top). Note that this is in accord with the best-fit
4038 search rule. In effect, av->top is treated as larger (and thus
4039 less well fitting) than any other available chunk since it can
4040 be extended to be as large as necessary (up to system
4043 We require that av->top always exists (i.e., has size >=
4044 MINSIZE) after initialization, so if it would otherwise be
4045 exhausted by current request, it is replenished. (The main
4046 reason for ensuring it exists is that we may need MINSIZE space
4047 to put in fenceposts in sysmalloc.)
4051 size
= chunksize (victim
);
4053 if ((unsigned long) (size
) >= (unsigned long) (nb
+ MINSIZE
))
4055 remainder_size
= size
- nb
;
4056 remainder
= chunk_at_offset (victim
, nb
);
4057 av
->top
= remainder
;
4058 set_head (victim
, nb
| PREV_INUSE
|
4059 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4060 set_head (remainder
, remainder_size
| PREV_INUSE
);
4062 check_malloced_chunk (av
, victim
, nb
);
4063 void *p
= chunk2mem (victim
);
4064 alloc_perturb (p
, bytes
);
4068 /* When we are using atomic ops to free fast chunks we can get
4069 here for all block sizes. */
4070 else if (have_fastchunks (av
))
4072 malloc_consolidate (av
);
4073 /* restore original bin index */
4074 if (in_smallbin_range (nb
))
4075 idx
= smallbin_index (nb
);
4077 idx
= largebin_index (nb
);
4081 Otherwise, relay to handle system-dependent cases
4085 void *p
= sysmalloc (nb
, av
);
4087 alloc_perturb (p
, bytes
);
4094 ------------------------------ free ------------------------------
4098 _int_free (mstate av
, mchunkptr p
, int have_lock
)
4100 INTERNAL_SIZE_T size
; /* its size */
4101 mfastbinptr
*fb
; /* associated fastbin */
4102 mchunkptr nextchunk
; /* next contiguous chunk */
4103 INTERNAL_SIZE_T nextsize
; /* its size */
4104 int nextinuse
; /* true if nextchunk is used */
4105 INTERNAL_SIZE_T prevsize
; /* size of previous contiguous chunk */
4106 mchunkptr bck
; /* misc temp for linking */
4107 mchunkptr fwd
; /* misc temp for linking */
4111 size
= chunksize (p
);
4113 /* Little security check which won't hurt performance: the
4114 allocator never wrapps around at the end of the address space.
4115 Therefore we can exclude some size values which might appear
4116 here by accident or by "design" from some intruder. */
4117 if (__builtin_expect ((uintptr_t) p
> (uintptr_t) -size
, 0)
4118 || __builtin_expect (misaligned_chunk (p
), 0))
4119 malloc_printerr ("free(): invalid pointer");
4120 /* We know that each chunk is at least MINSIZE bytes in size or a
4121 multiple of MALLOC_ALIGNMENT. */
4122 if (__glibc_unlikely (size
< MINSIZE
|| !aligned_OK (size
)))
4123 malloc_printerr ("free(): invalid size");
4125 check_inuse_chunk(av
, p
);
4129 size_t tc_idx
= csize2tidx (size
);
4132 && tc_idx
< mp_
.tcache_bins
4133 && tcache
->counts
[tc_idx
] < mp_
.tcache_count
)
4135 tcache_put (p
, tc_idx
);
4142 If eligible, place chunk on a fastbin so it can be found
4143 and used quickly in malloc.
4146 if ((unsigned long)(size
) <= (unsigned long)(get_max_fast ())
4150 If TRIM_FASTBINS set, don't place chunks
4151 bordering top into fastbins
4153 && (chunk_at_offset(p
, size
) != av
->top
)
4157 if (__builtin_expect (chunksize_nomask (chunk_at_offset (p
, size
))
4159 || __builtin_expect (chunksize (chunk_at_offset (p
, size
))
4160 >= av
->system_mem
, 0))
4162 /* We might not have a lock at this point and concurrent modifications
4163 of system_mem might have let to a false positive. Redo the test
4164 after getting the lock. */
4166 || ({ assert (locked
== 0);
4167 __libc_lock_lock (av
->mutex
);
4169 chunksize_nomask (chunk_at_offset (p
, size
)) <= 2 * SIZE_SZ
4170 || chunksize (chunk_at_offset (p
, size
)) >= av
->system_mem
;
4172 malloc_printerr ("free(): invalid next size (fast)");
4175 __libc_lock_unlock (av
->mutex
);
4180 free_perturb (chunk2mem(p
), size
- 2 * SIZE_SZ
);
4183 unsigned int idx
= fastbin_index(size
);
4184 fb
= &fastbin (av
, idx
);
4186 /* Atomically link P to its fastbin: P->FD = *FB; *FB = P; */
4187 mchunkptr old
= *fb
, old2
;
4188 unsigned int old_idx
= ~0u;
4191 /* Check that the top of the bin is not the record we are going to add
4192 (i.e., double free). */
4193 if (__builtin_expect (old
== p
, 0))
4194 malloc_printerr ("double free or corruption (fasttop)");
4195 /* Check that size of fastbin chunk at the top is the same as
4196 size of the chunk that we are adding. We can dereference OLD
4197 only if we have the lock, otherwise it might have already been
4198 deallocated. See use of OLD_IDX below for the actual check. */
4199 if (have_lock
&& old
!= NULL
)
4200 old_idx
= fastbin_index(chunksize(old
));
4203 while ((old
= catomic_compare_and_exchange_val_rel (fb
, p
, old2
)) != old2
);
4205 if (have_lock
&& old
!= NULL
&& __builtin_expect (old_idx
!= idx
, 0))
4206 malloc_printerr ("invalid fastbin entry (free)");
4210 Consolidate other non-mmapped chunks as they arrive.
4213 else if (!chunk_is_mmapped(p
)) {
4215 __libc_lock_lock (av
->mutex
);
4219 nextchunk
= chunk_at_offset(p
, size
);
4221 /* Lightweight tests: check whether the block is already the
4223 if (__glibc_unlikely (p
== av
->top
))
4224 malloc_printerr ("double free or corruption (top)");
4225 /* Or whether the next chunk is beyond the boundaries of the arena. */
4226 if (__builtin_expect (contiguous (av
)
4227 && (char *) nextchunk
4228 >= ((char *) av
->top
+ chunksize(av
->top
)), 0))
4229 malloc_printerr ("double free or corruption (out)");
4230 /* Or whether the block is actually not marked used. */
4231 if (__glibc_unlikely (!prev_inuse(nextchunk
)))
4232 malloc_printerr ("double free or corruption (!prev)");
4234 nextsize
= chunksize(nextchunk
);
4235 if (__builtin_expect (chunksize_nomask (nextchunk
) <= 2 * SIZE_SZ
, 0)
4236 || __builtin_expect (nextsize
>= av
->system_mem
, 0))
4237 malloc_printerr ("free(): invalid next size (normal)");
4239 free_perturb (chunk2mem(p
), size
- 2 * SIZE_SZ
);
4241 /* consolidate backward */
4242 if (!prev_inuse(p
)) {
4243 prevsize
= prev_size (p
);
4245 p
= chunk_at_offset(p
, -((long) prevsize
));
4246 unlink(av
, p
, bck
, fwd
);
4249 if (nextchunk
!= av
->top
) {
4250 /* get and clear inuse bit */
4251 nextinuse
= inuse_bit_at_offset(nextchunk
, nextsize
);
4253 /* consolidate forward */
4255 unlink(av
, nextchunk
, bck
, fwd
);
4258 clear_inuse_bit_at_offset(nextchunk
, 0);
4261 Place the chunk in unsorted chunk list. Chunks are
4262 not placed into regular bins until after they have
4263 been given one chance to be used in malloc.
4266 bck
= unsorted_chunks(av
);
4268 if (__glibc_unlikely (fwd
->bk
!= bck
))
4269 malloc_printerr ("free(): corrupted unsorted chunks");
4272 if (!in_smallbin_range(size
))
4274 p
->fd_nextsize
= NULL
;
4275 p
->bk_nextsize
= NULL
;
4280 set_head(p
, size
| PREV_INUSE
);
4283 check_free_chunk(av
, p
);
4287 If the chunk borders the current high end of memory,
4288 consolidate into top
4293 set_head(p
, size
| PREV_INUSE
);
4299 If freeing a large space, consolidate possibly-surrounding
4300 chunks. Then, if the total unused topmost memory exceeds trim
4301 threshold, ask malloc_trim to reduce top.
4303 Unless max_fast is 0, we don't know if there are fastbins
4304 bordering top, so we cannot tell for sure whether threshold
4305 has been reached unless fastbins are consolidated. But we
4306 don't want to consolidate on each free. As a compromise,
4307 consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
4311 if ((unsigned long)(size
) >= FASTBIN_CONSOLIDATION_THRESHOLD
) {
4312 if (have_fastchunks(av
))
4313 malloc_consolidate(av
);
4315 if (av
== &main_arena
) {
4316 #ifndef MORECORE_CANNOT_TRIM
4317 if ((unsigned long)(chunksize(av
->top
)) >=
4318 (unsigned long)(mp_
.trim_threshold
))
4319 systrim(mp_
.top_pad
, av
);
4322 /* Always try heap_trim(), even if the top chunk is not
4323 large, because the corresponding heap might go away. */
4324 heap_info
*heap
= heap_for_ptr(top(av
));
4326 assert(heap
->ar_ptr
== av
);
4327 heap_trim(heap
, mp_
.top_pad
);
4333 __libc_lock_unlock (av
->mutex
);
4337 If the chunk was allocated via mmap, release via munmap().
4346 ------------------------- malloc_consolidate -------------------------
4348 malloc_consolidate is a specialized version of free() that tears
4349 down chunks held in fastbins. Free itself cannot be used for this
4350 purpose since, among other things, it might place chunks back onto
4351 fastbins. So, instead, we need to use a minor variant of the same
4354 Also, because this routine needs to be called the first time through
4355 malloc anyway, it turns out to be the perfect place to trigger
4356 initialization code.
4359 static void malloc_consolidate(mstate av
)
4361 mfastbinptr
* fb
; /* current fastbin being consolidated */
4362 mfastbinptr
* maxfb
; /* last fastbin (for loop control) */
4363 mchunkptr p
; /* current chunk being consolidated */
4364 mchunkptr nextp
; /* next chunk to consolidate */
4365 mchunkptr unsorted_bin
; /* bin header */
4366 mchunkptr first_unsorted
; /* chunk to link to */
4368 /* These have same use as in free() */
4369 mchunkptr nextchunk
;
4370 INTERNAL_SIZE_T size
;
4371 INTERNAL_SIZE_T nextsize
;
4372 INTERNAL_SIZE_T prevsize
;
4378 If max_fast is 0, we know that av hasn't
4379 yet been initialized, in which case do so below
4382 if (get_max_fast () != 0) {
4383 clear_fastchunks(av
);
4385 unsorted_bin
= unsorted_chunks(av
);
4388 Remove each chunk from fast bin and consolidate it, placing it
4389 then in unsorted bin. Among other reasons for doing this,
4390 placing in unsorted bin avoids needing to calculate actual bins
4391 until malloc is sure that chunks aren't immediately going to be
4395 maxfb
= &fastbin (av
, NFASTBINS
- 1);
4396 fb
= &fastbin (av
, 0);
4398 p
= atomic_exchange_acq (fb
, NULL
);
4401 check_inuse_chunk(av
, p
);
4404 /* Slightly streamlined version of consolidation code in free() */
4405 size
= chunksize (p
);
4406 nextchunk
= chunk_at_offset(p
, size
);
4407 nextsize
= chunksize(nextchunk
);
4409 if (!prev_inuse(p
)) {
4410 prevsize
= prev_size (p
);
4412 p
= chunk_at_offset(p
, -((long) prevsize
));
4413 unlink(av
, p
, bck
, fwd
);
4416 if (nextchunk
!= av
->top
) {
4417 nextinuse
= inuse_bit_at_offset(nextchunk
, nextsize
);
4421 unlink(av
, nextchunk
, bck
, fwd
);
4423 clear_inuse_bit_at_offset(nextchunk
, 0);
4425 first_unsorted
= unsorted_bin
->fd
;
4426 unsorted_bin
->fd
= p
;
4427 first_unsorted
->bk
= p
;
4429 if (!in_smallbin_range (size
)) {
4430 p
->fd_nextsize
= NULL
;
4431 p
->bk_nextsize
= NULL
;
4434 set_head(p
, size
| PREV_INUSE
);
4435 p
->bk
= unsorted_bin
;
4436 p
->fd
= first_unsorted
;
4442 set_head(p
, size
| PREV_INUSE
);
4446 } while ( (p
= nextp
) != 0);
4449 } while (fb
++ != maxfb
);
4452 malloc_init_state(av
);
4453 check_malloc_state(av
);
4458 ------------------------------ realloc ------------------------------
4462 _int_realloc(mstate av
, mchunkptr oldp
, INTERNAL_SIZE_T oldsize
,
4465 mchunkptr newp
; /* chunk to return */
4466 INTERNAL_SIZE_T newsize
; /* its size */
4467 void* newmem
; /* corresponding user mem */
4469 mchunkptr next
; /* next contiguous chunk after oldp */
4471 mchunkptr remainder
; /* extra space at end of newp */
4472 unsigned long remainder_size
; /* its size */
4474 mchunkptr bck
; /* misc temp for linking */
4475 mchunkptr fwd
; /* misc temp for linking */
4477 unsigned long copysize
; /* bytes to copy */
4478 unsigned int ncopies
; /* INTERNAL_SIZE_T words to copy */
4479 INTERNAL_SIZE_T
* s
; /* copy source */
4480 INTERNAL_SIZE_T
* d
; /* copy destination */
4483 if (__builtin_expect (chunksize_nomask (oldp
) <= 2 * SIZE_SZ
, 0)
4484 || __builtin_expect (oldsize
>= av
->system_mem
, 0))
4485 malloc_printerr ("realloc(): invalid old size");
4487 check_inuse_chunk (av
, oldp
);
4489 /* All callers already filter out mmap'ed chunks. */
4490 assert (!chunk_is_mmapped (oldp
));
4492 next
= chunk_at_offset (oldp
, oldsize
);
4493 INTERNAL_SIZE_T nextsize
= chunksize (next
);
4494 if (__builtin_expect (chunksize_nomask (next
) <= 2 * SIZE_SZ
, 0)
4495 || __builtin_expect (nextsize
>= av
->system_mem
, 0))
4496 malloc_printerr ("realloc(): invalid next size");
4498 if ((unsigned long) (oldsize
) >= (unsigned long) (nb
))
4500 /* already big enough; split below */
4507 /* Try to expand forward into top */
4508 if (next
== av
->top
&&
4509 (unsigned long) (newsize
= oldsize
+ nextsize
) >=
4510 (unsigned long) (nb
+ MINSIZE
))
4512 set_head_size (oldp
, nb
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4513 av
->top
= chunk_at_offset (oldp
, nb
);
4514 set_head (av
->top
, (newsize
- nb
) | PREV_INUSE
);
4515 check_inuse_chunk (av
, oldp
);
4516 return chunk2mem (oldp
);
4519 /* Try to expand forward into next chunk; split off remainder below */
4520 else if (next
!= av
->top
&&
4522 (unsigned long) (newsize
= oldsize
+ nextsize
) >=
4523 (unsigned long) (nb
))
4526 unlink (av
, next
, bck
, fwd
);
4529 /* allocate, copy, free */
4532 newmem
= _int_malloc (av
, nb
- MALLOC_ALIGN_MASK
);
4534 return 0; /* propagate failure */
4536 newp
= mem2chunk (newmem
);
4537 newsize
= chunksize (newp
);
4540 Avoid copy if newp is next chunk after oldp.
4550 Unroll copy of <= 36 bytes (72 if 8byte sizes)
4551 We know that contents have an odd number of
4552 INTERNAL_SIZE_T-sized words; minimally 3.
4555 copysize
= oldsize
- SIZE_SZ
;
4556 s
= (INTERNAL_SIZE_T
*) (chunk2mem (oldp
));
4557 d
= (INTERNAL_SIZE_T
*) (newmem
);
4558 ncopies
= copysize
/ sizeof (INTERNAL_SIZE_T
);
4559 assert (ncopies
>= 3);
4562 memcpy (d
, s
, copysize
);
4566 *(d
+ 0) = *(s
+ 0);
4567 *(d
+ 1) = *(s
+ 1);
4568 *(d
+ 2) = *(s
+ 2);
4571 *(d
+ 3) = *(s
+ 3);
4572 *(d
+ 4) = *(s
+ 4);
4575 *(d
+ 5) = *(s
+ 5);
4576 *(d
+ 6) = *(s
+ 6);
4579 *(d
+ 7) = *(s
+ 7);
4580 *(d
+ 8) = *(s
+ 8);
4586 _int_free (av
, oldp
, 1);
4587 check_inuse_chunk (av
, newp
);
4588 return chunk2mem (newp
);
4593 /* If possible, free extra space in old or extended chunk */
4595 assert ((unsigned long) (newsize
) >= (unsigned long) (nb
));
4597 remainder_size
= newsize
- nb
;
4599 if (remainder_size
< MINSIZE
) /* not enough extra to split off */
4601 set_head_size (newp
, newsize
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4602 set_inuse_bit_at_offset (newp
, newsize
);
4604 else /* split remainder */
4606 remainder
= chunk_at_offset (newp
, nb
);
4607 set_head_size (newp
, nb
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4608 set_head (remainder
, remainder_size
| PREV_INUSE
|
4609 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4610 /* Mark remainder as inuse so free() won't complain */
4611 set_inuse_bit_at_offset (remainder
, remainder_size
);
4612 _int_free (av
, remainder
, 1);
4615 check_inuse_chunk (av
, newp
);
4616 return chunk2mem (newp
);
4620 ------------------------------ memalign ------------------------------
4624 _int_memalign (mstate av
, size_t alignment
, size_t bytes
)
4626 INTERNAL_SIZE_T nb
; /* padded request size */
4627 char *m
; /* memory returned by malloc call */
4628 mchunkptr p
; /* corresponding chunk */
4629 char *brk
; /* alignment point within p */
4630 mchunkptr newp
; /* chunk to return */
4631 INTERNAL_SIZE_T newsize
; /* its size */
4632 INTERNAL_SIZE_T leadsize
; /* leading space before alignment point */
4633 mchunkptr remainder
; /* spare room at end to split off */
4634 unsigned long remainder_size
; /* its size */
4635 INTERNAL_SIZE_T size
;
4639 checked_request2size (bytes
, nb
);
4642 Strategy: find a spot within that chunk that meets the alignment
4643 request, and then possibly free the leading and trailing space.
4647 /* Call malloc with worst case padding to hit alignment. */
4649 m
= (char *) (_int_malloc (av
, nb
+ alignment
+ MINSIZE
));
4652 return 0; /* propagate failure */
4656 if ((((unsigned long) (m
)) % alignment
) != 0) /* misaligned */
4659 Find an aligned spot inside chunk. Since we need to give back
4660 leading space in a chunk of at least MINSIZE, if the first
4661 calculation places us at a spot with less than MINSIZE leader,
4662 we can move to the next aligned spot -- we've allocated enough
4663 total room so that this is always possible.
4665 brk
= (char *) mem2chunk (((unsigned long) (m
+ alignment
- 1)) &
4666 - ((signed long) alignment
));
4667 if ((unsigned long) (brk
- (char *) (p
)) < MINSIZE
)
4670 newp
= (mchunkptr
) brk
;
4671 leadsize
= brk
- (char *) (p
);
4672 newsize
= chunksize (p
) - leadsize
;
4674 /* For mmapped chunks, just adjust offset */
4675 if (chunk_is_mmapped (p
))
4677 set_prev_size (newp
, prev_size (p
) + leadsize
);
4678 set_head (newp
, newsize
| IS_MMAPPED
);
4679 return chunk2mem (newp
);
4682 /* Otherwise, give back leader, use the rest */
4683 set_head (newp
, newsize
| PREV_INUSE
|
4684 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4685 set_inuse_bit_at_offset (newp
, newsize
);
4686 set_head_size (p
, leadsize
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4687 _int_free (av
, p
, 1);
4690 assert (newsize
>= nb
&&
4691 (((unsigned long) (chunk2mem (p
))) % alignment
) == 0);
4694 /* Also give back spare room at the end */
4695 if (!chunk_is_mmapped (p
))
4697 size
= chunksize (p
);
4698 if ((unsigned long) (size
) > (unsigned long) (nb
+ MINSIZE
))
4700 remainder_size
= size
- nb
;
4701 remainder
= chunk_at_offset (p
, nb
);
4702 set_head (remainder
, remainder_size
| PREV_INUSE
|
4703 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4704 set_head_size (p
, nb
);
4705 _int_free (av
, remainder
, 1);
4709 check_inuse_chunk (av
, p
);
4710 return chunk2mem (p
);
4715 ------------------------------ malloc_trim ------------------------------
4719 mtrim (mstate av
, size_t pad
)
4721 /* Ensure initialization/consolidation */
4722 malloc_consolidate (av
);
4724 const size_t ps
= GLRO (dl_pagesize
);
4725 int psindex
= bin_index (ps
);
4726 const size_t psm1
= ps
- 1;
4729 for (int i
= 1; i
< NBINS
; ++i
)
4730 if (i
== 1 || i
>= psindex
)
4732 mbinptr bin
= bin_at (av
, i
);
4734 for (mchunkptr p
= last (bin
); p
!= bin
; p
= p
->bk
)
4736 INTERNAL_SIZE_T size
= chunksize (p
);
4738 if (size
> psm1
+ sizeof (struct malloc_chunk
))
4740 /* See whether the chunk contains at least one unused page. */
4741 char *paligned_mem
= (char *) (((uintptr_t) p
4742 + sizeof (struct malloc_chunk
)
4745 assert ((char *) chunk2mem (p
) + 4 * SIZE_SZ
<= paligned_mem
);
4746 assert ((char *) p
+ size
> paligned_mem
);
4748 /* This is the size we could potentially free. */
4749 size
-= paligned_mem
- (char *) p
;
4754 /* When debugging we simulate destroying the memory
4756 memset (paligned_mem
, 0x89, size
& ~psm1
);
4758 __madvise (paligned_mem
, size
& ~psm1
, MADV_DONTNEED
);
4766 #ifndef MORECORE_CANNOT_TRIM
4767 return result
| (av
== &main_arena
? systrim (pad
, av
) : 0);
4776 __malloc_trim (size_t s
)
4780 if (__malloc_initialized
< 0)
4783 mstate ar_ptr
= &main_arena
;
4786 __libc_lock_lock (ar_ptr
->mutex
);
4787 result
|= mtrim (ar_ptr
, s
);
4788 __libc_lock_unlock (ar_ptr
->mutex
);
4790 ar_ptr
= ar_ptr
->next
;
4792 while (ar_ptr
!= &main_arena
);
4799 ------------------------- malloc_usable_size -------------------------
4808 p
= mem2chunk (mem
);
4810 if (__builtin_expect (using_malloc_checking
== 1, 0))
4811 return malloc_check_get_size (p
);
4813 if (chunk_is_mmapped (p
))
4815 if (DUMPED_MAIN_ARENA_CHUNK (p
))
4816 return chunksize (p
) - SIZE_SZ
;
4818 return chunksize (p
) - 2 * SIZE_SZ
;
4821 return chunksize (p
) - SIZE_SZ
;
4828 __malloc_usable_size (void *m
)
4832 result
= musable (m
);
4837 ------------------------------ mallinfo ------------------------------
4838 Accumulate malloc statistics for arena AV into M.
4842 int_mallinfo (mstate av
, struct mallinfo
*m
)
4847 INTERNAL_SIZE_T avail
;
4848 INTERNAL_SIZE_T fastavail
;
4852 /* Ensure initialization */
4854 malloc_consolidate (av
);
4856 check_malloc_state (av
);
4858 /* Account for top */
4859 avail
= chunksize (av
->top
);
4860 nblocks
= 1; /* top always exists */
4862 /* traverse fastbins */
4866 for (i
= 0; i
< NFASTBINS
; ++i
)
4868 for (p
= fastbin (av
, i
); p
!= 0; p
= p
->fd
)
4871 fastavail
+= chunksize (p
);
4877 /* traverse regular bins */
4878 for (i
= 1; i
< NBINS
; ++i
)
4881 for (p
= last (b
); p
!= b
; p
= p
->bk
)
4884 avail
+= chunksize (p
);
4888 m
->smblks
+= nfastblocks
;
4889 m
->ordblks
+= nblocks
;
4890 m
->fordblks
+= avail
;
4891 m
->uordblks
+= av
->system_mem
- avail
;
4892 m
->arena
+= av
->system_mem
;
4893 m
->fsmblks
+= fastavail
;
4894 if (av
== &main_arena
)
4896 m
->hblks
= mp_
.n_mmaps
;
4897 m
->hblkhd
= mp_
.mmapped_mem
;
4899 m
->keepcost
= chunksize (av
->top
);
4905 __libc_mallinfo (void)
4910 if (__malloc_initialized
< 0)
4913 memset (&m
, 0, sizeof (m
));
4914 ar_ptr
= &main_arena
;
4917 __libc_lock_lock (ar_ptr
->mutex
);
4918 int_mallinfo (ar_ptr
, &m
);
4919 __libc_lock_unlock (ar_ptr
->mutex
);
4921 ar_ptr
= ar_ptr
->next
;
4923 while (ar_ptr
!= &main_arena
);
4929 ------------------------------ malloc_stats ------------------------------
4933 __malloc_stats (void)
4937 unsigned int in_use_b
= mp_
.mmapped_mem
, system_b
= in_use_b
;
4939 if (__malloc_initialized
< 0)
4941 _IO_flockfile (stderr
);
4942 int old_flags2
= ((_IO_FILE
*) stderr
)->_flags2
;
4943 ((_IO_FILE
*) stderr
)->_flags2
|= _IO_FLAGS2_NOTCANCEL
;
4944 for (i
= 0, ar_ptr
= &main_arena
;; i
++)
4948 memset (&mi
, 0, sizeof (mi
));
4949 __libc_lock_lock (ar_ptr
->mutex
);
4950 int_mallinfo (ar_ptr
, &mi
);
4951 fprintf (stderr
, "Arena %d:\n", i
);
4952 fprintf (stderr
, "system bytes = %10u\n", (unsigned int) mi
.arena
);
4953 fprintf (stderr
, "in use bytes = %10u\n", (unsigned int) mi
.uordblks
);
4954 #if MALLOC_DEBUG > 1
4956 dump_heap (heap_for_ptr (top (ar_ptr
)));
4958 system_b
+= mi
.arena
;
4959 in_use_b
+= mi
.uordblks
;
4960 __libc_lock_unlock (ar_ptr
->mutex
);
4961 ar_ptr
= ar_ptr
->next
;
4962 if (ar_ptr
== &main_arena
)
4965 fprintf (stderr
, "Total (incl. mmap):\n");
4966 fprintf (stderr
, "system bytes = %10u\n", system_b
);
4967 fprintf (stderr
, "in use bytes = %10u\n", in_use_b
);
4968 fprintf (stderr
, "max mmap regions = %10u\n", (unsigned int) mp_
.max_n_mmaps
);
4969 fprintf (stderr
, "max mmap bytes = %10lu\n",
4970 (unsigned long) mp_
.max_mmapped_mem
);
4971 ((_IO_FILE
*) stderr
)->_flags2
|= old_flags2
;
4972 _IO_funlockfile (stderr
);
4977 ------------------------------ mallopt ------------------------------
4981 do_set_trim_threshold (size_t value
)
4983 LIBC_PROBE (memory_mallopt_trim_threshold
, 3, value
, mp_
.trim_threshold
,
4984 mp_
.no_dyn_threshold
);
4985 mp_
.trim_threshold
= value
;
4986 mp_
.no_dyn_threshold
= 1;
4992 do_set_top_pad (size_t value
)
4994 LIBC_PROBE (memory_mallopt_top_pad
, 3, value
, mp_
.top_pad
,
4995 mp_
.no_dyn_threshold
);
4996 mp_
.top_pad
= value
;
4997 mp_
.no_dyn_threshold
= 1;
5003 do_set_mmap_threshold (size_t value
)
5005 /* Forbid setting the threshold too high. */
5006 if (value
<= HEAP_MAX_SIZE
/ 2)
5008 LIBC_PROBE (memory_mallopt_mmap_threshold
, 3, value
, mp_
.mmap_threshold
,
5009 mp_
.no_dyn_threshold
);
5010 mp_
.mmap_threshold
= value
;
5011 mp_
.no_dyn_threshold
= 1;
5019 do_set_mmaps_max (int32_t value
)
5021 LIBC_PROBE (memory_mallopt_mmap_max
, 3, value
, mp_
.n_mmaps_max
,
5022 mp_
.no_dyn_threshold
);
5023 mp_
.n_mmaps_max
= value
;
5024 mp_
.no_dyn_threshold
= 1;
5030 do_set_mallopt_check (int32_t value
)
5037 do_set_perturb_byte (int32_t value
)
5039 LIBC_PROBE (memory_mallopt_perturb
, 2, value
, perturb_byte
);
5040 perturb_byte
= value
;
5046 do_set_arena_test (size_t value
)
5048 LIBC_PROBE (memory_mallopt_arena_test
, 2, value
, mp_
.arena_test
);
5049 mp_
.arena_test
= value
;
5055 do_set_arena_max (size_t value
)
5057 LIBC_PROBE (memory_mallopt_arena_max
, 2, value
, mp_
.arena_max
);
5058 mp_
.arena_max
= value
;
5065 do_set_tcache_max (size_t value
)
5067 if (value
>= 0 && value
<= MAX_TCACHE_SIZE
)
5069 LIBC_PROBE (memory_tunable_tcache_max_bytes
, 2, value
, mp_
.tcache_max_bytes
);
5070 mp_
.tcache_max_bytes
= value
;
5071 mp_
.tcache_bins
= csize2tidx (request2size(value
)) + 1;
5078 do_set_tcache_count (size_t value
)
5080 LIBC_PROBE (memory_tunable_tcache_count
, 2, value
, mp_
.tcache_count
);
5081 mp_
.tcache_count
= value
;
5087 do_set_tcache_unsorted_limit (size_t value
)
5089 LIBC_PROBE (memory_tunable_tcache_unsorted_limit
, 2, value
, mp_
.tcache_unsorted_limit
);
5090 mp_
.tcache_unsorted_limit
= value
;
5096 __libc_mallopt (int param_number
, int value
)
5098 mstate av
= &main_arena
;
5101 if (__malloc_initialized
< 0)
5103 __libc_lock_lock (av
->mutex
);
5104 /* Ensure initialization/consolidation */
5105 malloc_consolidate (av
);
5107 LIBC_PROBE (memory_mallopt
, 2, param_number
, value
);
5109 switch (param_number
)
5112 if (value
>= 0 && value
<= MAX_FAST_SIZE
)
5114 LIBC_PROBE (memory_mallopt_mxfast
, 2, value
, get_max_fast ());
5115 set_max_fast (value
);
5121 case M_TRIM_THRESHOLD
:
5122 do_set_trim_threshold (value
);
5126 do_set_top_pad (value
);
5129 case M_MMAP_THRESHOLD
:
5130 res
= do_set_mmap_threshold (value
);
5134 do_set_mmaps_max (value
);
5137 case M_CHECK_ACTION
:
5138 do_set_mallopt_check (value
);
5142 do_set_perturb_byte (value
);
5147 do_set_arena_test (value
);
5152 do_set_arena_max (value
);
5155 __libc_lock_unlock (av
->mutex
);
5158 libc_hidden_def (__libc_mallopt
)
5162 -------------------- Alternative MORECORE functions --------------------
5167 General Requirements for MORECORE.
5169 The MORECORE function must have the following properties:
5171 If MORECORE_CONTIGUOUS is false:
5173 * MORECORE must allocate in multiples of pagesize. It will
5174 only be called with arguments that are multiples of pagesize.
5176 * MORECORE(0) must return an address that is at least
5177 MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
5179 else (i.e. If MORECORE_CONTIGUOUS is true):
5181 * Consecutive calls to MORECORE with positive arguments
5182 return increasing addresses, indicating that space has been
5183 contiguously extended.
5185 * MORECORE need not allocate in multiples of pagesize.
5186 Calls to MORECORE need not have args of multiples of pagesize.
5188 * MORECORE need not page-align.
5192 * MORECORE may allocate more memory than requested. (Or even less,
5193 but this will generally result in a malloc failure.)
5195 * MORECORE must not allocate memory when given argument zero, but
5196 instead return one past the end address of memory from previous
5197 nonzero call. This malloc does NOT call MORECORE(0)
5198 until at least one call with positive arguments is made, so
5199 the initial value returned is not important.
5201 * Even though consecutive calls to MORECORE need not return contiguous
5202 addresses, it must be OK for malloc'ed chunks to span multiple
5203 regions in those cases where they do happen to be contiguous.
5205 * MORECORE need not handle negative arguments -- it may instead
5206 just return MORECORE_FAILURE when given negative arguments.
5207 Negative arguments are always multiples of pagesize. MORECORE
5208 must not misinterpret negative args as large positive unsigned
5209 args. You can suppress all such calls from even occurring by defining
5210 MORECORE_CANNOT_TRIM,
5212 There is some variation across systems about the type of the
5213 argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
5214 actually be size_t, because sbrk supports negative args, so it is
5215 normally the signed type of the same width as size_t (sometimes
5216 declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
5217 matter though. Internally, we use "long" as arguments, which should
5218 work across all reasonable possibilities.
5220 Additionally, if MORECORE ever returns failure for a positive
5221 request, then mmap is used as a noncontiguous system allocator. This
5222 is a useful backup strategy for systems with holes in address spaces
5223 -- in this case sbrk cannot contiguously expand the heap, but mmap
5224 may be able to map noncontiguous space.
5226 If you'd like mmap to ALWAYS be used, you can define MORECORE to be
5227 a function that always returns MORECORE_FAILURE.
5229 If you are using this malloc with something other than sbrk (or its
5230 emulation) to supply memory regions, you probably want to set
5231 MORECORE_CONTIGUOUS as false. As an example, here is a custom
5232 allocator kindly contributed for pre-OSX macOS. It uses virtually
5233 but not necessarily physically contiguous non-paged memory (locked
5234 in, present and won't get swapped out). You can use it by
5235 uncommenting this section, adding some #includes, and setting up the
5236 appropriate defines above:
5238 *#define MORECORE osMoreCore
5239 *#define MORECORE_CONTIGUOUS 0
5241 There is also a shutdown routine that should somehow be called for
5242 cleanup upon program exit.
5244 *#define MAX_POOL_ENTRIES 100
5245 *#define MINIMUM_MORECORE_SIZE (64 * 1024)
5246 static int next_os_pool;
5247 void *our_os_pools[MAX_POOL_ENTRIES];
5249 void *osMoreCore(int size)
5252 static void *sbrk_top = 0;
5256 if (size < MINIMUM_MORECORE_SIZE)
5257 size = MINIMUM_MORECORE_SIZE;
5258 if (CurrentExecutionLevel() == kTaskLevel)
5259 ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
5262 return (void *) MORECORE_FAILURE;
5264 // save ptrs so they can be freed during cleanup
5265 our_os_pools[next_os_pool] = ptr;
5267 ptr = (void *) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
5268 sbrk_top = (char *) ptr + size;
5273 // we don't currently support shrink behavior
5274 return (void *) MORECORE_FAILURE;
5282 // cleanup any allocated memory pools
5283 // called as last thing before shutting down driver
5285 void osCleanupMem(void)
5289 for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
5292 PoolDeallocate(*ptr);
5302 extern char **__libc_argv attribute_hidden
;
5305 malloc_printerr (const char *str
)
5307 __libc_message (do_abort
, "%s\n", str
);
5308 __builtin_unreachable ();
5311 /* We need a wrapper function for one of the additions of POSIX. */
5313 __posix_memalign (void **memptr
, size_t alignment
, size_t size
)
5317 /* Test whether the SIZE argument is valid. It must be a power of
5318 two multiple of sizeof (void *). */
5319 if (alignment
% sizeof (void *) != 0
5320 || !powerof2 (alignment
/ sizeof (void *))
5325 void *address
= RETURN_ADDRESS (0);
5326 mem
= _mid_memalign (alignment
, size
, address
);
5336 weak_alias (__posix_memalign
, posix_memalign
)
5340 __malloc_info (int options
, FILE *fp
)
5342 /* For now, at least. */
5347 size_t total_nblocks
= 0;
5348 size_t total_nfastblocks
= 0;
5349 size_t total_avail
= 0;
5350 size_t total_fastavail
= 0;
5351 size_t total_system
= 0;
5352 size_t total_max_system
= 0;
5353 size_t total_aspace
= 0;
5354 size_t total_aspace_mprotect
= 0;
5358 if (__malloc_initialized
< 0)
5361 fputs ("<malloc version=\"1\">\n", fp
);
5363 /* Iterate over all arenas currently in use. */
5364 mstate ar_ptr
= &main_arena
;
5367 fprintf (fp
, "<heap nr=\"%d\">\n<sizes>\n", n
++);
5370 size_t nfastblocks
= 0;
5372 size_t fastavail
= 0;
5379 } sizes
[NFASTBINS
+ NBINS
- 1];
5380 #define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5382 __libc_lock_lock (ar_ptr
->mutex
);
5384 for (size_t i
= 0; i
< NFASTBINS
; ++i
)
5386 mchunkptr p
= fastbin (ar_ptr
, i
);
5389 size_t nthissize
= 0;
5390 size_t thissize
= chunksize (p
);
5398 fastavail
+= nthissize
* thissize
;
5399 nfastblocks
+= nthissize
;
5400 sizes
[i
].from
= thissize
- (MALLOC_ALIGNMENT
- 1);
5401 sizes
[i
].to
= thissize
;
5402 sizes
[i
].count
= nthissize
;
5405 sizes
[i
].from
= sizes
[i
].to
= sizes
[i
].count
= 0;
5407 sizes
[i
].total
= sizes
[i
].count
* sizes
[i
].to
;
5412 struct malloc_chunk
*r
;
5414 for (size_t i
= 1; i
< NBINS
; ++i
)
5416 bin
= bin_at (ar_ptr
, i
);
5418 sizes
[NFASTBINS
- 1 + i
].from
= ~((size_t) 0);
5419 sizes
[NFASTBINS
- 1 + i
].to
= sizes
[NFASTBINS
- 1 + i
].total
5420 = sizes
[NFASTBINS
- 1 + i
].count
= 0;
5425 size_t r_size
= chunksize_nomask (r
);
5426 ++sizes
[NFASTBINS
- 1 + i
].count
;
5427 sizes
[NFASTBINS
- 1 + i
].total
+= r_size
;
5428 sizes
[NFASTBINS
- 1 + i
].from
5429 = MIN (sizes
[NFASTBINS
- 1 + i
].from
, r_size
);
5430 sizes
[NFASTBINS
- 1 + i
].to
= MAX (sizes
[NFASTBINS
- 1 + i
].to
,
5436 if (sizes
[NFASTBINS
- 1 + i
].count
== 0)
5437 sizes
[NFASTBINS
- 1 + i
].from
= 0;
5438 nblocks
+= sizes
[NFASTBINS
- 1 + i
].count
;
5439 avail
+= sizes
[NFASTBINS
- 1 + i
].total
;
5442 __libc_lock_unlock (ar_ptr
->mutex
);
5444 total_nfastblocks
+= nfastblocks
;
5445 total_fastavail
+= fastavail
;
5447 total_nblocks
+= nblocks
;
5448 total_avail
+= avail
;
5450 for (size_t i
= 0; i
< nsizes
; ++i
)
5451 if (sizes
[i
].count
!= 0 && i
!= NFASTBINS
)
5453 <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5454 sizes
[i
].from
, sizes
[i
].to
, sizes
[i
].total
, sizes
[i
].count
);
5456 if (sizes
[NFASTBINS
].count
!= 0)
5458 <unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5459 sizes
[NFASTBINS
].from
, sizes
[NFASTBINS
].to
,
5460 sizes
[NFASTBINS
].total
, sizes
[NFASTBINS
].count
);
5462 total_system
+= ar_ptr
->system_mem
;
5463 total_max_system
+= ar_ptr
->max_system_mem
;
5466 "</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5467 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5468 "<system type=\"current\" size=\"%zu\"/>\n"
5469 "<system type=\"max\" size=\"%zu\"/>\n",
5470 nfastblocks
, fastavail
, nblocks
, avail
,
5471 ar_ptr
->system_mem
, ar_ptr
->max_system_mem
);
5473 if (ar_ptr
!= &main_arena
)
5475 heap_info
*heap
= heap_for_ptr (top (ar_ptr
));
5477 "<aspace type=\"total\" size=\"%zu\"/>\n"
5478 "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5479 heap
->size
, heap
->mprotect_size
);
5480 total_aspace
+= heap
->size
;
5481 total_aspace_mprotect
+= heap
->mprotect_size
;
5486 "<aspace type=\"total\" size=\"%zu\"/>\n"
5487 "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5488 ar_ptr
->system_mem
, ar_ptr
->system_mem
);
5489 total_aspace
+= ar_ptr
->system_mem
;
5490 total_aspace_mprotect
+= ar_ptr
->system_mem
;
5493 fputs ("</heap>\n", fp
);
5494 ar_ptr
= ar_ptr
->next
;
5496 while (ar_ptr
!= &main_arena
);
5499 "<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5500 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5501 "<total type=\"mmap\" count=\"%d\" size=\"%zu\"/>\n"
5502 "<system type=\"current\" size=\"%zu\"/>\n"
5503 "<system type=\"max\" size=\"%zu\"/>\n"
5504 "<aspace type=\"total\" size=\"%zu\"/>\n"
5505 "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5507 total_nfastblocks
, total_fastavail
, total_nblocks
, total_avail
,
5508 mp_
.n_mmaps
, mp_
.mmapped_mem
,
5509 total_system
, total_max_system
,
5510 total_aspace
, total_aspace_mprotect
);
5514 weak_alias (__malloc_info
, malloc_info
)
5517 strong_alias (__libc_calloc
, __calloc
) weak_alias (__libc_calloc
, calloc
)
5518 strong_alias (__libc_free
, __free
) strong_alias (__libc_free
, free
)
5519 strong_alias (__libc_malloc
, __malloc
) strong_alias (__libc_malloc
, malloc
)
5520 strong_alias (__libc_memalign
, __memalign
)
5521 weak_alias (__libc_memalign
, memalign
)
5522 strong_alias (__libc_realloc
, __realloc
) strong_alias (__libc_realloc
, realloc
)
5523 strong_alias (__libc_valloc
, __valloc
) weak_alias (__libc_valloc
, valloc
)
5524 strong_alias (__libc_pvalloc
, __pvalloc
) weak_alias (__libc_pvalloc
, pvalloc
)
5525 strong_alias (__libc_mallinfo
, __mallinfo
)
5526 weak_alias (__libc_mallinfo
, mallinfo
)
5527 strong_alias (__libc_mallopt
, __mallopt
) weak_alias (__libc_mallopt
, mallopt
)
5529 weak_alias (__malloc_stats
, malloc_stats
)
5530 weak_alias (__malloc_usable_size
, malloc_usable_size
)
5531 weak_alias (__malloc_trim
, malloc_trim
)
5533 #if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_26)
5534 compat_symbol (libc
, __libc_free
, cfree
, GLIBC_2_0
);
5537 /* ------------------------------------------------------------
5540 [see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]