]>
Commit | Line | Data |
---|---|---|
b9215da1 MT |
1 | From 98c293c61f770b6b7a22f89a6ea81b711ecb1952 Mon Sep 17 00:00:00 2001 |
2 | From: Florian Weimer <fweimer@redhat.com> | |
3 | Date: Fri, 11 Aug 2023 11:18:17 +0200 | |
4 | Subject: [PATCH 07/27] malloc: Enable merging of remainders in memalign (bug | |
5 | 30723) | |
6 | ||
7 | Previously, calling _int_free from _int_memalign could put remainders | |
8 | into the tcache or into fastbins, where they are invisible to the | |
9 | low-level allocator. This results in missed merge opportunities | |
10 | because once these freed chunks become available to the low-level | |
11 | allocator, further memalign allocations (even of the same size are) | |
12 | likely obstructing merges. | |
13 | ||
14 | Furthermore, during forwards merging in _int_memalign, do not | |
15 | completely give up when the remainder is too small to serve as a | |
16 | chunk on its own. We can still give it back if it can be merged | |
17 | with the following unused chunk. This makes it more likely that | |
18 | memalign calls in a loop achieve a compact memory layout, | |
19 | independently of initial heap layout. | |
20 | ||
21 | Drop some useless (unsigned long) casts along the way, and tweak | |
22 | the style to more closely match GNU on changed lines. | |
23 | ||
24 | Reviewed-by: DJ Delorie <dj@redhat.com> | |
25 | (cherry picked from commit 542b1105852568c3ebc712225ae78b8c8ba31a78) | |
26 | --- | |
27 | malloc/malloc.c | 197 +++++++++++++++++++++++++++++------------------- | |
28 | 1 file changed, 121 insertions(+), 76 deletions(-) | |
29 | ||
30 | diff --git a/malloc/malloc.c b/malloc/malloc.c | |
31 | index e2f1a615a4..948f9759af 100644 | |
32 | --- a/malloc/malloc.c | |
33 | +++ b/malloc/malloc.c | |
34 | @@ -1086,6 +1086,11 @@ typedef struct malloc_chunk* mchunkptr; | |
35 | ||
36 | static void* _int_malloc(mstate, size_t); | |
37 | static void _int_free(mstate, mchunkptr, int); | |
38 | +static void _int_free_merge_chunk (mstate, mchunkptr, INTERNAL_SIZE_T); | |
39 | +static INTERNAL_SIZE_T _int_free_create_chunk (mstate, | |
40 | + mchunkptr, INTERNAL_SIZE_T, | |
41 | + mchunkptr, INTERNAL_SIZE_T); | |
42 | +static void _int_free_maybe_consolidate (mstate, INTERNAL_SIZE_T); | |
43 | static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T, | |
44 | INTERNAL_SIZE_T); | |
45 | static void* _int_memalign(mstate, size_t, size_t); | |
46 | @@ -4637,31 +4642,52 @@ _int_free (mstate av, mchunkptr p, int have_lock) | |
47 | if (!have_lock) | |
48 | __libc_lock_lock (av->mutex); | |
49 | ||
50 | - nextchunk = chunk_at_offset(p, size); | |
51 | - | |
52 | - /* Lightweight tests: check whether the block is already the | |
53 | - top block. */ | |
54 | - if (__glibc_unlikely (p == av->top)) | |
55 | - malloc_printerr ("double free or corruption (top)"); | |
56 | - /* Or whether the next chunk is beyond the boundaries of the arena. */ | |
57 | - if (__builtin_expect (contiguous (av) | |
58 | - && (char *) nextchunk | |
59 | - >= ((char *) av->top + chunksize(av->top)), 0)) | |
60 | - malloc_printerr ("double free or corruption (out)"); | |
61 | - /* Or whether the block is actually not marked used. */ | |
62 | - if (__glibc_unlikely (!prev_inuse(nextchunk))) | |
63 | - malloc_printerr ("double free or corruption (!prev)"); | |
64 | - | |
65 | - nextsize = chunksize(nextchunk); | |
66 | - if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0) | |
67 | - || __builtin_expect (nextsize >= av->system_mem, 0)) | |
68 | - malloc_printerr ("free(): invalid next size (normal)"); | |
69 | + _int_free_merge_chunk (av, p, size); | |
70 | ||
71 | - free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); | |
72 | + if (!have_lock) | |
73 | + __libc_lock_unlock (av->mutex); | |
74 | + } | |
75 | + /* | |
76 | + If the chunk was allocated via mmap, release via munmap(). | |
77 | + */ | |
78 | + | |
79 | + else { | |
80 | + munmap_chunk (p); | |
81 | + } | |
82 | +} | |
83 | + | |
84 | +/* Try to merge chunk P of SIZE bytes with its neighbors. Put the | |
85 | + resulting chunk on the appropriate bin list. P must not be on a | |
86 | + bin list yet, and it can be in use. */ | |
87 | +static void | |
88 | +_int_free_merge_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size) | |
89 | +{ | |
90 | + mchunkptr nextchunk = chunk_at_offset(p, size); | |
91 | + | |
92 | + /* Lightweight tests: check whether the block is already the | |
93 | + top block. */ | |
94 | + if (__glibc_unlikely (p == av->top)) | |
95 | + malloc_printerr ("double free or corruption (top)"); | |
96 | + /* Or whether the next chunk is beyond the boundaries of the arena. */ | |
97 | + if (__builtin_expect (contiguous (av) | |
98 | + && (char *) nextchunk | |
99 | + >= ((char *) av->top + chunksize(av->top)), 0)) | |
100 | + malloc_printerr ("double free or corruption (out)"); | |
101 | + /* Or whether the block is actually not marked used. */ | |
102 | + if (__glibc_unlikely (!prev_inuse(nextchunk))) | |
103 | + malloc_printerr ("double free or corruption (!prev)"); | |
104 | + | |
105 | + INTERNAL_SIZE_T nextsize = chunksize(nextchunk); | |
106 | + if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0) | |
107 | + || __builtin_expect (nextsize >= av->system_mem, 0)) | |
108 | + malloc_printerr ("free(): invalid next size (normal)"); | |
109 | + | |
110 | + free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); | |
111 | ||
112 | - /* consolidate backward */ | |
113 | - if (!prev_inuse(p)) { | |
114 | - prevsize = prev_size (p); | |
115 | + /* Consolidate backward. */ | |
116 | + if (!prev_inuse(p)) | |
117 | + { | |
118 | + INTERNAL_SIZE_T prevsize = prev_size (p); | |
119 | size += prevsize; | |
120 | p = chunk_at_offset(p, -((long) prevsize)); | |
121 | if (__glibc_unlikely (chunksize(p) != prevsize)) | |
122 | @@ -4669,9 +4695,25 @@ _int_free (mstate av, mchunkptr p, int have_lock) | |
123 | unlink_chunk (av, p); | |
124 | } | |
125 | ||
126 | - if (nextchunk != av->top) { | |
127 | + /* Write the chunk header, maybe after merging with the following chunk. */ | |
128 | + size = _int_free_create_chunk (av, p, size, nextchunk, nextsize); | |
129 | + _int_free_maybe_consolidate (av, size); | |
130 | +} | |
131 | + | |
132 | +/* Create a chunk at P of SIZE bytes, with SIZE potentially increased | |
133 | + to cover the immediately following chunk NEXTCHUNK of NEXTSIZE | |
134 | + bytes (if NEXTCHUNK is unused). The chunk at P is not actually | |
135 | + read and does not have to be initialized. After creation, it is | |
136 | + placed on the appropriate bin list. The function returns the size | |
137 | + of the new chunk. */ | |
138 | +static INTERNAL_SIZE_T | |
139 | +_int_free_create_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, | |
140 | + mchunkptr nextchunk, INTERNAL_SIZE_T nextsize) | |
141 | +{ | |
142 | + if (nextchunk != av->top) | |
143 | + { | |
144 | /* get and clear inuse bit */ | |
145 | - nextinuse = inuse_bit_at_offset(nextchunk, nextsize); | |
146 | + bool nextinuse = inuse_bit_at_offset (nextchunk, nextsize); | |
147 | ||
148 | /* consolidate forward */ | |
149 | if (!nextinuse) { | |
150 | @@ -4686,8 +4728,8 @@ _int_free (mstate av, mchunkptr p, int have_lock) | |
151 | been given one chance to be used in malloc. | |
152 | */ | |
153 | ||
154 | - bck = unsorted_chunks(av); | |
155 | - fwd = bck->fd; | |
156 | + mchunkptr bck = unsorted_chunks (av); | |
157 | + mchunkptr fwd = bck->fd; | |
158 | if (__glibc_unlikely (fwd->bk != bck)) | |
159 | malloc_printerr ("free(): corrupted unsorted chunks"); | |
160 | p->fd = fwd; | |
161 | @@ -4706,61 +4748,52 @@ _int_free (mstate av, mchunkptr p, int have_lock) | |
162 | check_free_chunk(av, p); | |
163 | } | |
164 | ||
165 | - /* | |
166 | - If the chunk borders the current high end of memory, | |
167 | - consolidate into top | |
168 | - */ | |
169 | - | |
170 | - else { | |
171 | + else | |
172 | + { | |
173 | + /* If the chunk borders the current high end of memory, | |
174 | + consolidate into top. */ | |
175 | size += nextsize; | |
176 | set_head(p, size | PREV_INUSE); | |
177 | av->top = p; | |
178 | check_chunk(av, p); | |
179 | } | |
180 | ||
181 | - /* | |
182 | - If freeing a large space, consolidate possibly-surrounding | |
183 | - chunks. Then, if the total unused topmost memory exceeds trim | |
184 | - threshold, ask malloc_trim to reduce top. | |
185 | - | |
186 | - Unless max_fast is 0, we don't know if there are fastbins | |
187 | - bordering top, so we cannot tell for sure whether threshold | |
188 | - has been reached unless fastbins are consolidated. But we | |
189 | - don't want to consolidate on each free. As a compromise, | |
190 | - consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD | |
191 | - is reached. | |
192 | - */ | |
193 | + return size; | |
194 | +} | |
195 | ||
196 | - if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) { | |
197 | +/* If freeing a large space, consolidate possibly-surrounding | |
198 | + chunks. Then, if the total unused topmost memory exceeds trim | |
199 | + threshold, ask malloc_trim to reduce top. */ | |
200 | +static void | |
201 | +_int_free_maybe_consolidate (mstate av, INTERNAL_SIZE_T size) | |
202 | +{ | |
203 | + /* Unless max_fast is 0, we don't know if there are fastbins | |
204 | + bordering top, so we cannot tell for sure whether threshold has | |
205 | + been reached unless fastbins are consolidated. But we don't want | |
206 | + to consolidate on each free. As a compromise, consolidation is | |
207 | + performed if FASTBIN_CONSOLIDATION_THRESHOLD is reached. */ | |
208 | + if (size >= FASTBIN_CONSOLIDATION_THRESHOLD) | |
209 | + { | |
210 | if (atomic_load_relaxed (&av->have_fastchunks)) | |
211 | malloc_consolidate(av); | |
212 | ||
213 | - if (av == &main_arena) { | |
214 | + if (av == &main_arena) | |
215 | + { | |
216 | #ifndef MORECORE_CANNOT_TRIM | |
217 | - if ((unsigned long)(chunksize(av->top)) >= | |
218 | - (unsigned long)(mp_.trim_threshold)) | |
219 | - systrim(mp_.top_pad, av); | |
220 | + if (chunksize (av->top) >= mp_.trim_threshold) | |
221 | + systrim (mp_.top_pad, av); | |
222 | #endif | |
223 | - } else { | |
224 | - /* Always try heap_trim(), even if the top chunk is not | |
225 | - large, because the corresponding heap might go away. */ | |
226 | - heap_info *heap = heap_for_ptr(top(av)); | |
227 | + } | |
228 | + else | |
229 | + { | |
230 | + /* Always try heap_trim, even if the top chunk is not large, | |
231 | + because the corresponding heap might go away. */ | |
232 | + heap_info *heap = heap_for_ptr (top (av)); | |
233 | ||
234 | - assert(heap->ar_ptr == av); | |
235 | - heap_trim(heap, mp_.top_pad); | |
236 | - } | |
237 | + assert (heap->ar_ptr == av); | |
238 | + heap_trim (heap, mp_.top_pad); | |
239 | + } | |
240 | } | |
241 | - | |
242 | - if (!have_lock) | |
243 | - __libc_lock_unlock (av->mutex); | |
244 | - } | |
245 | - /* | |
246 | - If the chunk was allocated via mmap, release via munmap(). | |
247 | - */ | |
248 | - | |
249 | - else { | |
250 | - munmap_chunk (p); | |
251 | - } | |
252 | } | |
253 | ||
254 | /* | |
255 | @@ -5221,7 +5254,7 @@ _int_memalign (mstate av, size_t alignment, size_t bytes) | |
256 | (av != &main_arena ? NON_MAIN_ARENA : 0)); | |
257 | set_inuse_bit_at_offset (newp, newsize); | |
258 | set_head_size (p, leadsize | (av != &main_arena ? NON_MAIN_ARENA : 0)); | |
259 | - _int_free (av, p, 1); | |
260 | + _int_free_merge_chunk (av, p, leadsize); | |
261 | p = newp; | |
262 | ||
263 | assert (newsize >= nb && | |
264 | @@ -5232,15 +5265,27 @@ _int_memalign (mstate av, size_t alignment, size_t bytes) | |
265 | if (!chunk_is_mmapped (p)) | |
266 | { | |
267 | size = chunksize (p); | |
268 | - if ((unsigned long) (size) > (unsigned long) (nb + MINSIZE)) | |
269 | + mchunkptr nextchunk = chunk_at_offset(p, size); | |
270 | + INTERNAL_SIZE_T nextsize = chunksize(nextchunk); | |
271 | + if (size > nb) | |
272 | { | |
273 | remainder_size = size - nb; | |
274 | - remainder = chunk_at_offset (p, nb); | |
275 | - set_head (remainder, remainder_size | PREV_INUSE | | |
276 | - (av != &main_arena ? NON_MAIN_ARENA : 0)); | |
277 | - set_head_size (p, nb); | |
278 | - _int_free (av, remainder, 1); | |
279 | - } | |
280 | + if (remainder_size >= MINSIZE | |
281 | + || nextchunk == av->top | |
282 | + || !inuse_bit_at_offset (nextchunk, nextsize)) | |
283 | + { | |
284 | + /* We can only give back the tail if it is larger than | |
285 | + MINSIZE, or if the following chunk is unused (top | |
286 | + chunk or unused in-heap chunk). Otherwise we would | |
287 | + create a chunk that is smaller than MINSIZE. */ | |
288 | + remainder = chunk_at_offset (p, nb); | |
289 | + set_head_size (p, nb); | |
290 | + remainder_size = _int_free_create_chunk (av, remainder, | |
291 | + remainder_size, | |
292 | + nextchunk, nextsize); | |
293 | + _int_free_maybe_consolidate (av, remainder_size); | |
294 | + } | |
295 | + } | |
296 | } | |
297 | ||
298 | check_inuse_chunk (av, p); | |
299 | -- | |
300 | 2.39.2 | |
301 |