]>
Commit | Line | Data |
---|---|---|
83ffe9cd | 1 | /* Copyright (C) 2020-2023 Free Software Foundation, Inc. |
e1071571 JJ |
2 | Contributed by Jakub Jelinek <jakub@redhat.com>. |
3 | ||
4 | This file is part of the GNU Offloading and Multi Processing Library | |
5 | (libgomp). | |
6 | ||
7 | Libgomp is free software; you can redistribute it and/or modify it | |
8 | under the terms of the GNU General Public License as published by | |
9 | the Free Software Foundation; either version 3, or (at your option) | |
10 | any later version. | |
11 | ||
12 | Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
14 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
15 | more details. | |
16 | ||
17 | Under Section 7 of GPL version 3, you are granted additional | |
18 | permissions described in the GCC Runtime Library Exception, version | |
19 | 3.1, as published by the Free Software Foundation. | |
20 | ||
21 | You should have received a copy of the GNU General Public License and | |
22 | a copy of the GCC Runtime Library Exception along with this program; | |
23 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
24 | <http://www.gnu.org/licenses/>. */ | |
25 | ||
26 | /* This file contains wrappers for the system allocation routines. Most | |
27 | places in the OpenMP API do not make any provision for failure, so in | |
28 | general we cannot allow memory allocation to fail. */ | |
29 | ||
30 | #define _GNU_SOURCE | |
31 | #include "libgomp.h" | |
32 | #include <stdlib.h> | |
b38a4bd1 | 33 | #include <string.h> |
17f52a1c JJ |
34 | #ifdef LIBGOMP_USE_MEMKIND |
35 | #include <dlfcn.h> | |
36 | #endif | |
e1071571 JJ |
37 | |
38 | #define omp_max_predefined_alloc omp_thread_mem_alloc | |
39 | ||
17f52a1c JJ |
40 | enum gomp_memkind_kind |
41 | { | |
42 | GOMP_MEMKIND_NONE = 0, | |
43 | #define GOMP_MEMKIND_KINDS \ | |
44 | GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \ | |
45 | GOMP_MEMKIND_KIND (HBW_PREFERRED), \ | |
46 | GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \ | |
47 | GOMP_MEMKIND_KIND (DAX_KMEM), \ | |
48 | GOMP_MEMKIND_KIND (INTERLEAVE), \ | |
49 | GOMP_MEMKIND_KIND (DEFAULT) | |
50 | #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind | |
51 | GOMP_MEMKIND_KINDS, | |
52 | #undef GOMP_MEMKIND_KIND | |
53 | GOMP_MEMKIND_COUNT | |
54 | }; | |
55 | ||
e1071571 JJ |
56 | struct omp_allocator_data |
57 | { | |
58 | omp_memspace_handle_t memspace; | |
59 | omp_uintptr_t alignment; | |
60 | omp_uintptr_t pool_size; | |
61 | omp_uintptr_t used_pool_size; | |
62 | omp_allocator_handle_t fb_data; | |
63 | unsigned int sync_hint : 8; | |
64 | unsigned int access : 8; | |
65 | unsigned int fallback : 8; | |
66 | unsigned int pinned : 1; | |
67 | unsigned int partition : 7; | |
17f52a1c JJ |
68 | #ifdef LIBGOMP_USE_MEMKIND |
69 | unsigned int memkind : 8; | |
70 | #endif | |
e1071571 JJ |
71 | #ifndef HAVE_SYNC_BUILTINS |
72 | gomp_mutex_t lock; | |
73 | #endif | |
74 | }; | |
75 | ||
76 | struct omp_mem_header | |
77 | { | |
78 | void *ptr; | |
79 | size_t size; | |
80 | omp_allocator_handle_t allocator; | |
81 | void *pad; | |
82 | }; | |
83 | ||
17f52a1c JJ |
84 | struct gomp_memkind_data |
85 | { | |
86 | void *memkind_handle; | |
87 | void *(*memkind_malloc) (void *, size_t); | |
88 | void *(*memkind_calloc) (void *, size_t, size_t); | |
89 | void *(*memkind_realloc) (void *, void *, size_t); | |
90 | void (*memkind_free) (void *, void *); | |
91 | int (*memkind_check_available) (void *); | |
92 | void **kinds[GOMP_MEMKIND_COUNT]; | |
93 | }; | |
94 | ||
95 | #ifdef LIBGOMP_USE_MEMKIND | |
96 | static struct gomp_memkind_data *memkind_data; | |
97 | static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT; | |
98 | ||
99 | static void | |
100 | gomp_init_memkind (void) | |
101 | { | |
1eff4872 | 102 | void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY); |
17f52a1c JJ |
103 | struct gomp_memkind_data *data; |
104 | int i; | |
105 | static const char *kinds[] = { | |
106 | NULL, | |
107 | #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind | |
108 | GOMP_MEMKIND_KINDS | |
109 | #undef GOMP_MEMKIND_KIND | |
110 | }; | |
111 | ||
112 | data = calloc (1, sizeof (struct gomp_memkind_data)); | |
113 | if (data == NULL) | |
114 | { | |
115 | if (handle) | |
116 | dlclose (handle); | |
117 | return; | |
118 | } | |
119 | if (!handle) | |
120 | { | |
121 | __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE); | |
122 | return; | |
123 | } | |
124 | data->memkind_handle = handle; | |
125 | data->memkind_malloc | |
126 | = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc"); | |
127 | data->memkind_calloc | |
128 | = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc"); | |
129 | data->memkind_realloc | |
130 | = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc"); | |
131 | data->memkind_free | |
132 | = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free"); | |
133 | data->memkind_check_available | |
134 | = (__typeof (data->memkind_check_available)) | |
135 | dlsym (handle, "memkind_check_available"); | |
136 | if (data->memkind_malloc | |
137 | && data->memkind_calloc | |
138 | && data->memkind_realloc | |
139 | && data->memkind_free | |
140 | && data->memkind_check_available) | |
141 | for (i = 1; i < GOMP_MEMKIND_COUNT; ++i) | |
142 | { | |
143 | data->kinds[i] = (void **) dlsym (handle, kinds[i]); | |
144 | if (data->kinds[i] && data->memkind_check_available (*data->kinds[i])) | |
145 | data->kinds[i] = NULL; | |
146 | } | |
147 | __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE); | |
148 | } | |
149 | ||
150 | static struct gomp_memkind_data * | |
151 | gomp_get_memkind (void) | |
152 | { | |
153 | struct gomp_memkind_data *data | |
154 | = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE); | |
155 | if (data) | |
156 | return data; | |
157 | pthread_once (&memkind_data_once, gomp_init_memkind); | |
158 | return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE); | |
159 | } | |
160 | #endif | |
161 | ||
e1071571 JJ |
162 | omp_allocator_handle_t |
163 | omp_init_allocator (omp_memspace_handle_t memspace, int ntraits, | |
164 | const omp_alloctrait_t traits[]) | |
165 | { | |
166 | struct omp_allocator_data data | |
167 | = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all, | |
17f52a1c JJ |
168 | omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment, |
169 | #ifdef LIBGOMP_USE_MEMKIND | |
170 | GOMP_MEMKIND_NONE | |
171 | #endif | |
172 | }; | |
e1071571 JJ |
173 | struct omp_allocator_data *ret; |
174 | int i; | |
175 | ||
176 | if (memspace > omp_low_lat_mem_space) | |
177 | return omp_null_allocator; | |
178 | for (i = 0; i < ntraits; i++) | |
179 | switch (traits[i].key) | |
180 | { | |
181 | case omp_atk_sync_hint: | |
182 | switch (traits[i].value) | |
183 | { | |
184 | case omp_atv_default: | |
185 | data.sync_hint = omp_atv_contended; | |
186 | break; | |
187 | case omp_atv_contended: | |
188 | case omp_atv_uncontended: | |
f7097793 | 189 | case omp_atv_serialized: |
e1071571 JJ |
190 | case omp_atv_private: |
191 | data.sync_hint = traits[i].value; | |
192 | break; | |
193 | default: | |
194 | return omp_null_allocator; | |
195 | } | |
196 | break; | |
197 | case omp_atk_alignment: | |
ea82325a JJ |
198 | if (traits[i].value == omp_atv_default) |
199 | { | |
200 | data.alignment = 1; | |
201 | break; | |
202 | } | |
e1071571 JJ |
203 | if ((traits[i].value & (traits[i].value - 1)) != 0 |
204 | || !traits[i].value) | |
205 | return omp_null_allocator; | |
206 | data.alignment = traits[i].value; | |
207 | break; | |
208 | case omp_atk_access: | |
209 | switch (traits[i].value) | |
210 | { | |
211 | case omp_atv_default: | |
212 | data.access = omp_atv_all; | |
213 | break; | |
214 | case omp_atv_all: | |
215 | case omp_atv_cgroup: | |
216 | case omp_atv_pteam: | |
217 | case omp_atv_thread: | |
218 | data.access = traits[i].value; | |
219 | break; | |
220 | default: | |
221 | return omp_null_allocator; | |
222 | } | |
223 | break; | |
224 | case omp_atk_pool_size: | |
ea82325a JJ |
225 | if (traits[i].value == omp_atv_default) |
226 | data.pool_size = ~(uintptr_t) 0; | |
227 | else | |
228 | data.pool_size = traits[i].value; | |
e1071571 JJ |
229 | break; |
230 | case omp_atk_fallback: | |
231 | switch (traits[i].value) | |
232 | { | |
233 | case omp_atv_default: | |
234 | data.fallback = omp_atv_default_mem_fb; | |
235 | break; | |
236 | case omp_atv_default_mem_fb: | |
237 | case omp_atv_null_fb: | |
238 | case omp_atv_abort_fb: | |
239 | case omp_atv_allocator_fb: | |
240 | data.fallback = traits[i].value; | |
241 | break; | |
242 | default: | |
243 | return omp_null_allocator; | |
244 | } | |
245 | break; | |
246 | case omp_atk_fb_data: | |
247 | data.fb_data = traits[i].value; | |
248 | break; | |
249 | case omp_atk_pinned: | |
250 | switch (traits[i].value) | |
251 | { | |
252 | case omp_atv_default: | |
253 | case omp_atv_false: | |
254 | data.pinned = omp_atv_false; | |
255 | break; | |
256 | case omp_atv_true: | |
257 | data.pinned = omp_atv_true; | |
258 | break; | |
259 | default: | |
260 | return omp_null_allocator; | |
261 | } | |
262 | break; | |
263 | case omp_atk_partition: | |
264 | switch (traits[i].value) | |
265 | { | |
266 | case omp_atv_default: | |
267 | data.partition = omp_atv_environment; | |
268 | break; | |
269 | case omp_atv_environment: | |
270 | case omp_atv_nearest: | |
271 | case omp_atv_blocked: | |
272 | case omp_atv_interleaved: | |
273 | data.partition = traits[i].value; | |
274 | break; | |
275 | default: | |
276 | return omp_null_allocator; | |
277 | } | |
278 | break; | |
279 | default: | |
280 | return omp_null_allocator; | |
281 | } | |
282 | ||
283 | if (data.alignment < sizeof (void *)) | |
284 | data.alignment = sizeof (void *); | |
285 | ||
17f52a1c JJ |
286 | switch (memspace) |
287 | { | |
288 | case omp_high_bw_mem_space: | |
289 | #ifdef LIBGOMP_USE_MEMKIND | |
290 | struct gomp_memkind_data *memkind_data; | |
291 | memkind_data = gomp_get_memkind (); | |
292 | if (data.partition == omp_atv_interleaved | |
293 | && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE]) | |
294 | { | |
295 | data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE; | |
296 | break; | |
297 | } | |
298 | else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED]) | |
299 | { | |
300 | data.memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
301 | break; | |
302 | } | |
303 | #endif | |
8c2fc744 | 304 | break; |
17f52a1c JJ |
305 | case omp_large_cap_mem_space: |
306 | #ifdef LIBGOMP_USE_MEMKIND | |
307 | memkind_data = gomp_get_memkind (); | |
308 | if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL]) | |
309 | data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
310 | else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM]) | |
311 | data.memkind = GOMP_MEMKIND_DAX_KMEM; | |
312 | #endif | |
313 | break; | |
314 | default: | |
315 | #ifdef LIBGOMP_USE_MEMKIND | |
316 | if (data.partition == omp_atv_interleaved) | |
317 | { | |
318 | memkind_data = gomp_get_memkind (); | |
319 | if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE]) | |
320 | data.memkind = GOMP_MEMKIND_INTERLEAVE; | |
321 | } | |
322 | #endif | |
323 | break; | |
324 | } | |
325 | ||
326 | /* No support for this so far. */ | |
327 | if (data.pinned) | |
e1071571 JJ |
328 | return omp_null_allocator; |
329 | ||
330 | ret = gomp_malloc (sizeof (struct omp_allocator_data)); | |
331 | *ret = data; | |
332 | #ifndef HAVE_SYNC_BUILTINS | |
333 | gomp_mutex_init (&ret->lock); | |
334 | #endif | |
335 | return (omp_allocator_handle_t) ret; | |
336 | } | |
337 | ||
338 | void | |
339 | omp_destroy_allocator (omp_allocator_handle_t allocator) | |
340 | { | |
341 | if (allocator != omp_null_allocator) | |
342 | { | |
343 | #ifndef HAVE_SYNC_BUILTINS | |
344 | gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock); | |
345 | #endif | |
346 | free ((void *) allocator); | |
347 | } | |
348 | } | |
349 | ||
fff15bad TB |
350 | ialias (omp_init_allocator) |
351 | ialias (omp_destroy_allocator) | |
352 | ||
b38a4bd1 | 353 | void * |
6fcc3cac JJ |
354 | omp_aligned_alloc (size_t alignment, size_t size, |
355 | omp_allocator_handle_t allocator) | |
e1071571 JJ |
356 | { |
357 | struct omp_allocator_data *allocator_data; | |
b38a4bd1 | 358 | size_t new_size, new_alignment; |
e1071571 | 359 | void *ptr, *ret; |
17f52a1c JJ |
360 | #ifdef LIBGOMP_USE_MEMKIND |
361 | enum gomp_memkind_kind memkind; | |
362 | #endif | |
e1071571 | 363 | |
05e4db63 JJ |
364 | if (__builtin_expect (size == 0, 0)) |
365 | return NULL; | |
366 | ||
e1071571 | 367 | retry: |
b38a4bd1 | 368 | new_alignment = alignment; |
e1071571 JJ |
369 | if (allocator == omp_null_allocator) |
370 | { | |
371 | struct gomp_thread *thr = gomp_thread (); | |
372 | if (thr->ts.def_allocator == omp_null_allocator) | |
373 | thr->ts.def_allocator = gomp_def_allocator; | |
374 | allocator = (omp_allocator_handle_t) thr->ts.def_allocator; | |
375 | } | |
376 | ||
377 | if (allocator > omp_max_predefined_alloc) | |
378 | { | |
379 | allocator_data = (struct omp_allocator_data *) allocator; | |
b38a4bd1 JJ |
380 | if (new_alignment < allocator_data->alignment) |
381 | new_alignment = allocator_data->alignment; | |
17f52a1c JJ |
382 | #ifdef LIBGOMP_USE_MEMKIND |
383 | memkind = allocator_data->memkind; | |
384 | #endif | |
e1071571 JJ |
385 | } |
386 | else | |
387 | { | |
388 | allocator_data = NULL; | |
b38a4bd1 JJ |
389 | if (new_alignment < sizeof (void *)) |
390 | new_alignment = sizeof (void *); | |
17f52a1c JJ |
391 | #ifdef LIBGOMP_USE_MEMKIND |
392 | memkind = GOMP_MEMKIND_NONE; | |
393 | if (allocator == omp_high_bw_mem_alloc) | |
394 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
395 | else if (allocator == omp_large_cap_mem_alloc) | |
396 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
397 | if (memkind) | |
398 | { | |
399 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
400 | if (!memkind_data->kinds[memkind]) | |
401 | memkind = GOMP_MEMKIND_NONE; | |
402 | } | |
403 | #endif | |
e1071571 JJ |
404 | } |
405 | ||
406 | new_size = sizeof (struct omp_mem_header); | |
b38a4bd1 JJ |
407 | if (new_alignment > sizeof (void *)) |
408 | new_size += new_alignment - sizeof (void *); | |
e1071571 JJ |
409 | if (__builtin_add_overflow (size, new_size, &new_size)) |
410 | goto fail; | |
411 | ||
412 | if (__builtin_expect (allocator_data | |
413 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
414 | { | |
415 | uintptr_t used_pool_size; | |
416 | if (new_size > allocator_data->pool_size) | |
417 | goto fail; | |
418 | #ifdef HAVE_SYNC_BUILTINS | |
419 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
420 | MEMMODEL_RELAXED); | |
421 | do | |
422 | { | |
423 | uintptr_t new_pool_size; | |
424 | if (__builtin_add_overflow (used_pool_size, new_size, | |
425 | &new_pool_size) | |
426 | || new_pool_size > allocator_data->pool_size) | |
427 | goto fail; | |
428 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
429 | &used_pool_size, new_pool_size, | |
430 | true, MEMMODEL_RELAXED, | |
431 | MEMMODEL_RELAXED)) | |
432 | break; | |
433 | } | |
434 | while (1); | |
435 | #else | |
436 | gomp_mutex_lock (&allocator_data->lock); | |
437 | if (__builtin_add_overflow (allocator_data->used_pool_size, new_size, | |
438 | &used_pool_size) | |
439 | || used_pool_size > allocator_data->pool_size) | |
440 | { | |
441 | gomp_mutex_unlock (&allocator_data->lock); | |
442 | goto fail; | |
443 | } | |
444 | allocator_data->used_pool_size = used_pool_size; | |
445 | gomp_mutex_unlock (&allocator_data->lock); | |
446 | #endif | |
17f52a1c JJ |
447 | #ifdef LIBGOMP_USE_MEMKIND |
448 | if (memkind) | |
449 | { | |
450 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
451 | void *kind = *memkind_data->kinds[memkind]; | |
452 | ptr = memkind_data->memkind_malloc (kind, new_size); | |
453 | } | |
454 | else | |
455 | #endif | |
456 | ptr = malloc (new_size); | |
e1071571 JJ |
457 | if (ptr == NULL) |
458 | { | |
459 | #ifdef HAVE_SYNC_BUILTINS | |
460 | __atomic_add_fetch (&allocator_data->used_pool_size, -new_size, | |
461 | MEMMODEL_RELAXED); | |
462 | #else | |
463 | gomp_mutex_lock (&allocator_data->lock); | |
464 | allocator_data->used_pool_size -= new_size; | |
465 | gomp_mutex_unlock (&allocator_data->lock); | |
466 | #endif | |
467 | goto fail; | |
468 | } | |
469 | } | |
470 | else | |
471 | { | |
17f52a1c JJ |
472 | #ifdef LIBGOMP_USE_MEMKIND |
473 | if (memkind) | |
474 | { | |
475 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
476 | void *kind = *memkind_data->kinds[memkind]; | |
477 | ptr = memkind_data->memkind_malloc (kind, new_size); | |
478 | } | |
479 | else | |
480 | #endif | |
481 | ptr = malloc (new_size); | |
e1071571 JJ |
482 | if (ptr == NULL) |
483 | goto fail; | |
484 | } | |
485 | ||
b38a4bd1 | 486 | if (new_alignment > sizeof (void *)) |
e1071571 JJ |
487 | ret = (void *) (((uintptr_t) ptr |
488 | + sizeof (struct omp_mem_header) | |
b38a4bd1 JJ |
489 | + new_alignment - sizeof (void *)) |
490 | & ~(new_alignment - 1)); | |
e1071571 JJ |
491 | else |
492 | ret = (char *) ptr + sizeof (struct omp_mem_header); | |
493 | ((struct omp_mem_header *) ret)[-1].ptr = ptr; | |
494 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
495 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
496 | return ret; | |
497 | ||
498 | fail: | |
499 | if (allocator_data) | |
500 | { | |
501 | switch (allocator_data->fallback) | |
502 | { | |
503 | case omp_atv_default_mem_fb: | |
b38a4bd1 | 504 | if ((new_alignment > sizeof (void *) && new_alignment > alignment) |
17f52a1c JJ |
505 | #ifdef LIBGOMP_USE_MEMKIND |
506 | || memkind | |
507 | #endif | |
e1071571 JJ |
508 | || (allocator_data |
509 | && allocator_data->pool_size < ~(uintptr_t) 0)) | |
510 | { | |
511 | allocator = omp_default_mem_alloc; | |
512 | goto retry; | |
513 | } | |
514 | /* Otherwise, we've already performed default mem allocation | |
515 | and if that failed, it won't succeed again (unless it was | |
b38a4bd1 | 516 | intermittent. Return NULL then, as that is the fallback. */ |
e1071571 JJ |
517 | break; |
518 | case omp_atv_null_fb: | |
519 | break; | |
520 | default: | |
521 | case omp_atv_abort_fb: | |
522 | gomp_fatal ("Out of memory allocating %lu bytes", | |
523 | (unsigned long) size); | |
524 | case omp_atv_allocator_fb: | |
525 | allocator = allocator_data->fb_data; | |
526 | goto retry; | |
527 | } | |
528 | } | |
529 | return NULL; | |
530 | } | |
531 | ||
b38a4bd1 JJ |
532 | ialias (omp_aligned_alloc) |
533 | ||
6fcc3cac JJ |
534 | void * |
535 | omp_alloc (size_t size, omp_allocator_handle_t allocator) | |
536 | { | |
b38a4bd1 | 537 | return ialias_call (omp_aligned_alloc) (1, size, allocator); |
6fcc3cac JJ |
538 | } |
539 | ||
540 | /* Like omp_aligned_alloc, but apply on top of that: | |
541 | "For allocations that arise from this ... the null_fb value of the | |
542 | fallback allocator trait behaves as if the abort_fb had been specified." */ | |
543 | ||
544 | void * | |
545 | GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator) | |
546 | { | |
b38a4bd1 JJ |
547 | void *ret |
548 | = ialias_call (omp_aligned_alloc) (alignment, size, | |
549 | (omp_allocator_handle_t) allocator); | |
6fcc3cac JJ |
550 | if (__builtin_expect (ret == NULL, 0) && size) |
551 | gomp_fatal ("Out of memory allocating %lu bytes", | |
552 | (unsigned long) size); | |
553 | return ret; | |
554 | } | |
555 | ||
e1071571 JJ |
556 | void |
557 | omp_free (void *ptr, omp_allocator_handle_t allocator) | |
558 | { | |
559 | struct omp_mem_header *data; | |
560 | ||
561 | if (ptr == NULL) | |
562 | return; | |
563 | (void) allocator; | |
564 | data = &((struct omp_mem_header *) ptr)[-1]; | |
565 | if (data->allocator > omp_max_predefined_alloc) | |
566 | { | |
567 | struct omp_allocator_data *allocator_data | |
568 | = (struct omp_allocator_data *) (data->allocator); | |
569 | if (allocator_data->pool_size < ~(uintptr_t) 0) | |
570 | { | |
571 | #ifdef HAVE_SYNC_BUILTINS | |
572 | __atomic_add_fetch (&allocator_data->used_pool_size, -data->size, | |
573 | MEMMODEL_RELAXED); | |
574 | #else | |
575 | gomp_mutex_lock (&allocator_data->lock); | |
23438370 | 576 | allocator_data->used_pool_size -= data->size; |
e1071571 JJ |
577 | gomp_mutex_unlock (&allocator_data->lock); |
578 | #endif | |
579 | } | |
17f52a1c JJ |
580 | #ifdef LIBGOMP_USE_MEMKIND |
581 | if (allocator_data->memkind) | |
582 | { | |
583 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
584 | void *kind = *memkind_data->kinds[allocator_data->memkind]; | |
585 | memkind_data->memkind_free (kind, data->ptr); | |
586 | return; | |
587 | } | |
588 | #endif | |
e1071571 | 589 | } |
17f52a1c JJ |
590 | #ifdef LIBGOMP_USE_MEMKIND |
591 | else | |
592 | { | |
593 | enum gomp_memkind_kind memkind = GOMP_MEMKIND_NONE; | |
594 | if (data->allocator == omp_high_bw_mem_alloc) | |
595 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
596 | else if (data->allocator == omp_large_cap_mem_alloc) | |
597 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
598 | if (memkind) | |
599 | { | |
600 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
601 | if (memkind_data->kinds[memkind]) | |
602 | { | |
603 | void *kind = *memkind_data->kinds[memkind]; | |
604 | memkind_data->memkind_free (kind, data->ptr); | |
605 | return; | |
606 | } | |
607 | } | |
608 | } | |
609 | #endif | |
e1071571 JJ |
610 | free (data->ptr); |
611 | } | |
6fcc3cac JJ |
612 | |
613 | ialias (omp_free) | |
614 | ||
615 | void | |
616 | GOMP_free (void *ptr, uintptr_t allocator) | |
617 | { | |
b38a4bd1 JJ |
618 | return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator); |
619 | } | |
620 | ||
621 | void * | |
622 | omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size, | |
623 | omp_allocator_handle_t allocator) | |
624 | { | |
625 | struct omp_allocator_data *allocator_data; | |
626 | size_t new_size, size_temp, new_alignment; | |
627 | void *ptr, *ret; | |
17f52a1c JJ |
628 | #ifdef LIBGOMP_USE_MEMKIND |
629 | enum gomp_memkind_kind memkind; | |
630 | #endif | |
b38a4bd1 JJ |
631 | |
632 | if (__builtin_expect (size == 0 || nmemb == 0, 0)) | |
633 | return NULL; | |
634 | ||
635 | retry: | |
636 | new_alignment = alignment; | |
637 | if (allocator == omp_null_allocator) | |
638 | { | |
639 | struct gomp_thread *thr = gomp_thread (); | |
640 | if (thr->ts.def_allocator == omp_null_allocator) | |
641 | thr->ts.def_allocator = gomp_def_allocator; | |
642 | allocator = (omp_allocator_handle_t) thr->ts.def_allocator; | |
643 | } | |
644 | ||
645 | if (allocator > omp_max_predefined_alloc) | |
646 | { | |
647 | allocator_data = (struct omp_allocator_data *) allocator; | |
648 | if (new_alignment < allocator_data->alignment) | |
649 | new_alignment = allocator_data->alignment; | |
17f52a1c JJ |
650 | #ifdef LIBGOMP_USE_MEMKIND |
651 | memkind = allocator_data->memkind; | |
652 | #endif | |
b38a4bd1 JJ |
653 | } |
654 | else | |
655 | { | |
656 | allocator_data = NULL; | |
657 | if (new_alignment < sizeof (void *)) | |
658 | new_alignment = sizeof (void *); | |
17f52a1c JJ |
659 | #ifdef LIBGOMP_USE_MEMKIND |
660 | memkind = GOMP_MEMKIND_NONE; | |
661 | if (allocator == omp_high_bw_mem_alloc) | |
662 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
663 | else if (allocator == omp_large_cap_mem_alloc) | |
664 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
665 | if (memkind) | |
666 | { | |
667 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
668 | if (!memkind_data->kinds[memkind]) | |
669 | memkind = GOMP_MEMKIND_NONE; | |
670 | } | |
671 | #endif | |
b38a4bd1 JJ |
672 | } |
673 | ||
674 | new_size = sizeof (struct omp_mem_header); | |
675 | if (new_alignment > sizeof (void *)) | |
676 | new_size += new_alignment - sizeof (void *); | |
677 | if (__builtin_mul_overflow (size, nmemb, &size_temp)) | |
678 | goto fail; | |
679 | if (__builtin_add_overflow (size_temp, new_size, &new_size)) | |
680 | goto fail; | |
681 | ||
682 | if (__builtin_expect (allocator_data | |
683 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
684 | { | |
685 | uintptr_t used_pool_size; | |
686 | if (new_size > allocator_data->pool_size) | |
687 | goto fail; | |
688 | #ifdef HAVE_SYNC_BUILTINS | |
689 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
690 | MEMMODEL_RELAXED); | |
691 | do | |
692 | { | |
693 | uintptr_t new_pool_size; | |
694 | if (__builtin_add_overflow (used_pool_size, new_size, | |
695 | &new_pool_size) | |
696 | || new_pool_size > allocator_data->pool_size) | |
697 | goto fail; | |
698 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
699 | &used_pool_size, new_pool_size, | |
700 | true, MEMMODEL_RELAXED, | |
701 | MEMMODEL_RELAXED)) | |
702 | break; | |
703 | } | |
704 | while (1); | |
705 | #else | |
706 | gomp_mutex_lock (&allocator_data->lock); | |
707 | if (__builtin_add_overflow (allocator_data->used_pool_size, new_size, | |
708 | &used_pool_size) | |
709 | || used_pool_size > allocator_data->pool_size) | |
710 | { | |
711 | gomp_mutex_unlock (&allocator_data->lock); | |
712 | goto fail; | |
713 | } | |
714 | allocator_data->used_pool_size = used_pool_size; | |
715 | gomp_mutex_unlock (&allocator_data->lock); | |
716 | #endif | |
17f52a1c JJ |
717 | #ifdef LIBGOMP_USE_MEMKIND |
718 | if (memkind) | |
719 | { | |
720 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
721 | void *kind = *memkind_data->kinds[memkind]; | |
722 | ptr = memkind_data->memkind_calloc (kind, 1, new_size); | |
723 | } | |
724 | else | |
725 | #endif | |
726 | ptr = calloc (1, new_size); | |
b38a4bd1 JJ |
727 | if (ptr == NULL) |
728 | { | |
729 | #ifdef HAVE_SYNC_BUILTINS | |
730 | __atomic_add_fetch (&allocator_data->used_pool_size, -new_size, | |
731 | MEMMODEL_RELAXED); | |
732 | #else | |
733 | gomp_mutex_lock (&allocator_data->lock); | |
734 | allocator_data->used_pool_size -= new_size; | |
735 | gomp_mutex_unlock (&allocator_data->lock); | |
736 | #endif | |
737 | goto fail; | |
738 | } | |
739 | } | |
740 | else | |
741 | { | |
17f52a1c JJ |
742 | #ifdef LIBGOMP_USE_MEMKIND |
743 | if (memkind) | |
744 | { | |
745 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
746 | void *kind = *memkind_data->kinds[memkind]; | |
747 | ptr = memkind_data->memkind_calloc (kind, 1, new_size); | |
748 | } | |
749 | else | |
750 | #endif | |
751 | ptr = calloc (1, new_size); | |
b38a4bd1 JJ |
752 | if (ptr == NULL) |
753 | goto fail; | |
754 | } | |
755 | ||
756 | if (new_alignment > sizeof (void *)) | |
757 | ret = (void *) (((uintptr_t) ptr | |
758 | + sizeof (struct omp_mem_header) | |
759 | + new_alignment - sizeof (void *)) | |
760 | & ~(new_alignment - 1)); | |
761 | else | |
762 | ret = (char *) ptr + sizeof (struct omp_mem_header); | |
763 | ((struct omp_mem_header *) ret)[-1].ptr = ptr; | |
764 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
765 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
766 | return ret; | |
767 | ||
768 | fail: | |
769 | if (allocator_data) | |
770 | { | |
771 | switch (allocator_data->fallback) | |
772 | { | |
773 | case omp_atv_default_mem_fb: | |
774 | if ((new_alignment > sizeof (void *) && new_alignment > alignment) | |
17f52a1c JJ |
775 | #ifdef LIBGOMP_USE_MEMKIND |
776 | || memkind | |
777 | #endif | |
b38a4bd1 JJ |
778 | || (allocator_data |
779 | && allocator_data->pool_size < ~(uintptr_t) 0)) | |
780 | { | |
781 | allocator = omp_default_mem_alloc; | |
782 | goto retry; | |
783 | } | |
784 | /* Otherwise, we've already performed default mem allocation | |
785 | and if that failed, it won't succeed again (unless it was | |
786 | intermittent. Return NULL then, as that is the fallback. */ | |
787 | break; | |
788 | case omp_atv_null_fb: | |
789 | break; | |
790 | default: | |
791 | case omp_atv_abort_fb: | |
792 | gomp_fatal ("Out of memory allocating %lu bytes", | |
793 | (unsigned long) (size * nmemb)); | |
794 | case omp_atv_allocator_fb: | |
795 | allocator = allocator_data->fb_data; | |
796 | goto retry; | |
797 | } | |
798 | } | |
799 | return NULL; | |
800 | } | |
801 | ||
802 | ialias (omp_aligned_calloc) | |
803 | ||
804 | void * | |
805 | omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator) | |
806 | { | |
807 | return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator); | |
808 | } | |
809 | ||
810 | void * | |
811 | omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator, | |
812 | omp_allocator_handle_t free_allocator) | |
813 | { | |
814 | struct omp_allocator_data *allocator_data, *free_allocator_data; | |
815 | size_t new_size, old_size, new_alignment, old_alignment; | |
816 | void *new_ptr, *ret; | |
817 | struct omp_mem_header *data; | |
17f52a1c JJ |
818 | #ifdef LIBGOMP_USE_MEMKIND |
819 | enum gomp_memkind_kind memkind, free_memkind; | |
820 | #endif | |
b38a4bd1 JJ |
821 | |
822 | if (__builtin_expect (ptr == NULL, 0)) | |
823 | return ialias_call (omp_aligned_alloc) (1, size, allocator); | |
824 | ||
825 | if (__builtin_expect (size == 0, 0)) | |
826 | { | |
827 | ialias_call (omp_free) (ptr, free_allocator); | |
828 | return NULL; | |
829 | } | |
830 | ||
831 | data = &((struct omp_mem_header *) ptr)[-1]; | |
832 | free_allocator = data->allocator; | |
833 | ||
834 | retry: | |
835 | new_alignment = sizeof (void *); | |
836 | if (allocator == omp_null_allocator) | |
837 | allocator = free_allocator; | |
838 | ||
839 | if (allocator > omp_max_predefined_alloc) | |
840 | { | |
841 | allocator_data = (struct omp_allocator_data *) allocator; | |
842 | if (new_alignment < allocator_data->alignment) | |
843 | new_alignment = allocator_data->alignment; | |
17f52a1c JJ |
844 | #ifdef LIBGOMP_USE_MEMKIND |
845 | memkind = allocator_data->memkind; | |
846 | #endif | |
b38a4bd1 JJ |
847 | } |
848 | else | |
17f52a1c JJ |
849 | { |
850 | allocator_data = NULL; | |
851 | #ifdef LIBGOMP_USE_MEMKIND | |
852 | memkind = GOMP_MEMKIND_NONE; | |
853 | if (allocator == omp_high_bw_mem_alloc) | |
854 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
855 | else if (allocator == omp_large_cap_mem_alloc) | |
856 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
857 | if (memkind) | |
858 | { | |
859 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
860 | if (!memkind_data->kinds[memkind]) | |
861 | memkind = GOMP_MEMKIND_NONE; | |
862 | } | |
863 | #endif | |
864 | } | |
b38a4bd1 | 865 | if (free_allocator > omp_max_predefined_alloc) |
17f52a1c JJ |
866 | { |
867 | free_allocator_data = (struct omp_allocator_data *) free_allocator; | |
868 | #ifdef LIBGOMP_USE_MEMKIND | |
869 | free_memkind = free_allocator_data->memkind; | |
870 | #endif | |
871 | } | |
b38a4bd1 | 872 | else |
17f52a1c JJ |
873 | { |
874 | free_allocator_data = NULL; | |
875 | #ifdef LIBGOMP_USE_MEMKIND | |
876 | free_memkind = GOMP_MEMKIND_NONE; | |
877 | if (free_allocator == omp_high_bw_mem_alloc) | |
878 | free_memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
879 | else if (free_allocator == omp_large_cap_mem_alloc) | |
880 | free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
881 | if (free_memkind) | |
882 | { | |
883 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
884 | if (!memkind_data->kinds[free_memkind]) | |
885 | free_memkind = GOMP_MEMKIND_NONE; | |
886 | } | |
887 | #endif | |
888 | } | |
b38a4bd1 JJ |
889 | old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr); |
890 | ||
891 | new_size = sizeof (struct omp_mem_header); | |
892 | if (new_alignment > sizeof (void *)) | |
893 | new_size += new_alignment - sizeof (void *); | |
894 | if (__builtin_add_overflow (size, new_size, &new_size)) | |
895 | goto fail; | |
896 | old_size = data->size; | |
897 | ||
898 | if (__builtin_expect (allocator_data | |
899 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
900 | { | |
901 | uintptr_t used_pool_size; | |
902 | size_t prev_size = 0; | |
903 | /* Check if we can use realloc. Don't use it if extra alignment | |
904 | was used previously or newly, because realloc might return a pointer | |
905 | with different alignment and then we'd need to memmove the data | |
906 | again. */ | |
907 | if (free_allocator_data | |
908 | && free_allocator_data == allocator_data | |
909 | && new_alignment == sizeof (void *) | |
910 | && old_alignment == sizeof (struct omp_mem_header)) | |
911 | prev_size = old_size; | |
912 | if (new_size > prev_size | |
913 | && new_size - prev_size > allocator_data->pool_size) | |
914 | goto fail; | |
915 | #ifdef HAVE_SYNC_BUILTINS | |
916 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
917 | MEMMODEL_RELAXED); | |
918 | do | |
919 | { | |
920 | uintptr_t new_pool_size; | |
921 | if (new_size > prev_size) | |
922 | { | |
923 | if (__builtin_add_overflow (used_pool_size, new_size - prev_size, | |
924 | &new_pool_size) | |
925 | || new_pool_size > allocator_data->pool_size) | |
926 | goto fail; | |
927 | } | |
928 | else | |
929 | new_pool_size = used_pool_size + new_size - prev_size; | |
930 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
931 | &used_pool_size, new_pool_size, | |
932 | true, MEMMODEL_RELAXED, | |
933 | MEMMODEL_RELAXED)) | |
934 | break; | |
935 | } | |
936 | while (1); | |
937 | #else | |
938 | gomp_mutex_lock (&allocator_data->lock); | |
939 | if (new_size > prev_size) | |
940 | { | |
941 | if (__builtin_add_overflow (allocator_data->used_pool_size, | |
942 | new_size - prev_size, | |
943 | &used_pool_size) | |
944 | || used_pool_size > allocator_data->pool_size) | |
945 | { | |
946 | gomp_mutex_unlock (&allocator_data->lock); | |
947 | goto fail; | |
948 | } | |
949 | } | |
950 | else | |
951 | used_pool_size = (allocator_data->used_pool_size | |
952 | + new_size - prev_size); | |
953 | allocator_data->used_pool_size = used_pool_size; | |
954 | gomp_mutex_unlock (&allocator_data->lock); | |
17f52a1c JJ |
955 | #endif |
956 | #ifdef LIBGOMP_USE_MEMKIND | |
957 | if (memkind) | |
958 | { | |
959 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
960 | void *kind = *memkind_data->kinds[memkind]; | |
961 | if (prev_size) | |
962 | new_ptr = memkind_data->memkind_realloc (kind, data->ptr, | |
963 | new_size); | |
964 | else | |
965 | new_ptr = memkind_data->memkind_malloc (kind, new_size); | |
966 | } | |
967 | else | |
b38a4bd1 JJ |
968 | #endif |
969 | if (prev_size) | |
970 | new_ptr = realloc (data->ptr, new_size); | |
971 | else | |
972 | new_ptr = malloc (new_size); | |
973 | if (new_ptr == NULL) | |
974 | { | |
975 | #ifdef HAVE_SYNC_BUILTINS | |
976 | __atomic_add_fetch (&allocator_data->used_pool_size, | |
977 | prev_size - new_size, | |
978 | MEMMODEL_RELAXED); | |
979 | #else | |
980 | gomp_mutex_lock (&allocator_data->lock); | |
981 | allocator_data->used_pool_size -= new_size - prev_size; | |
982 | gomp_mutex_unlock (&allocator_data->lock); | |
983 | #endif | |
984 | goto fail; | |
985 | } | |
986 | else if (prev_size) | |
987 | { | |
988 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
989 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
990 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
991 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
992 | return ret; | |
993 | } | |
994 | } | |
995 | else if (new_alignment == sizeof (void *) | |
996 | && old_alignment == sizeof (struct omp_mem_header) | |
17f52a1c JJ |
997 | #ifdef LIBGOMP_USE_MEMKIND |
998 | && memkind == free_memkind | |
999 | #endif | |
b38a4bd1 JJ |
1000 | && (free_allocator_data == NULL |
1001 | || free_allocator_data->pool_size == ~(uintptr_t) 0)) | |
1002 | { | |
17f52a1c JJ |
1003 | #ifdef LIBGOMP_USE_MEMKIND |
1004 | if (memkind) | |
1005 | { | |
1006 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1007 | void *kind = *memkind_data->kinds[memkind]; | |
1008 | new_ptr = memkind_data->memkind_realloc (kind, data->ptr, | |
1009 | new_size); | |
1010 | } | |
1011 | else | |
1012 | #endif | |
1013 | new_ptr = realloc (data->ptr, new_size); | |
b38a4bd1 JJ |
1014 | if (new_ptr == NULL) |
1015 | goto fail; | |
1016 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
1017 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
1018 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1019 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1020 | return ret; | |
1021 | } | |
1022 | else | |
1023 | { | |
17f52a1c JJ |
1024 | #ifdef LIBGOMP_USE_MEMKIND |
1025 | if (memkind) | |
1026 | { | |
1027 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1028 | void *kind = *memkind_data->kinds[memkind]; | |
1029 | new_ptr = memkind_data->memkind_malloc (kind, new_size); | |
1030 | } | |
1031 | else | |
1032 | #endif | |
1033 | new_ptr = malloc (new_size); | |
b38a4bd1 JJ |
1034 | if (new_ptr == NULL) |
1035 | goto fail; | |
1036 | } | |
1037 | ||
1038 | if (new_alignment > sizeof (void *)) | |
1039 | ret = (void *) (((uintptr_t) new_ptr | |
1040 | + sizeof (struct omp_mem_header) | |
1041 | + new_alignment - sizeof (void *)) | |
1042 | & ~(new_alignment - 1)); | |
1043 | else | |
1044 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
1045 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
1046 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1047 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1048 | if (old_size - old_alignment < size) | |
1049 | size = old_size - old_alignment; | |
1050 | memcpy (ret, ptr, size); | |
1051 | if (__builtin_expect (free_allocator_data | |
1052 | && free_allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
1053 | { | |
1054 | #ifdef HAVE_SYNC_BUILTINS | |
1055 | __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size, | |
1056 | MEMMODEL_RELAXED); | |
1057 | #else | |
1058 | gomp_mutex_lock (&free_allocator_data->lock); | |
1059 | free_allocator_data->used_pool_size -= data->size; | |
1060 | gomp_mutex_unlock (&free_allocator_data->lock); | |
1061 | #endif | |
1062 | } | |
17f52a1c JJ |
1063 | #ifdef LIBGOMP_USE_MEMKIND |
1064 | if (free_memkind) | |
1065 | { | |
1066 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1067 | void *kind = *memkind_data->kinds[free_memkind]; | |
1068 | memkind_data->memkind_free (kind, data->ptr); | |
1069 | return ret; | |
1070 | } | |
1071 | #endif | |
b38a4bd1 JJ |
1072 | free (data->ptr); |
1073 | return ret; | |
1074 | ||
1075 | fail: | |
1076 | if (allocator_data) | |
1077 | { | |
1078 | switch (allocator_data->fallback) | |
1079 | { | |
1080 | case omp_atv_default_mem_fb: | |
1081 | if (new_alignment > sizeof (void *) | |
17f52a1c JJ |
1082 | #ifdef LIBGOMP_USE_MEMKIND |
1083 | || memkind | |
1084 | #endif | |
b38a4bd1 JJ |
1085 | || (allocator_data |
1086 | && allocator_data->pool_size < ~(uintptr_t) 0)) | |
1087 | { | |
1088 | allocator = omp_default_mem_alloc; | |
1089 | goto retry; | |
1090 | } | |
1091 | /* Otherwise, we've already performed default mem allocation | |
1092 | and if that failed, it won't succeed again (unless it was | |
1093 | intermittent. Return NULL then, as that is the fallback. */ | |
1094 | break; | |
1095 | case omp_atv_null_fb: | |
1096 | break; | |
1097 | default: | |
1098 | case omp_atv_abort_fb: | |
1099 | gomp_fatal ("Out of memory allocating %lu bytes", | |
1100 | (unsigned long) size); | |
1101 | case omp_atv_allocator_fb: | |
1102 | allocator = allocator_data->fb_data; | |
1103 | goto retry; | |
1104 | } | |
1105 | } | |
1106 | return NULL; | |
6fcc3cac | 1107 | } |