]> git.ipfire.org Git - thirdparty/gcc.git/blame_incremental - libgomp/allocator.c
projects / thirdparty / gcc.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Fix some auto-profile issues
[thirdparty/gcc.git] / libgomp / allocator.c
... / ...
CommitLineData
1/* Copyright (C) 2020-2025 Free Software Foundation, Inc.
2 Contributed by Jakub Jelinek <jakub@redhat.com>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26/* This file contains wrappers for the system allocation routines. Most
27 places in the OpenMP API do not make any provision for failure, so in
28 general we cannot allow memory allocation to fail. */
29
30#define _GNU_SOURCE
31#include "libgomp.h"
32#include <stdlib.h>
33#include <string.h>
34#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
35#include <dlfcn.h>
36#endif
37
38/* Keeping track whether a Fortran scalar allocatable/pointer has been
39 allocated via 'omp allocators'/'omp allocate'. */
40
41struct fort_alloc_splay_tree_key_s {
42 void *ptr;
43};
44
45typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node;
46typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree;
47typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key;
48
49static inline int
50fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y)
51{
52 if (x->ptr < y->ptr)
53 return -1;
54 if (x->ptr > y->ptr)
55 return 1;
56 return 0;
57}
58#define splay_tree_prefix fort_alloc
59#define splay_tree_static
60#include "splay-tree.h"
61
62#define splay_tree_prefix fort_alloc
63#define splay_tree_static
64#define splay_tree_c
65#include "splay-tree.h"
66
67static struct fort_alloc_splay_tree_s fort_alloc_scalars;
68
69/* Add pointer as being alloced by GOMP_alloc. */
70void
71GOMP_add_alloc (void *ptr)
72{
73 if (ptr == NULL)
74 return;
75 fort_alloc_splay_tree_node item;
76 item = gomp_malloc (sizeof (struct splay_tree_node_s));
77 item->key.ptr = ptr;
78 item->left = NULL;
79 item->right = NULL;
80 fort_alloc_splay_tree_insert (&fort_alloc_scalars, item);
81}
82
83/* Remove pointer, either called by FREE or by REALLOC,
84 either of them can change the allocation status. */
85bool
86GOMP_is_alloc (void *ptr)
87{
88 struct fort_alloc_splay_tree_key_s needle;
89 fort_alloc_splay_tree_node n;
90 needle.ptr = ptr;
91 n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle);
92 if (n)
93 {
94 fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key);
95 free (n);
96 }
97 return n != NULL;
98}
99
100
101#define omp_max_predefined_alloc omp_thread_mem_alloc
102#define ompx_gnu_min_predefined_alloc ompx_gnu_pinned_mem_alloc
103#define ompx_gnu_max_predefined_alloc ompx_gnu_pinned_mem_alloc
104
105_Static_assert (GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc,
106 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
107_Static_assert (GOMP_OMPX_PREDEF_ALLOC_MIN == ompx_gnu_min_predefined_alloc,
108 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
109_Static_assert (GOMP_OMPX_PREDEF_ALLOC_MAX == ompx_gnu_max_predefined_alloc,
110 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
111_Static_assert (GOMP_OMP_PREDEF_ALLOC_THREADS == omp_thread_mem_alloc,
112 "GOMP_OMP_PREDEF_ALLOC_THREADS == omp_thread_mem_alloc");
113
114/* These macros may be overridden in config/<target>/allocator.c.
115 The defaults (no override) are to return NULL for pinned memory requests
116 and pass through to the regular OS calls otherwise.
117 The following definitions (ab)use comma operators to avoid unused
118 variable errors. */
119#ifndef MEMSPACE_ALLOC
120#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
121 (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
122#endif
123#ifndef MEMSPACE_CALLOC
124#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
125 (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
126#endif
127#ifndef MEMSPACE_REALLOC
128#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
129 ((PIN) || (OLDPIN) ? NULL \
130 : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
131#endif
132#ifndef MEMSPACE_FREE
133#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
134 if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
135#endif
136#ifndef MEMSPACE_VALIDATE
137#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
138 (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1))
139#endif
140
141/* Map the predefined allocators to the correct memory space.
142 The index to this table is the omp_allocator_handle_t enum value.
143 When the user calls omp_alloc with a predefined allocator this
144 table determines what memory they get. */
145static const omp_memspace_handle_t predefined_omp_alloc_mapping[] = {
146 omp_default_mem_space, /* omp_null_allocator doesn't actually use this. */
147 omp_default_mem_space, /* omp_default_mem_alloc. */
148 omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
149 omp_const_mem_space, /* omp_const_mem_alloc. */
150 omp_high_bw_mem_space, /* omp_high_bw_mem_alloc. */
151 omp_low_lat_mem_space, /* omp_low_lat_mem_alloc. */
152 omp_low_lat_mem_space, /* omp_cgroup_mem_alloc (implementation defined). */
153 omp_low_lat_mem_space, /* omp_pteam_mem_alloc (implementation defined). */
154 omp_low_lat_mem_space, /* omp_thread_mem_alloc (implementation defined). */
155};
156static const omp_memspace_handle_t predefined_ompx_gnu_alloc_mapping[] = {
157 omp_default_mem_space, /* ompx_gnu_pinned_mem_alloc. */
158};
159
160#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
161_Static_assert (ARRAY_SIZE (predefined_omp_alloc_mapping)
162 == omp_max_predefined_alloc + 1,
163 "predefined_omp_alloc_mapping must match omp_memspace_handle_t");
164_Static_assert (ARRAY_SIZE (predefined_ompx_gnu_alloc_mapping)
165 == (ompx_gnu_max_predefined_alloc
166 - ompx_gnu_min_predefined_alloc) + 1,
167 "predefined_ompx_gnu_alloc_mapping must match"
168 " omp_memspace_handle_t");
169
170static inline bool
171predefined_allocator_p (omp_allocator_handle_t allocator)
172{
173 return allocator <= ompx_gnu_max_predefined_alloc;
174}
175
176static inline omp_memspace_handle_t
177predefined_alloc_mapping (omp_allocator_handle_t allocator)
178{
179 if (allocator <= omp_max_predefined_alloc)
180 return predefined_omp_alloc_mapping[allocator];
181 else if (allocator >= ompx_gnu_min_predefined_alloc
182 && allocator <= ompx_gnu_max_predefined_alloc)
183 {
184 int index = allocator - ompx_gnu_min_predefined_alloc;
185 return predefined_ompx_gnu_alloc_mapping[index];
186 }
187 else
188 /* This should never happen. */
189 return omp_default_mem_space;
190}
191
192enum gomp_numa_memkind_kind
193{
194 GOMP_MEMKIND_NONE = 0,
195#define GOMP_MEMKIND_KINDS \
196 GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \
197 GOMP_MEMKIND_KIND (HBW_PREFERRED), \
198 GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \
199 GOMP_MEMKIND_KIND (DAX_KMEM), \
200 GOMP_MEMKIND_KIND (INTERLEAVE), \
201 GOMP_MEMKIND_KIND (DEFAULT)
202#define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
203 GOMP_MEMKIND_KINDS,
204#undef GOMP_MEMKIND_KIND
205 GOMP_MEMKIND_COUNT,
206 GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
207};
208
209struct omp_allocator_data
210{
211 omp_memspace_handle_t memspace;
212 omp_uintptr_t alignment;
213 omp_uintptr_t pool_size;
214 omp_uintptr_t used_pool_size;
215 omp_allocator_handle_t fb_data;
216 unsigned int sync_hint : 8;
217 unsigned int access : 8;
218 unsigned int fallback : 8;
219 unsigned int pinned : 1;
220 unsigned int partition : 7;
221#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
222 unsigned int memkind : 8;
223#endif
224#ifndef HAVE_SYNC_BUILTINS
225 gomp_mutex_t lock;
226#endif
227};
228
229struct omp_mem_header
230{
231 void *ptr;
232 size_t size;
233 omp_allocator_handle_t allocator;
234 void *pad;
235};
236
237struct gomp_libnuma_data
238{
239 void *numa_handle;
240 void *(*numa_alloc_local) (size_t);
241 void *(*numa_realloc) (void *, size_t, size_t);
242 void (*numa_free) (void *, size_t);
243};
244
245struct gomp_memkind_data
246{
247 void *memkind_handle;
248 void *(*memkind_malloc) (void *, size_t);
249 void *(*memkind_calloc) (void *, size_t, size_t);
250 void *(*memkind_realloc) (void *, void *, size_t);
251 void (*memkind_free) (void *, void *);
252 int (*memkind_check_available) (void *);
253 void **kinds[GOMP_MEMKIND_COUNT];
254};
255
256#ifdef LIBGOMP_USE_LIBNUMA
257static struct gomp_libnuma_data *libnuma_data;
258static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
259
260static void
261gomp_init_libnuma (void)
262{
263 void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
264 struct gomp_libnuma_data *data;
265
266 data = calloc (1, sizeof (struct gomp_libnuma_data));
267 if (data == NULL)
268 {
269 if (handle)
270 dlclose (handle);
271 return;
272 }
273 if (handle)
274 {
275 int (*numa_available) (void);
276 numa_available
277 = (__typeof (numa_available)) dlsym (handle, "numa_available");
278 if (!numa_available || numa_available () != 0)
279 {
280 dlclose (handle);
281 handle = NULL;
282 }
283 }
284 if (!handle)
285 {
286 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
287 return;
288 }
289 data->numa_handle = handle;
290 data->numa_alloc_local
291 = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
292 data->numa_realloc
293 = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
294 data->numa_free
295 = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
296 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
297}
298
299static struct gomp_libnuma_data *
300gomp_get_libnuma (void)
301{
302 struct gomp_libnuma_data *data
303 = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
304 if (data)
305 return data;
306 pthread_once (&libnuma_data_once, gomp_init_libnuma);
307 return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
308}
309#endif
310
311#ifdef LIBGOMP_USE_MEMKIND
312static struct gomp_memkind_data *memkind_data;
313static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
314
315static void
316gomp_init_memkind (void)
317{
318 void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
319 struct gomp_memkind_data *data;
320 int i;
321 static const char *kinds[] = {
322 NULL,
323#define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
324 GOMP_MEMKIND_KINDS
325#undef GOMP_MEMKIND_KIND
326 };
327
328 data = calloc (1, sizeof (struct gomp_memkind_data));
329 if (data == NULL)
330 {
331 if (handle)
332 dlclose (handle);
333 return;
334 }
335 if (!handle)
336 {
337 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
338 return;
339 }
340 data->memkind_handle = handle;
341 data->memkind_malloc
342 = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
343 data->memkind_calloc
344 = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
345 data->memkind_realloc
346 = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
347 data->memkind_free
348 = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
349 data->memkind_check_available
350 = (__typeof (data->memkind_check_available))
351 dlsym (handle, "memkind_check_available");
352 if (data->memkind_malloc
353 && data->memkind_calloc
354 && data->memkind_realloc
355 && data->memkind_free
356 && data->memkind_check_available)
357 for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
358 {
359 data->kinds[i] = (void **) dlsym (handle, kinds[i]);
360 if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
361 data->kinds[i] = NULL;
362 }
363 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
364}
365
366static struct gomp_memkind_data *
367gomp_get_memkind (void)
368{
369 struct gomp_memkind_data *data
370 = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
371 if (data)
372 return data;
373 pthread_once (&memkind_data_once, gomp_init_memkind);
374 return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
375}
376#endif
377
378omp_allocator_handle_t
379omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
380 const omp_alloctrait_t traits[])
381{
382 struct omp_allocator_data data
383 = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
384 omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
385#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
386 GOMP_MEMKIND_NONE
387#endif
388 };
389 struct omp_allocator_data *ret;
390 int i;
391
392 if (memspace > omp_low_lat_mem_space)
393 return omp_null_allocator;
394 for (i = 0; i < ntraits; i++)
395 switch (traits[i].key)
396 {
397 case omp_atk_sync_hint:
398 switch (traits[i].value)
399 {
400 case omp_atv_default:
401 data.sync_hint = omp_atv_contended;
402 break;
403 case omp_atv_contended:
404 case omp_atv_uncontended:
405 case omp_atv_serialized:
406 case omp_atv_private:
407 data.sync_hint = traits[i].value;
408 break;
409 default:
410 return omp_null_allocator;
411 }
412 break;
413 case omp_atk_alignment:
414 if (traits[i].value == omp_atv_default)
415 {
416 data.alignment = 1;
417 break;
418 }
419 if ((traits[i].value & (traits[i].value - 1)) != 0
420 || !traits[i].value)
421 return omp_null_allocator;
422 data.alignment = traits[i].value;
423 break;
424 case omp_atk_access:
425 switch (traits[i].value)
426 {
427 case omp_atv_default:
428 data.access = omp_atv_all;
429 break;
430 case omp_atv_all:
431 case omp_atv_cgroup:
432 case omp_atv_pteam:
433 case omp_atv_thread:
434 data.access = traits[i].value;
435 break;
436 default:
437 return omp_null_allocator;
438 }
439 break;
440 case omp_atk_pool_size:
441 if (traits[i].value == omp_atv_default)
442 data.pool_size = ~(uintptr_t) 0;
443 else
444 data.pool_size = traits[i].value;
445 break;
446 case omp_atk_fallback:
447 switch (traits[i].value)
448 {
449 case omp_atv_default:
450 data.fallback = omp_atv_default_mem_fb;
451 break;
452 case omp_atv_default_mem_fb:
453 case omp_atv_null_fb:
454 case omp_atv_abort_fb:
455 case omp_atv_allocator_fb:
456 data.fallback = traits[i].value;
457 break;
458 default:
459 return omp_null_allocator;
460 }
461 break;
462 case omp_atk_fb_data:
463 data.fb_data = traits[i].value;
464 break;
465 case omp_atk_pinned:
466 switch (traits[i].value)
467 {
468 case omp_atv_default:
469 case omp_atv_false:
470 data.pinned = omp_atv_false;
471 break;
472 case omp_atv_true:
473 data.pinned = omp_atv_true;
474 break;
475 default:
476 return omp_null_allocator;
477 }
478 break;
479 case omp_atk_partition:
480 switch (traits[i].value)
481 {
482 case omp_atv_default:
483 data.partition = omp_atv_environment;
484 break;
485 case omp_atv_environment:
486 case omp_atv_nearest:
487 case omp_atv_blocked:
488 case omp_atv_interleaved:
489 data.partition = traits[i].value;
490 break;
491 default:
492 return omp_null_allocator;
493 }
494 break;
495 default:
496 return omp_null_allocator;
497 }
498
499 if (data.alignment < sizeof (void *))
500 data.alignment = sizeof (void *);
501
502 switch (memspace)
503 {
504#ifdef LIBGOMP_USE_MEMKIND
505 case omp_high_bw_mem_space:
506 struct gomp_memkind_data *memkind_data;
507 memkind_data = gomp_get_memkind ();
508 if (data.partition == omp_atv_interleaved
509 && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
510 {
511 data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
512 break;
513 }
514 else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
515 {
516 data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
517 break;
518 }
519 break;
520 case omp_large_cap_mem_space:
521 memkind_data = gomp_get_memkind ();
522 if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
523 data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
524 else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
525 data.memkind = GOMP_MEMKIND_DAX_KMEM;
526 break;
527#endif
528 default:
529#ifdef LIBGOMP_USE_MEMKIND
530 if (data.partition == omp_atv_interleaved)
531 {
532 memkind_data = gomp_get_memkind ();
533 if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
534 data.memkind = GOMP_MEMKIND_INTERLEAVE;
535 }
536#endif
537 break;
538 }
539
540#ifdef LIBGOMP_USE_LIBNUMA
541 if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
542 {
543 libnuma_data = gomp_get_libnuma ();
544 if (libnuma_data->numa_alloc_local != NULL)
545 data.memkind = GOMP_MEMKIND_LIBNUMA;
546 }
547#endif
548
549 /* Reject unsupported memory spaces. */
550 if (!MEMSPACE_VALIDATE (data.memspace, data.access, data.pinned))
551 return omp_null_allocator;
552
553 ret = gomp_malloc (sizeof (struct omp_allocator_data));
554 *ret = data;
555#ifndef HAVE_SYNC_BUILTINS
556 gomp_mutex_init (&ret->lock);
557#endif
558 return (omp_allocator_handle_t) ret;
559}
560
561void
562omp_destroy_allocator (omp_allocator_handle_t allocator)
563{
564 if (allocator != omp_null_allocator)
565 {
566#ifndef HAVE_SYNC_BUILTINS
567 gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
568#endif
569 free ((void *) allocator);
570 }
571}
572
573ialias (omp_init_allocator)
574ialias (omp_destroy_allocator)
575
576void *
577omp_aligned_alloc (size_t alignment, size_t size,
578 omp_allocator_handle_t allocator)
579{
580 struct omp_allocator_data *allocator_data;
581 size_t new_size, new_alignment;
582 void *ptr, *ret;
583#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
584 enum gomp_numa_memkind_kind memkind;
585#endif
586
587 if (__builtin_expect (size == 0, 0))
588 return NULL;
589
590retry:
591 new_alignment = alignment;
592 if (allocator == omp_null_allocator)
593 {
594 struct gomp_thread *thr = gomp_thread ();
595 if (thr->ts.def_allocator == omp_null_allocator)
596 thr->ts.def_allocator = gomp_def_allocator;
597 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
598 }
599
600 if (!predefined_allocator_p (allocator))
601 {
602 allocator_data = (struct omp_allocator_data *) allocator;
603 if (new_alignment < allocator_data->alignment)
604 new_alignment = allocator_data->alignment;
605#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
606 memkind = allocator_data->memkind;
607#endif
608 }
609 else
610 {
611 allocator_data = NULL;
612 if (new_alignment < sizeof (void *))
613 new_alignment = sizeof (void *);
614#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
615 memkind = GOMP_MEMKIND_NONE;
616#endif
617#ifdef LIBGOMP_USE_MEMKIND
618 if (allocator == omp_high_bw_mem_alloc)
619 memkind = GOMP_MEMKIND_HBW_PREFERRED;
620 else if (allocator == omp_large_cap_mem_alloc)
621 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
622 if (memkind)
623 {
624 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
625 if (!memkind_data->kinds[memkind])
626 memkind = GOMP_MEMKIND_NONE;
627 }
628#endif
629 }
630
631 new_size = sizeof (struct omp_mem_header);
632 if (new_alignment > sizeof (void *))
633 new_size += new_alignment - sizeof (void *);
634 if (__builtin_add_overflow (size, new_size, &new_size))
635 goto fail;
636#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
637 if (allocator == omp_low_lat_mem_alloc)
638 goto fail;
639#endif
640
641 if (__builtin_expect (allocator_data
642 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
643 {
644 uintptr_t used_pool_size;
645 if (new_size > allocator_data->pool_size)
646 goto fail;
647#ifdef HAVE_SYNC_BUILTINS
648 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
649 MEMMODEL_RELAXED);
650 do
651 {
652 uintptr_t new_pool_size;
653 if (__builtin_add_overflow (used_pool_size, new_size,
654 &new_pool_size)
655 || new_pool_size > allocator_data->pool_size)
656 goto fail;
657 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
658 &used_pool_size, new_pool_size,
659 true, MEMMODEL_RELAXED,
660 MEMMODEL_RELAXED))
661 break;
662 }
663 while (1);
664#else
665 gomp_mutex_lock (&allocator_data->lock);
666 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
667 &used_pool_size)
668 || used_pool_size > allocator_data->pool_size)
669 {
670 gomp_mutex_unlock (&allocator_data->lock);
671 goto fail;
672 }
673 allocator_data->used_pool_size = used_pool_size;
674 gomp_mutex_unlock (&allocator_data->lock);
675#endif
676#ifdef LIBGOMP_USE_LIBNUMA
677 if (memkind == GOMP_MEMKIND_LIBNUMA)
678 ptr = libnuma_data->numa_alloc_local (new_size);
679# ifdef LIBGOMP_USE_MEMKIND
680 else
681# endif
682#endif
683#ifdef LIBGOMP_USE_MEMKIND
684 if (memkind)
685 {
686 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
687 void *kind = *memkind_data->kinds[memkind];
688 ptr = memkind_data->memkind_malloc (kind, new_size);
689 }
690 else
691#endif
692 ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
693 allocator_data->pinned);
694 if (ptr == NULL)
695 {
696#ifdef HAVE_SYNC_BUILTINS
697 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
698 MEMMODEL_RELAXED);
699#else
700 gomp_mutex_lock (&allocator_data->lock);
701 allocator_data->used_pool_size -= new_size;
702 gomp_mutex_unlock (&allocator_data->lock);
703#endif
704 goto fail;
705 }
706 }
707 else
708 {
709#ifdef LIBGOMP_USE_LIBNUMA
710 if (memkind == GOMP_MEMKIND_LIBNUMA)
711 ptr = libnuma_data->numa_alloc_local (new_size);
712# ifdef LIBGOMP_USE_MEMKIND
713 else
714# endif
715#endif
716#ifdef LIBGOMP_USE_MEMKIND
717 if (memkind)
718 {
719 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
720 void *kind = *memkind_data->kinds[memkind];
721 ptr = memkind_data->memkind_malloc (kind, new_size);
722 }
723 else
724#endif
725 {
726 omp_memspace_handle_t memspace;
727 memspace = (allocator_data
728 ? allocator_data->memspace
729 : predefined_alloc_mapping (allocator));
730 int pinned = (allocator_data
731 ? allocator_data->pinned
732 : allocator == ompx_gnu_pinned_mem_alloc);
733 ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
734 }
735 if (ptr == NULL)
736 goto fail;
737 }
738
739 if (new_alignment > sizeof (void *))
740 ret = (void *) (((uintptr_t) ptr
741 + sizeof (struct omp_mem_header)
742 + new_alignment - sizeof (void *))
743 & ~(new_alignment - 1));
744 else
745 ret = (char *) ptr + sizeof (struct omp_mem_header);
746 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
747 ((struct omp_mem_header *) ret)[-1].size = new_size;
748 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
749 return ret;
750
751fail:;
752 int fallback = (allocator_data
753 ? allocator_data->fallback
754 : (allocator == omp_default_mem_alloc
755 || allocator == ompx_gnu_pinned_mem_alloc)
756 ? omp_atv_null_fb
757 : omp_atv_default_mem_fb);
758 switch (fallback)
759 {
760 case omp_atv_default_mem_fb:
761 allocator = omp_default_mem_alloc;
762 goto retry;
763 case omp_atv_null_fb:
764 break;
765 default:
766 case omp_atv_abort_fb:
767 gomp_fatal ("Out of memory allocating %lu bytes",
768 (unsigned long) size);
769 case omp_atv_allocator_fb:
770 allocator = allocator_data->fb_data;
771 goto retry;
772 }
773 return NULL;
774}
775
776ialias (omp_aligned_alloc)
777
778void *
779omp_alloc (size_t size, omp_allocator_handle_t allocator)
780{
781 return ialias_call (omp_aligned_alloc) (1, size, allocator);
782}
783
784/* Like omp_aligned_alloc, but apply on top of that:
785 "For allocations that arise from this ... the null_fb value of the
786 fallback allocator trait behaves as if the abort_fb had been specified." */
787
788void *
789GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
790{
791 void *ret
792 = ialias_call (omp_aligned_alloc) (alignment, size,
793 (omp_allocator_handle_t) allocator);
794 if (__builtin_expect (ret == NULL, 0) && size)
795 gomp_fatal ("Out of memory allocating %lu bytes",
796 (unsigned long) size);
797 return ret;
798}
799
800void
801omp_free (void *ptr, omp_allocator_handle_t allocator)
802{
803 struct omp_mem_header *data;
804 omp_memspace_handle_t memspace = omp_default_mem_space;
805 int pinned = false;
806
807 if (ptr == NULL)
808 return;
809 (void) allocator;
810 data = &((struct omp_mem_header *) ptr)[-1];
811 if (!predefined_allocator_p (data->allocator))
812 {
813 struct omp_allocator_data *allocator_data
814 = (struct omp_allocator_data *) (data->allocator);
815 if (allocator_data->pool_size < ~(uintptr_t) 0)
816 {
817#ifdef HAVE_SYNC_BUILTINS
818 __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
819 MEMMODEL_RELAXED);
820#else
821 gomp_mutex_lock (&allocator_data->lock);
822 allocator_data->used_pool_size -= data->size;
823 gomp_mutex_unlock (&allocator_data->lock);
824#endif
825 }
826#ifdef LIBGOMP_USE_LIBNUMA
827 if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
828 {
829 libnuma_data->numa_free (data->ptr, data->size);
830 return;
831 }
832# ifdef LIBGOMP_USE_MEMKIND
833 else
834# endif
835#endif
836#ifdef LIBGOMP_USE_MEMKIND
837 if (allocator_data->memkind)
838 {
839 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
840 void *kind = *memkind_data->kinds[allocator_data->memkind];
841 memkind_data->memkind_free (kind, data->ptr);
842 return;
843 }
844#endif
845
846 memspace = allocator_data->memspace;
847 pinned = allocator_data->pinned;
848 }
849 else
850 {
851#ifdef LIBGOMP_USE_MEMKIND
852 enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
853 if (data->allocator == omp_high_bw_mem_alloc)
854 memkind = GOMP_MEMKIND_HBW_PREFERRED;
855 else if (data->allocator == omp_large_cap_mem_alloc)
856 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
857 if (memkind)
858 {
859 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
860 if (memkind_data->kinds[memkind])
861 {
862 void *kind = *memkind_data->kinds[memkind];
863 memkind_data->memkind_free (kind, data->ptr);
864 return;
865 }
866 }
867#endif
868
869 memspace = predefined_alloc_mapping (data->allocator);
870 pinned = (data->allocator == ompx_gnu_pinned_mem_alloc);
871 }
872
873 MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
874}
875
876ialias (omp_free)
877
878void
879GOMP_free (void *ptr, uintptr_t allocator)
880{
881 return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
882}
883
884void *
885omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
886 omp_allocator_handle_t allocator)
887{
888 struct omp_allocator_data *allocator_data;
889 size_t new_size, size_temp, new_alignment;
890 void *ptr, *ret;
891#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
892 enum gomp_numa_memkind_kind memkind;
893#endif
894
895 if (__builtin_expect (size == 0 || nmemb == 0, 0))
896 return NULL;
897
898retry:
899 new_alignment = alignment;
900 if (allocator == omp_null_allocator)
901 {
902 struct gomp_thread *thr = gomp_thread ();
903 if (thr->ts.def_allocator == omp_null_allocator)
904 thr->ts.def_allocator = gomp_def_allocator;
905 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
906 }
907
908 if (!predefined_allocator_p (allocator))
909 {
910 allocator_data = (struct omp_allocator_data *) allocator;
911 if (new_alignment < allocator_data->alignment)
912 new_alignment = allocator_data->alignment;
913#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
914 memkind = allocator_data->memkind;
915#endif
916 }
917 else
918 {
919 allocator_data = NULL;
920 if (new_alignment < sizeof (void *))
921 new_alignment = sizeof (void *);
922#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
923 memkind = GOMP_MEMKIND_NONE;
924#endif
925#ifdef LIBGOMP_USE_MEMKIND
926 if (allocator == omp_high_bw_mem_alloc)
927 memkind = GOMP_MEMKIND_HBW_PREFERRED;
928 else if (allocator == omp_large_cap_mem_alloc)
929 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
930 if (memkind)
931 {
932 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
933 if (!memkind_data->kinds[memkind])
934 memkind = GOMP_MEMKIND_NONE;
935 }
936#endif
937 }
938
939 new_size = sizeof (struct omp_mem_header);
940 if (new_alignment > sizeof (void *))
941 new_size += new_alignment - sizeof (void *);
942 if (__builtin_mul_overflow (size, nmemb, &size_temp))
943 goto fail;
944 if (__builtin_add_overflow (size_temp, new_size, &new_size))
945 goto fail;
946#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
947 if (allocator == omp_low_lat_mem_alloc)
948 goto fail;
949#endif
950
951 if (__builtin_expect (allocator_data
952 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
953 {
954 uintptr_t used_pool_size;
955 if (new_size > allocator_data->pool_size)
956 goto fail;
957#ifdef HAVE_SYNC_BUILTINS
958 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
959 MEMMODEL_RELAXED);
960 do
961 {
962 uintptr_t new_pool_size;
963 if (__builtin_add_overflow (used_pool_size, new_size,
964 &new_pool_size)
965 || new_pool_size > allocator_data->pool_size)
966 goto fail;
967 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
968 &used_pool_size, new_pool_size,
969 true, MEMMODEL_RELAXED,
970 MEMMODEL_RELAXED))
971 break;
972 }
973 while (1);
974#else
975 gomp_mutex_lock (&allocator_data->lock);
976 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
977 &used_pool_size)
978 || used_pool_size > allocator_data->pool_size)
979 {
980 gomp_mutex_unlock (&allocator_data->lock);
981 goto fail;
982 }
983 allocator_data->used_pool_size = used_pool_size;
984 gomp_mutex_unlock (&allocator_data->lock);
985#endif
986#ifdef LIBGOMP_USE_LIBNUMA
987 if (memkind == GOMP_MEMKIND_LIBNUMA)
988 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
989 memory that is initialized to zero. */
990 ptr = libnuma_data->numa_alloc_local (new_size);
991# ifdef LIBGOMP_USE_MEMKIND
992 else
993# endif
994#endif
995#ifdef LIBGOMP_USE_MEMKIND
996 if (memkind)
997 {
998 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
999 void *kind = *memkind_data->kinds[memkind];
1000 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
1001 }
1002 else
1003#endif
1004 ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size,
1005 allocator_data->pinned);
1006 if (ptr == NULL)
1007 {
1008#ifdef HAVE_SYNC_BUILTINS
1009 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
1010 MEMMODEL_RELAXED);
1011#else
1012 gomp_mutex_lock (&allocator_data->lock);
1013 allocator_data->used_pool_size -= new_size;
1014 gomp_mutex_unlock (&allocator_data->lock);
1015#endif
1016 goto fail;
1017 }
1018 }
1019 else
1020 {
1021#ifdef LIBGOMP_USE_LIBNUMA
1022 if (memkind == GOMP_MEMKIND_LIBNUMA)
1023 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
1024 memory that is initialized to zero. */
1025 ptr = libnuma_data->numa_alloc_local (new_size);
1026# ifdef LIBGOMP_USE_MEMKIND
1027 else
1028# endif
1029#endif
1030#ifdef LIBGOMP_USE_MEMKIND
1031 if (memkind)
1032 {
1033 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1034 void *kind = *memkind_data->kinds[memkind];
1035 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
1036 }
1037 else
1038#endif
1039 {
1040 omp_memspace_handle_t memspace;
1041 memspace = (allocator_data
1042 ? allocator_data->memspace
1043 : predefined_alloc_mapping (allocator));
1044 int pinned = (allocator_data
1045 ? allocator_data->pinned
1046 : allocator == ompx_gnu_pinned_mem_alloc);
1047 ptr = MEMSPACE_CALLOC (memspace, new_size, pinned);
1048 }
1049 if (ptr == NULL)
1050 goto fail;
1051 }
1052
1053 if (new_alignment > sizeof (void *))
1054 ret = (void *) (((uintptr_t) ptr
1055 + sizeof (struct omp_mem_header)
1056 + new_alignment - sizeof (void *))
1057 & ~(new_alignment - 1));
1058 else
1059 ret = (char *) ptr + sizeof (struct omp_mem_header);
1060 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
1061 ((struct omp_mem_header *) ret)[-1].size = new_size;
1062 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1063 return ret;
1064
1065fail:;
1066 int fallback = (allocator_data
1067 ? allocator_data->fallback
1068 : (allocator == omp_default_mem_alloc
1069 || allocator == ompx_gnu_pinned_mem_alloc)
1070 ? omp_atv_null_fb
1071 : omp_atv_default_mem_fb);
1072 switch (fallback)
1073 {
1074 case omp_atv_default_mem_fb:
1075 allocator = omp_default_mem_alloc;
1076 goto retry;
1077 case omp_atv_null_fb:
1078 break;
1079 default:
1080 case omp_atv_abort_fb:
1081 gomp_fatal ("Out of memory allocating %lu bytes",
1082 (unsigned long) (size * nmemb));
1083 case omp_atv_allocator_fb:
1084 allocator = allocator_data->fb_data;
1085 goto retry;
1086 }
1087 return NULL;
1088}
1089
1090ialias (omp_aligned_calloc)
1091
1092void *
1093omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
1094{
1095 return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
1096}
1097
1098void *
1099omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
1100 omp_allocator_handle_t free_allocator)
1101{
1102 struct omp_allocator_data *allocator_data, *free_allocator_data;
1103 size_t new_size, old_size, new_alignment, old_alignment;
1104 void *new_ptr, *ret;
1105 struct omp_mem_header *data;
1106#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1107 enum gomp_numa_memkind_kind memkind, free_memkind;
1108#endif
1109
1110 if (__builtin_expect (ptr == NULL, 0))
1111 return ialias_call (omp_aligned_alloc) (1, size, allocator);
1112
1113 if (__builtin_expect (size == 0, 0))
1114 {
1115 ialias_call (omp_free) (ptr, free_allocator);
1116 return NULL;
1117 }
1118
1119 data = &((struct omp_mem_header *) ptr)[-1];
1120 free_allocator = data->allocator;
1121
1122retry:
1123 new_alignment = sizeof (void *);
1124 if (allocator == omp_null_allocator)
1125 allocator = free_allocator;
1126
1127 if (!predefined_allocator_p (allocator))
1128 {
1129 allocator_data = (struct omp_allocator_data *) allocator;
1130 if (new_alignment < allocator_data->alignment)
1131 new_alignment = allocator_data->alignment;
1132#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1133 memkind = allocator_data->memkind;
1134#endif
1135 }
1136 else
1137 {
1138 allocator_data = NULL;
1139#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1140 memkind = GOMP_MEMKIND_NONE;
1141#endif
1142#ifdef LIBGOMP_USE_MEMKIND
1143 if (allocator == omp_high_bw_mem_alloc)
1144 memkind = GOMP_MEMKIND_HBW_PREFERRED;
1145 else if (allocator == omp_large_cap_mem_alloc)
1146 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1147 if (memkind)
1148 {
1149 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1150 if (!memkind_data->kinds[memkind])
1151 memkind = GOMP_MEMKIND_NONE;
1152 }
1153#endif
1154 }
1155 if (!predefined_allocator_p (free_allocator))
1156 {
1157 free_allocator_data = (struct omp_allocator_data *) free_allocator;
1158#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1159 free_memkind = free_allocator_data->memkind;
1160#endif
1161 }
1162 else
1163 {
1164 free_allocator_data = NULL;
1165#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1166 free_memkind = GOMP_MEMKIND_NONE;
1167#endif
1168#ifdef LIBGOMP_USE_MEMKIND
1169 if (free_allocator == omp_high_bw_mem_alloc)
1170 free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1171 else if (free_allocator == omp_large_cap_mem_alloc)
1172 free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1173 if (free_memkind)
1174 {
1175 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1176 if (!memkind_data->kinds[free_memkind])
1177 free_memkind = GOMP_MEMKIND_NONE;
1178 }
1179#endif
1180 }
1181 old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1182
1183 new_size = sizeof (struct omp_mem_header);
1184 if (new_alignment > sizeof (void *))
1185 new_size += new_alignment - sizeof (void *);
1186 if (__builtin_add_overflow (size, new_size, &new_size))
1187 goto fail;
1188 old_size = data->size;
1189#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1190 if (allocator == omp_low_lat_mem_alloc)
1191 goto fail;
1192#endif
1193
1194 if (__builtin_expect (allocator_data
1195 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1196 {
1197 uintptr_t used_pool_size;
1198 size_t prev_size = 0;
1199 /* Check if we can use realloc. Don't use it if extra alignment
1200 was used previously or newly, because realloc might return a pointer
1201 with different alignment and then we'd need to memmove the data
1202 again. */
1203 if (free_allocator_data
1204 && free_allocator_data == allocator_data
1205 && new_alignment == sizeof (void *)
1206 && old_alignment == sizeof (struct omp_mem_header))
1207 prev_size = old_size;
1208 if (new_size > prev_size
1209 && new_size - prev_size > allocator_data->pool_size)
1210 goto fail;
1211#ifdef HAVE_SYNC_BUILTINS
1212 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1213 MEMMODEL_RELAXED);
1214 do
1215 {
1216 uintptr_t new_pool_size;
1217 if (new_size > prev_size)
1218 {
1219 if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1220 &new_pool_size)
1221 || new_pool_size > allocator_data->pool_size)
1222 goto fail;
1223 }
1224 else
1225 new_pool_size = used_pool_size + new_size - prev_size;
1226 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1227 &used_pool_size, new_pool_size,
1228 true, MEMMODEL_RELAXED,
1229 MEMMODEL_RELAXED))
1230 break;
1231 }
1232 while (1);
1233#else
1234 gomp_mutex_lock (&allocator_data->lock);
1235 if (new_size > prev_size)
1236 {
1237 if (__builtin_add_overflow (allocator_data->used_pool_size,
1238 new_size - prev_size,
1239 &used_pool_size)
1240 || used_pool_size > allocator_data->pool_size)
1241 {
1242 gomp_mutex_unlock (&allocator_data->lock);
1243 goto fail;
1244 }
1245 }
1246 else
1247 used_pool_size = (allocator_data->used_pool_size
1248 + new_size - prev_size);
1249 allocator_data->used_pool_size = used_pool_size;
1250 gomp_mutex_unlock (&allocator_data->lock);
1251#endif
1252#ifdef LIBGOMP_USE_LIBNUMA
1253 if (memkind == GOMP_MEMKIND_LIBNUMA)
1254 {
1255 if (prev_size)
1256 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1257 new_size);
1258 else
1259 new_ptr = libnuma_data->numa_alloc_local (new_size);
1260 }
1261# ifdef LIBGOMP_USE_MEMKIND
1262 else
1263# endif
1264#endif
1265#ifdef LIBGOMP_USE_MEMKIND
1266 if (memkind)
1267 {
1268 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1269 void *kind = *memkind_data->kinds[memkind];
1270 if (prev_size)
1271 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1272 new_size);
1273 else
1274 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1275 }
1276 else
1277#endif
1278 if (prev_size)
1279 {
1280 int was_pinned = (free_allocator_data
1281 ? free_allocator_data->pinned
1282 : free_allocator == ompx_gnu_pinned_mem_alloc);
1283 new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
1284 data->size, new_size, was_pinned,
1285 allocator_data->pinned);
1286 }
1287 else
1288 new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
1289 allocator_data->pinned);
1290 if (new_ptr == NULL)
1291 {
1292#ifdef HAVE_SYNC_BUILTINS
1293 __atomic_add_fetch (&allocator_data->used_pool_size,
1294 prev_size - new_size,
1295 MEMMODEL_RELAXED);
1296#else
1297 gomp_mutex_lock (&allocator_data->lock);
1298 allocator_data->used_pool_size -= new_size - prev_size;
1299 gomp_mutex_unlock (&allocator_data->lock);
1300#endif
1301 goto fail;
1302 }
1303 else if (prev_size)
1304 {
1305 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1306 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1307 ((struct omp_mem_header *) ret)[-1].size = new_size;
1308 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1309 return ret;
1310 }
1311 }
1312 else if (new_alignment == sizeof (void *)
1313 && old_alignment == sizeof (struct omp_mem_header)
1314#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1315 && memkind == free_memkind
1316#endif
1317 && (free_allocator_data == NULL
1318 || free_allocator_data->pool_size == ~(uintptr_t) 0))
1319 {
1320#ifdef LIBGOMP_USE_LIBNUMA
1321 if (memkind == GOMP_MEMKIND_LIBNUMA)
1322 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1323# ifdef LIBGOMP_USE_MEMKIND
1324 else
1325# endif
1326#endif
1327#ifdef LIBGOMP_USE_MEMKIND
1328 if (memkind)
1329 {
1330 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1331 void *kind = *memkind_data->kinds[memkind];
1332 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1333 new_size);
1334 }
1335 else
1336#endif
1337 {
1338 omp_memspace_handle_t memspace;
1339 memspace = (allocator_data
1340 ? allocator_data->memspace
1341 : predefined_alloc_mapping (allocator));
1342 int was_pinned = (free_allocator_data
1343 ? free_allocator_data->pinned
1344 : free_allocator == ompx_gnu_pinned_mem_alloc);
1345 int pinned = (allocator_data
1346 ? allocator_data->pinned
1347 : allocator == ompx_gnu_pinned_mem_alloc);
1348 new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
1349 was_pinned, pinned);
1350 }
1351 if (new_ptr == NULL)
1352 goto fail;
1353
1354 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1355 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1356 ((struct omp_mem_header *) ret)[-1].size = new_size;
1357 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1358 return ret;
1359 }
1360 else
1361 {
1362#ifdef LIBGOMP_USE_LIBNUMA
1363 if (memkind == GOMP_MEMKIND_LIBNUMA)
1364 new_ptr = libnuma_data->numa_alloc_local (new_size);
1365# ifdef LIBGOMP_USE_MEMKIND
1366 else
1367# endif
1368#endif
1369#ifdef LIBGOMP_USE_MEMKIND
1370 if (memkind)
1371 {
1372 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1373 void *kind = *memkind_data->kinds[memkind];
1374 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1375 }
1376 else
1377#endif
1378 {
1379 omp_memspace_handle_t memspace;
1380 memspace = (allocator_data
1381 ? allocator_data->memspace
1382 : predefined_alloc_mapping (allocator));
1383 int pinned = (allocator_data
1384 ? allocator_data->pinned
1385 : allocator == ompx_gnu_pinned_mem_alloc);
1386 new_ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
1387 }
1388 if (new_ptr == NULL)
1389 goto fail;
1390 }
1391
1392 if (new_alignment > sizeof (void *))
1393 ret = (void *) (((uintptr_t) new_ptr
1394 + sizeof (struct omp_mem_header)
1395 + new_alignment - sizeof (void *))
1396 & ~(new_alignment - 1));
1397 else
1398 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1399 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1400 ((struct omp_mem_header *) ret)[-1].size = new_size;
1401 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1402 if (old_size - old_alignment < size)
1403 size = old_size - old_alignment;
1404 memcpy (ret, ptr, size);
1405 if (__builtin_expect (free_allocator_data
1406 && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1407 {
1408#ifdef HAVE_SYNC_BUILTINS
1409 __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1410 MEMMODEL_RELAXED);
1411#else
1412 gomp_mutex_lock (&free_allocator_data->lock);
1413 free_allocator_data->used_pool_size -= data->size;
1414 gomp_mutex_unlock (&free_allocator_data->lock);
1415#endif
1416 }
1417#ifdef LIBGOMP_USE_LIBNUMA
1418 if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1419 {
1420 libnuma_data->numa_free (data->ptr, data->size);
1421 return ret;
1422 }
1423# ifdef LIBGOMP_USE_MEMKIND
1424 else
1425# endif
1426#endif
1427#ifdef LIBGOMP_USE_MEMKIND
1428 if (free_memkind)
1429 {
1430 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1431 void *kind = *memkind_data->kinds[free_memkind];
1432 memkind_data->memkind_free (kind, data->ptr);
1433 return ret;
1434 }
1435#endif
1436 {
1437 omp_memspace_handle_t was_memspace;
1438 was_memspace = (free_allocator_data
1439 ? free_allocator_data->memspace
1440 : predefined_alloc_mapping (free_allocator));
1441 int was_pinned = (free_allocator_data
1442 ? free_allocator_data->pinned
1443 : free_allocator == ompx_gnu_pinned_mem_alloc);
1444 MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
1445 }
1446 return ret;
1447
1448fail:;
1449 int fallback = (allocator_data
1450 ? allocator_data->fallback
1451 : (allocator == omp_default_mem_alloc
1452 || allocator == ompx_gnu_pinned_mem_alloc)
1453 ? omp_atv_null_fb
1454 : omp_atv_default_mem_fb);
1455 switch (fallback)
1456 {
1457 case omp_atv_default_mem_fb:
1458 allocator = omp_default_mem_alloc;
1459 goto retry;
1460 case omp_atv_null_fb:
1461 break;
1462 default:
1463 case omp_atv_abort_fb:
1464 gomp_fatal ("Out of memory allocating %lu bytes",
1465 (unsigned long) size);
1466 case omp_atv_allocator_fb:
1467 allocator = allocator_data->fb_data;
1468 goto retry;
1469 }
1470 return NULL;
1471}
Mirror of https://gcc.gnu.org/git/gcc.git