]> git.ipfire.org Git - thirdparty/gcc.git/blame - libgomp/allocator.c
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
lra: Reallow reloading user hard registers if the insn is not asm [PR 120983]
[thirdparty/gcc.git] / libgomp / allocator.c
CommitLineData
6441eb6d 1/* Copyright (C) 2020-2025 Free Software Foundation, Inc.
e1071571
JJ		 2 Contributed by Jakub Jelinek <jakub@redhat.com>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26/* This file contains wrappers for the system allocation routines. Most
27 places in the OpenMP API do not make any provision for failure, so in
28 general we cannot allow memory allocation to fail. */
29
30#define _GNU_SOURCE
31#include "libgomp.h"
32#include <stdlib.h>
b38a4bd1 33#include <string.h>
450b05ce 34#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 35#include <dlfcn.h>
36#endif
e1071571 37
d4b6d147
TB		 38/* Keeping track whether a Fortran scalar allocatable/pointer has been
39 allocated via 'omp allocators'/'omp allocate'. */
40
41struct fort_alloc_splay_tree_key_s {
42 void *ptr;
43};
44
45typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node;
46typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree;
47typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key;
48
49static inline int
50fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y)
51{
52 if (x->ptr < y->ptr)
53 return -1;
54 if (x->ptr > y->ptr)
55 return 1;
56 return 0;
57}
58#define splay_tree_prefix fort_alloc
59#define splay_tree_static
60#include "splay-tree.h"
61
62#define splay_tree_prefix fort_alloc
63#define splay_tree_static
64#define splay_tree_c
65#include "splay-tree.h"
66
67static struct fort_alloc_splay_tree_s fort_alloc_scalars;
68
69/* Add pointer as being alloced by GOMP_alloc. */
70void
71GOMP_add_alloc (void *ptr)
72{
73 if (ptr == NULL)
74 return;
75 fort_alloc_splay_tree_node item;
76 item = gomp_malloc (sizeof (struct splay_tree_node_s));
77 item->key.ptr = ptr;
78 item->left = NULL;
79 item->right = NULL;
80 fort_alloc_splay_tree_insert (&fort_alloc_scalars, item);
81}
82
83/* Remove pointer, either called by FREE or by REALLOC,
84 either of them can change the allocation status. */
85bool
86GOMP_is_alloc (void *ptr)
87{
88 struct fort_alloc_splay_tree_key_s needle;
89 fort_alloc_splay_tree_node n;
90 needle.ptr = ptr;
91 n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle);
92 if (n)
93 {
94 fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key);
95 free (n);
96 }
97 return n != NULL;
98}
99
100
e1071571 101#define omp_max_predefined_alloc omp_thread_mem_alloc
64001441
AS		 102#define ompx_gnu_min_predefined_alloc ompx_gnu_pinned_mem_alloc
103#define ompx_gnu_max_predefined_alloc ompx_gnu_pinned_mem_alloc
e1071571 104
a8caeaac
TB		 105_Static_assert (GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc,
106 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
107_Static_assert (GOMP_OMPX_PREDEF_ALLOC_MIN == ompx_gnu_min_predefined_alloc,
108 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
109_Static_assert (GOMP_OMPX_PREDEF_ALLOC_MAX == ompx_gnu_max_predefined_alloc,
110 "GOMP_OMP_PREDEF_ALLOC_MAX == omp_thread_mem_alloc");
111_Static_assert (GOMP_OMP_PREDEF_ALLOC_THREADS == omp_thread_mem_alloc,
112 "GOMP_OMP_PREDEF_ALLOC_THREADS == omp_thread_mem_alloc");
113
30486fab 114/* These macros may be overridden in config/<target>/allocator.c.
348874f0
AS		 115 The defaults (no override) are to return NULL for pinned memory requests
116 and pass through to the regular OS calls otherwise.
30486fab
AS		 117 The following definitions (ab)use comma operators to avoid unused
118 variable errors. */
119#ifndef MEMSPACE_ALLOC
348874f0
AS		 120#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
121 (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
30486fab
AS		 122#endif
123#ifndef MEMSPACE_CALLOC
348874f0
AS		 124#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
125 (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
30486fab
AS		 126#endif
127#ifndef MEMSPACE_REALLOC
348874f0
AS		 128#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
129 ((PIN) || (OLDPIN) ? NULL \
130 : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
30486fab
AS		 131#endif
132#ifndef MEMSPACE_FREE
348874f0
AS		 133#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
134 if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
30486fab 135#endif
e9a19ead 136#ifndef MEMSPACE_VALIDATE
348874f0
AS		 137#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
138 (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1))
e9a19ead 139#endif
30486fab
AS		 140
141/* Map the predefined allocators to the correct memory space.
142 The index to this table is the omp_allocator_handle_t enum value.
143 When the user calls omp_alloc with a predefined allocator this
144 table determines what memory they get. */
64001441 145static const omp_memspace_handle_t predefined_omp_alloc_mapping[] = {
30486fab
AS		 146 omp_default_mem_space, /* omp_null_allocator doesn't actually use this. */
147 omp_default_mem_space, /* omp_default_mem_alloc. */
148 omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
149 omp_const_mem_space, /* omp_const_mem_alloc. */
150 omp_high_bw_mem_space, /* omp_high_bw_mem_alloc. */
151 omp_low_lat_mem_space, /* omp_low_lat_mem_alloc. */
152 omp_low_lat_mem_space, /* omp_cgroup_mem_alloc (implementation defined). */
153 omp_low_lat_mem_space, /* omp_pteam_mem_alloc (implementation defined). */
154 omp_low_lat_mem_space, /* omp_thread_mem_alloc (implementation defined). */
155};
64001441
AS		 156static const omp_memspace_handle_t predefined_ompx_gnu_alloc_mapping[] = {
157 omp_default_mem_space, /* ompx_gnu_pinned_mem_alloc. */
158};
30486fab
AS		 159
160#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
64001441 161_Static_assert (ARRAY_SIZE (predefined_omp_alloc_mapping)
30486fab 162 == omp_max_predefined_alloc + 1,
64001441
AS		 163 "predefined_omp_alloc_mapping must match omp_memspace_handle_t");
164_Static_assert (ARRAY_SIZE (predefined_ompx_gnu_alloc_mapping)
165 == (ompx_gnu_max_predefined_alloc
166 - ompx_gnu_min_predefined_alloc) + 1,
167 "predefined_ompx_gnu_alloc_mapping must match"
168 " omp_memspace_handle_t");
169
170static inline bool
171predefined_allocator_p (omp_allocator_handle_t allocator)
172{
173 return allocator <= ompx_gnu_max_predefined_alloc;
174}
175
176static inline omp_memspace_handle_t
177predefined_alloc_mapping (omp_allocator_handle_t allocator)
178{
179 if (allocator <= omp_max_predefined_alloc)
180 return predefined_omp_alloc_mapping[allocator];
181 else if (allocator >= ompx_gnu_min_predefined_alloc
182 && allocator <= ompx_gnu_max_predefined_alloc)
183 {
184 int index = allocator - ompx_gnu_min_predefined_alloc;
185 return predefined_ompx_gnu_alloc_mapping[index];
186 }
187 else
188 /* This should never happen. */
189 return omp_default_mem_space;
190}
30486fab 191
450b05ce 192enum gomp_numa_memkind_kind
17f52a1c
JJ		 193{
194 GOMP_MEMKIND_NONE = 0,
195#define GOMP_MEMKIND_KINDS \
196 GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \
197 GOMP_MEMKIND_KIND (HBW_PREFERRED), \
198 GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \
199 GOMP_MEMKIND_KIND (DAX_KMEM), \
200 GOMP_MEMKIND_KIND (INTERLEAVE), \
201 GOMP_MEMKIND_KIND (DEFAULT)
202#define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
203 GOMP_MEMKIND_KINDS,
204#undef GOMP_MEMKIND_KIND
450b05ce
TB		 205 GOMP_MEMKIND_COUNT,
206 GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
17f52a1c
JJ		 207};
208
e1071571
JJ		 209struct omp_allocator_data
210{
211 omp_memspace_handle_t memspace;
212 omp_uintptr_t alignment;
213 omp_uintptr_t pool_size;
214 omp_uintptr_t used_pool_size;
215 omp_allocator_handle_t fb_data;
216 unsigned int sync_hint : 8;
217 unsigned int access : 8;
218 unsigned int fallback : 8;
219 unsigned int pinned : 1;
220 unsigned int partition : 7;
450b05ce 221#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 222 unsigned int memkind : 8;
223#endif
e1071571
JJ		 224#ifndef HAVE_SYNC_BUILTINS
225 gomp_mutex_t lock;
226#endif
227};
228
229struct omp_mem_header
230{
231 void *ptr;
232 size_t size;
233 omp_allocator_handle_t allocator;
234 void *pad;
235};
236
450b05ce
TB		 237struct gomp_libnuma_data
238{
239 void *numa_handle;
240 void *(*numa_alloc_local) (size_t);
241 void *(*numa_realloc) (void *, size_t, size_t);
242 void (*numa_free) (void *, size_t);
243};
244
17f52a1c
JJ		 245struct gomp_memkind_data
246{
247 void *memkind_handle;
248 void *(*memkind_malloc) (void *, size_t);
249 void *(*memkind_calloc) (void *, size_t, size_t);
250 void *(*memkind_realloc) (void *, void *, size_t);
251 void (*memkind_free) (void *, void *);
252 int (*memkind_check_available) (void *);
253 void **kinds[GOMP_MEMKIND_COUNT];
254};
255
450b05ce
TB		 256#ifdef LIBGOMP_USE_LIBNUMA
257static struct gomp_libnuma_data *libnuma_data;
258static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
259
260static void
261gomp_init_libnuma (void)
262{
263 void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
264 struct gomp_libnuma_data *data;
265
266 data = calloc (1, sizeof (struct gomp_libnuma_data));
267 if (data == NULL)
268 {
269 if (handle)
270 dlclose (handle);
271 return;
272 }
8f3c4517
TB		 273 if (handle)
274 {
275 int (*numa_available) (void);
276 numa_available
277 = (__typeof (numa_available)) dlsym (handle, "numa_available");
278 if (!numa_available || numa_available () != 0)
279 {
280 dlclose (handle);
281 handle = NULL;
282 }
283 }
450b05ce
TB		 284 if (!handle)
285 {
286 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
287 return;
288 }
289 data->numa_handle = handle;
290 data->numa_alloc_local
291 = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
292 data->numa_realloc
293 = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
294 data->numa_free
295 = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
296 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
297}
298
299static struct gomp_libnuma_data *
300gomp_get_libnuma (void)
301{
302 struct gomp_libnuma_data *data
303 = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
304 if (data)
305 return data;
306 pthread_once (&libnuma_data_once, gomp_init_libnuma);
307 return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
308}
309#endif
310
17f52a1c
JJ		 311#ifdef LIBGOMP_USE_MEMKIND
312static struct gomp_memkind_data *memkind_data;
313static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
314
315static void
316gomp_init_memkind (void)
317{
1eff4872 318 void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
17f52a1c
JJ		 319 struct gomp_memkind_data *data;
320 int i;
321 static const char *kinds[] = {
322 NULL,
323#define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
324 GOMP_MEMKIND_KINDS
325#undef GOMP_MEMKIND_KIND
326 };
327
328 data = calloc (1, sizeof (struct gomp_memkind_data));
329 if (data == NULL)
330 {
331 if (handle)
332 dlclose (handle);
333 return;
334 }
335 if (!handle)
336 {
337 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
338 return;
339 }
340 data->memkind_handle = handle;
341 data->memkind_malloc
342 = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
343 data->memkind_calloc
344 = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
345 data->memkind_realloc
346 = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
347 data->memkind_free
348 = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
349 data->memkind_check_available
350 = (__typeof (data->memkind_check_available))
351 dlsym (handle, "memkind_check_available");
352 if (data->memkind_malloc
353 && data->memkind_calloc
354 && data->memkind_realloc
355 && data->memkind_free
356 && data->memkind_check_available)
357 for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
358 {
359 data->kinds[i] = (void **) dlsym (handle, kinds[i]);
360 if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
361 data->kinds[i] = NULL;
362 }
363 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
364}
365
366static struct gomp_memkind_data *
367gomp_get_memkind (void)
368{
369 struct gomp_memkind_data *data
370 = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
371 if (data)
372 return data;
373 pthread_once (&memkind_data_once, gomp_init_memkind);
374 return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
375}
376#endif
377
e1071571
JJ		 378omp_allocator_handle_t
379omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
380 const omp_alloctrait_t traits[])
381{
382 struct omp_allocator_data data
383 = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
17f52a1c 384 omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
450b05ce 385#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 386 GOMP_MEMKIND_NONE
387#endif
388 };
e1071571
JJ		 389 struct omp_allocator_data *ret;
390 int i;
391
392 if (memspace > omp_low_lat_mem_space)
393 return omp_null_allocator;
394 for (i = 0; i < ntraits; i++)
395 switch (traits[i].key)
396 {
397 case omp_atk_sync_hint:
398 switch (traits[i].value)
399 {
400 case omp_atv_default:
401 data.sync_hint = omp_atv_contended;
402 break;
403 case omp_atv_contended:
404 case omp_atv_uncontended:
f7097793 405 case omp_atv_serialized:
e1071571
JJ		 406 case omp_atv_private:
407 data.sync_hint = traits[i].value;
408 break;
409 default:
410 return omp_null_allocator;
411 }
412 break;
413 case omp_atk_alignment:
ea82325a
JJ		 414 if (traits[i].value == omp_atv_default)
415 {
416 data.alignment = 1;
417 break;
418 }
e1071571
JJ		 419 if ((traits[i].value & (traits[i].value - 1)) != 0
420 || !traits[i].value)
421 return omp_null_allocator;
422 data.alignment = traits[i].value;
423 break;
424 case omp_atk_access:
425 switch (traits[i].value)
426 {
427 case omp_atv_default:
428 data.access = omp_atv_all;
429 break;
430 case omp_atv_all:
431 case omp_atv_cgroup:
432 case omp_atv_pteam:
433 case omp_atv_thread:
434 data.access = traits[i].value;
435 break;
436 default:
437 return omp_null_allocator;
438 }
439 break;
440 case omp_atk_pool_size:
ea82325a
JJ		 441 if (traits[i].value == omp_atv_default)
442 data.pool_size = ~(uintptr_t) 0;
443 else
444 data.pool_size = traits[i].value;
e1071571
JJ		 445 break;
446 case omp_atk_fallback:
447 switch (traits[i].value)
448 {
449 case omp_atv_default:
450 data.fallback = omp_atv_default_mem_fb;
451 break;
452 case omp_atv_default_mem_fb:
453 case omp_atv_null_fb:
454 case omp_atv_abort_fb:
455 case omp_atv_allocator_fb:
456 data.fallback = traits[i].value;
457 break;
458 default:
459 return omp_null_allocator;
460 }
461 break;
462 case omp_atk_fb_data:
463 data.fb_data = traits[i].value;
464 break;
465 case omp_atk_pinned:
466 switch (traits[i].value)
467 {
468 case omp_atv_default:
469 case omp_atv_false:
470 data.pinned = omp_atv_false;
471 break;
472 case omp_atv_true:
473 data.pinned = omp_atv_true;
474 break;
475 default:
476 return omp_null_allocator;
477 }
478 break;
479 case omp_atk_partition:
480 switch (traits[i].value)
481 {
482 case omp_atv_default:
483 data.partition = omp_atv_environment;
484 break;
485 case omp_atv_environment:
486 case omp_atv_nearest:
487 case omp_atv_blocked:
488 case omp_atv_interleaved:
489 data.partition = traits[i].value;
490 break;
491 default:
492 return omp_null_allocator;
493 }
494 break;
495 default:
496 return omp_null_allocator;
497 }
498
499 if (data.alignment < sizeof (void *))
500 data.alignment = sizeof (void *);
501
17f52a1c
JJ		 502 switch (memspace)
503 {
17f52a1c 504#ifdef LIBGOMP_USE_MEMKIND
450b05ce 505 case omp_high_bw_mem_space:
17f52a1c
JJ		 506 struct gomp_memkind_data *memkind_data;
507 memkind_data = gomp_get_memkind ();
508 if (data.partition == omp_atv_interleaved
509 && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
510 {
511 data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
512 break;
513 }
514 else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
515 {
516 data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
517 break;
518 }
8c2fc744 519 break;
17f52a1c 520 case omp_large_cap_mem_space:
17f52a1c
JJ		 521 memkind_data = gomp_get_memkind ();
522 if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
523 data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
524 else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
525 data.memkind = GOMP_MEMKIND_DAX_KMEM;
17f52a1c 526 break;
450b05ce 527#endif
17f52a1c
JJ		 528 default:
529#ifdef LIBGOMP_USE_MEMKIND
530 if (data.partition == omp_atv_interleaved)
531 {
532 memkind_data = gomp_get_memkind ();
533 if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
534 data.memkind = GOMP_MEMKIND_INTERLEAVE;
535 }
536#endif
537 break;
538 }
539
450b05ce
TB		 540#ifdef LIBGOMP_USE_LIBNUMA
541 if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
542 {
450b05ce 543 libnuma_data = gomp_get_libnuma ();
407d68da
TB		 544 if (libnuma_data->numa_alloc_local != NULL)
545 data.memkind = GOMP_MEMKIND_LIBNUMA;
450b05ce
TB		 546 }
547#endif
548
e9a19ead 549 /* Reject unsupported memory spaces. */
348874f0 550 if (!MEMSPACE_VALIDATE (data.memspace, data.access, data.pinned))
e9a19ead
AS		 551 return omp_null_allocator;
552
e1071571
JJ		 553 ret = gomp_malloc (sizeof (struct omp_allocator_data));
554 *ret = data;
555#ifndef HAVE_SYNC_BUILTINS
556 gomp_mutex_init (&ret->lock);
557#endif
558 return (omp_allocator_handle_t) ret;
559}
560
561void
562omp_destroy_allocator (omp_allocator_handle_t allocator)
563{
564 if (allocator != omp_null_allocator)
565 {
566#ifndef HAVE_SYNC_BUILTINS
567 gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
568#endif
569 free ((void *) allocator);
570 }
571}
572
fff15bad
TB		 573ialias (omp_init_allocator)
574ialias (omp_destroy_allocator)
575
b38a4bd1 576void *
6fcc3cac
JJ		 577omp_aligned_alloc (size_t alignment, size_t size,
578 omp_allocator_handle_t allocator)
e1071571
JJ		 579{
580 struct omp_allocator_data *allocator_data;
b38a4bd1 581 size_t new_size, new_alignment;
e1071571 582 void *ptr, *ret;
450b05ce
TB		 583#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
584 enum gomp_numa_memkind_kind memkind;
17f52a1c 585#endif
e1071571 586
05e4db63
JJ		 587 if (__builtin_expect (size == 0, 0))
588 return NULL;
589
e1071571 590retry:
b38a4bd1 591 new_alignment = alignment;
e1071571
JJ		 592 if (allocator == omp_null_allocator)
593 {
594 struct gomp_thread *thr = gomp_thread ();
595 if (thr->ts.def_allocator == omp_null_allocator)
596 thr->ts.def_allocator = gomp_def_allocator;
597 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
598 }
599
64001441 600 if (!predefined_allocator_p (allocator))
e1071571
JJ		 601 {
602 allocator_data = (struct omp_allocator_data *) allocator;
b38a4bd1
JJ		 603 if (new_alignment < allocator_data->alignment)
604 new_alignment = allocator_data->alignment;
450b05ce 605#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 606 memkind = allocator_data->memkind;
607#endif
e1071571
JJ		 608 }
609 else
610 {
611 allocator_data = NULL;
b38a4bd1
JJ		 612 if (new_alignment < sizeof (void *))
613 new_alignment = sizeof (void *);
450b05ce 614#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 615 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 616#endif
617#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 618 if (allocator == omp_high_bw_mem_alloc)
619 memkind = GOMP_MEMKIND_HBW_PREFERRED;
620 else if (allocator == omp_large_cap_mem_alloc)
621 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
622 if (memkind)
623 {
624 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
625 if (!memkind_data->kinds[memkind])
626 memkind = GOMP_MEMKIND_NONE;
627 }
628#endif
e1071571
JJ		 629 }
630
631 new_size = sizeof (struct omp_mem_header);
b38a4bd1
JJ		 632 if (new_alignment > sizeof (void *))
633 new_size += new_alignment - sizeof (void *);
e1071571
JJ		 634 if (__builtin_add_overflow (size, new_size, &new_size))
635 goto fail;
e9a19ead
AS		 636#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
637 if (allocator == omp_low_lat_mem_alloc)
638 goto fail;
639#endif
e1071571
JJ		 640
641 if (__builtin_expect (allocator_data
642 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
643 {
644 uintptr_t used_pool_size;
645 if (new_size > allocator_data->pool_size)
646 goto fail;
647#ifdef HAVE_SYNC_BUILTINS
648 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
649 MEMMODEL_RELAXED);
650 do
651 {
652 uintptr_t new_pool_size;
653 if (__builtin_add_overflow (used_pool_size, new_size,
654 &new_pool_size)
655 || new_pool_size > allocator_data->pool_size)
656 goto fail;
657 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
658 &used_pool_size, new_pool_size,
659 true, MEMMODEL_RELAXED,
660 MEMMODEL_RELAXED))
661 break;
662 }
663 while (1);
664#else
665 gomp_mutex_lock (&allocator_data->lock);
666 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
667 &used_pool_size)
668 || used_pool_size > allocator_data->pool_size)
669 {
670 gomp_mutex_unlock (&allocator_data->lock);
671 goto fail;
672 }
673 allocator_data->used_pool_size = used_pool_size;
674 gomp_mutex_unlock (&allocator_data->lock);
675#endif
450b05ce
TB		 676#ifdef LIBGOMP_USE_LIBNUMA
677 if (memkind == GOMP_MEMKIND_LIBNUMA)
678 ptr = libnuma_data->numa_alloc_local (new_size);
679# ifdef LIBGOMP_USE_MEMKIND
680 else
681# endif
682#endif
17f52a1c
JJ		 683#ifdef LIBGOMP_USE_MEMKIND
684 if (memkind)
685 {
686 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
687 void *kind = *memkind_data->kinds[memkind];
688 ptr = memkind_data->memkind_malloc (kind, new_size);
689 }
690 else
691#endif
348874f0
AS		 692 ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
693 allocator_data->pinned);
e1071571
JJ		 694 if (ptr == NULL)
695 {
696#ifdef HAVE_SYNC_BUILTINS
697 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
698 MEMMODEL_RELAXED);
699#else
700 gomp_mutex_lock (&allocator_data->lock);
701 allocator_data->used_pool_size -= new_size;
702 gomp_mutex_unlock (&allocator_data->lock);
703#endif
704 goto fail;
705 }
706 }
707 else
708 {
450b05ce
TB		 709#ifdef LIBGOMP_USE_LIBNUMA
710 if (memkind == GOMP_MEMKIND_LIBNUMA)
711 ptr = libnuma_data->numa_alloc_local (new_size);
712# ifdef LIBGOMP_USE_MEMKIND
713 else
714# endif
715#endif
17f52a1c
JJ		 716#ifdef LIBGOMP_USE_MEMKIND
717 if (memkind)
718 {
719 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
720 void *kind = *memkind_data->kinds[memkind];
721 ptr = memkind_data->memkind_malloc (kind, new_size);
722 }
723 else
724#endif
30486fab
AS		 725 {
726 omp_memspace_handle_t memspace;
727 memspace = (allocator_data
728 ? allocator_data->memspace
64001441
AS		 729 : predefined_alloc_mapping (allocator));
730 int pinned = (allocator_data
731 ? allocator_data->pinned
732 : allocator == ompx_gnu_pinned_mem_alloc);
733 ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
30486fab 734 }
e1071571
JJ		 735 if (ptr == NULL)
736 goto fail;
737 }
738
b38a4bd1 739 if (new_alignment > sizeof (void *))
e1071571
JJ		 740 ret = (void *) (((uintptr_t) ptr
741 + sizeof (struct omp_mem_header)
b38a4bd1
JJ		 742 + new_alignment - sizeof (void *))
743 & ~(new_alignment - 1));
e1071571
JJ		 744 else
745 ret = (char *) ptr + sizeof (struct omp_mem_header);
746 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
747 ((struct omp_mem_header *) ret)[-1].size = new_size;
748 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
749 return ret;
750
30486fab
AS		 751fail:;
752 int fallback = (allocator_data
753 ? allocator_data->fallback
64001441
AS		 754 : (allocator == omp_default_mem_alloc
755 || allocator == ompx_gnu_pinned_mem_alloc)
30486fab
AS		 756 ? omp_atv_null_fb
757 : omp_atv_default_mem_fb);
758 switch (fallback)
e1071571 759 {
30486fab
AS		 760 case omp_atv_default_mem_fb:
761 allocator = omp_default_mem_alloc;
762 goto retry;
763 case omp_atv_null_fb:
764 break;
765 default:
766 case omp_atv_abort_fb:
767 gomp_fatal ("Out of memory allocating %lu bytes",
768 (unsigned long) size);
769 case omp_atv_allocator_fb:
770 allocator = allocator_data->fb_data;
771 goto retry;
e1071571
JJ		 772 }
773 return NULL;
774}
775
b38a4bd1
JJ		 776ialias (omp_aligned_alloc)
777
6fcc3cac
JJ		 778void *
779omp_alloc (size_t size, omp_allocator_handle_t allocator)
780{
b38a4bd1 781 return ialias_call (omp_aligned_alloc) (1, size, allocator);
6fcc3cac
JJ		 782}
783
784/* Like omp_aligned_alloc, but apply on top of that:
785 "For allocations that arise from this ... the null_fb value of the
786 fallback allocator trait behaves as if the abort_fb had been specified." */
787
788void *
789GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
790{
b38a4bd1
JJ		 791 void *ret
792 = ialias_call (omp_aligned_alloc) (alignment, size,
793 (omp_allocator_handle_t) allocator);
6fcc3cac
JJ		 794 if (__builtin_expect (ret == NULL, 0) && size)
795 gomp_fatal ("Out of memory allocating %lu bytes",
796 (unsigned long) size);
797 return ret;
798}
799
e1071571
JJ		 800void
801omp_free (void *ptr, omp_allocator_handle_t allocator)
802{
803 struct omp_mem_header *data;
30486fab 804 omp_memspace_handle_t memspace = omp_default_mem_space;
348874f0 805 int pinned = false;
e1071571
JJ		 806
807 if (ptr == NULL)
808 return;
809 (void) allocator;
810 data = &((struct omp_mem_header *) ptr)[-1];
64001441 811 if (!predefined_allocator_p (data->allocator))
e1071571
JJ		 812 {
813 struct omp_allocator_data *allocator_data
814 = (struct omp_allocator_data *) (data->allocator);
815 if (allocator_data->pool_size < ~(uintptr_t) 0)
816 {
817#ifdef HAVE_SYNC_BUILTINS
818 __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
819 MEMMODEL_RELAXED);
820#else
821 gomp_mutex_lock (&allocator_data->lock);
23438370 822 allocator_data->used_pool_size -= data->size;
e1071571
JJ		 823 gomp_mutex_unlock (&allocator_data->lock);
824#endif
825 }
450b05ce
TB		 826#ifdef LIBGOMP_USE_LIBNUMA
827 if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
828 {
829 libnuma_data->numa_free (data->ptr, data->size);
830 return;
831 }
832# ifdef LIBGOMP_USE_MEMKIND
833 else
834# endif
835#endif
17f52a1c
JJ		 836#ifdef LIBGOMP_USE_MEMKIND
837 if (allocator_data->memkind)
838 {
839 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
840 void *kind = *memkind_data->kinds[allocator_data->memkind];
841 memkind_data->memkind_free (kind, data->ptr);
842 return;
843 }
844#endif
30486fab
AS		 845
846 memspace = allocator_data->memspace;
348874f0 847 pinned = allocator_data->pinned;
e1071571 848 }
17f52a1c
JJ		 849 else
850 {
30486fab 851#ifdef LIBGOMP_USE_MEMKIND
450b05ce 852 enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
17f52a1c
JJ		 853 if (data->allocator == omp_high_bw_mem_alloc)
854 memkind = GOMP_MEMKIND_HBW_PREFERRED;
855 else if (data->allocator == omp_large_cap_mem_alloc)
856 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
857 if (memkind)
858 {
859 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
860 if (memkind_data->kinds[memkind])
861 {
862 void *kind = *memkind_data->kinds[memkind];
863 memkind_data->memkind_free (kind, data->ptr);
864 return;
865 }
866 }
17f52a1c 867#endif
30486fab 868
64001441
AS		 869 memspace = predefined_alloc_mapping (data->allocator);
870 pinned = (data->allocator == ompx_gnu_pinned_mem_alloc);
30486fab
AS		 871 }
872
348874f0 873 MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
e1071571 874}
6fcc3cac
JJ		 875
876ialias (omp_free)
877
878void
879GOMP_free (void *ptr, uintptr_t allocator)
880{
b38a4bd1
JJ		 881 return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
882}
883
884void *
885omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
886 omp_allocator_handle_t allocator)
887{
888 struct omp_allocator_data *allocator_data;
889 size_t new_size, size_temp, new_alignment;
890 void *ptr, *ret;
450b05ce
TB		 891#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
892 enum gomp_numa_memkind_kind memkind;
17f52a1c 893#endif
b38a4bd1
JJ		 894
895 if (__builtin_expect (size == 0 || nmemb == 0, 0))
896 return NULL;
897
898retry:
899 new_alignment = alignment;
900 if (allocator == omp_null_allocator)
901 {
902 struct gomp_thread *thr = gomp_thread ();
903 if (thr->ts.def_allocator == omp_null_allocator)
904 thr->ts.def_allocator = gomp_def_allocator;
905 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
906 }
907
64001441 908 if (!predefined_allocator_p (allocator))
b38a4bd1
JJ		 909 {
910 allocator_data = (struct omp_allocator_data *) allocator;
911 if (new_alignment < allocator_data->alignment)
912 new_alignment = allocator_data->alignment;
450b05ce 913#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 914 memkind = allocator_data->memkind;
915#endif
b38a4bd1
JJ		 916 }
917 else
918 {
919 allocator_data = NULL;
920 if (new_alignment < sizeof (void *))
921 new_alignment = sizeof (void *);
450b05ce 922#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 923 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 924#endif
925#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 926 if (allocator == omp_high_bw_mem_alloc)
927 memkind = GOMP_MEMKIND_HBW_PREFERRED;
928 else if (allocator == omp_large_cap_mem_alloc)
929 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
930 if (memkind)
931 {
932 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
933 if (!memkind_data->kinds[memkind])
934 memkind = GOMP_MEMKIND_NONE;
935 }
936#endif
b38a4bd1
JJ		 937 }
938
939 new_size = sizeof (struct omp_mem_header);
940 if (new_alignment > sizeof (void *))
941 new_size += new_alignment - sizeof (void *);
942 if (__builtin_mul_overflow (size, nmemb, &size_temp))
943 goto fail;
944 if (__builtin_add_overflow (size_temp, new_size, &new_size))
945 goto fail;
e9a19ead
AS		 946#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
947 if (allocator == omp_low_lat_mem_alloc)
948 goto fail;
949#endif
b38a4bd1
JJ		 950
951 if (__builtin_expect (allocator_data
952 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
953 {
954 uintptr_t used_pool_size;
955 if (new_size > allocator_data->pool_size)
956 goto fail;
957#ifdef HAVE_SYNC_BUILTINS
958 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
959 MEMMODEL_RELAXED);
960 do
961 {
962 uintptr_t new_pool_size;
963 if (__builtin_add_overflow (used_pool_size, new_size,
964 &new_pool_size)
965 || new_pool_size > allocator_data->pool_size)
966 goto fail;
967 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
968 &used_pool_size, new_pool_size,
969 true, MEMMODEL_RELAXED,
970 MEMMODEL_RELAXED))
971 break;
972 }
973 while (1);
974#else
975 gomp_mutex_lock (&allocator_data->lock);
976 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
977 &used_pool_size)
978 || used_pool_size > allocator_data->pool_size)
979 {
980 gomp_mutex_unlock (&allocator_data->lock);
981 goto fail;
982 }
983 allocator_data->used_pool_size = used_pool_size;
984 gomp_mutex_unlock (&allocator_data->lock);
985#endif
450b05ce
TB		 986#ifdef LIBGOMP_USE_LIBNUMA
987 if (memkind == GOMP_MEMKIND_LIBNUMA)
988 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
989 memory that is initialized to zero. */
990 ptr = libnuma_data->numa_alloc_local (new_size);
991# ifdef LIBGOMP_USE_MEMKIND
992 else
993# endif
994#endif
17f52a1c
JJ		 995#ifdef LIBGOMP_USE_MEMKIND
996 if (memkind)
997 {
998 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
999 void *kind = *memkind_data->kinds[memkind];
1000 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
1001 }
1002 else
1003#endif
348874f0
AS		 1004 ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size,
1005 allocator_data->pinned);
b38a4bd1
JJ		 1006 if (ptr == NULL)
1007 {
1008#ifdef HAVE_SYNC_BUILTINS
1009 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
1010 MEMMODEL_RELAXED);
1011#else
1012 gomp_mutex_lock (&allocator_data->lock);
1013 allocator_data->used_pool_size -= new_size;
1014 gomp_mutex_unlock (&allocator_data->lock);
1015#endif
1016 goto fail;
1017 }
1018 }
1019 else
1020 {
450b05ce
TB		 1021#ifdef LIBGOMP_USE_LIBNUMA
1022 if (memkind == GOMP_MEMKIND_LIBNUMA)
1023 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
1024 memory that is initialized to zero. */
1025 ptr = libnuma_data->numa_alloc_local (new_size);
1026# ifdef LIBGOMP_USE_MEMKIND
1027 else
1028# endif
1029#endif
17f52a1c
JJ		 1030#ifdef LIBGOMP_USE_MEMKIND
1031 if (memkind)
1032 {
1033 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1034 void *kind = *memkind_data->kinds[memkind];
1035 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
1036 }
1037 else
1038#endif
30486fab
AS		 1039 {
1040 omp_memspace_handle_t memspace;
1041 memspace = (allocator_data
1042 ? allocator_data->memspace
64001441
AS		 1043 : predefined_alloc_mapping (allocator));
1044 int pinned = (allocator_data
1045 ? allocator_data->pinned
1046 : allocator == ompx_gnu_pinned_mem_alloc);
1047 ptr = MEMSPACE_CALLOC (memspace, new_size, pinned);
30486fab 1048 }
b38a4bd1
JJ		 1049 if (ptr == NULL)
1050 goto fail;
1051 }
1052
1053 if (new_alignment > sizeof (void *))
1054 ret = (void *) (((uintptr_t) ptr
1055 + sizeof (struct omp_mem_header)
1056 + new_alignment - sizeof (void *))
1057 & ~(new_alignment - 1));
1058 else
1059 ret = (char *) ptr + sizeof (struct omp_mem_header);
1060 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
1061 ((struct omp_mem_header *) ret)[-1].size = new_size;
1062 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1063 return ret;
1064
30486fab
AS		 1065fail:;
1066 int fallback = (allocator_data
1067 ? allocator_data->fallback
64001441
AS		 1068 : (allocator == omp_default_mem_alloc
1069 || allocator == ompx_gnu_pinned_mem_alloc)
30486fab
AS		 1070 ? omp_atv_null_fb
1071 : omp_atv_default_mem_fb);
1072 switch (fallback)
b38a4bd1 1073 {
30486fab
AS		 1074 case omp_atv_default_mem_fb:
1075 allocator = omp_default_mem_alloc;
1076 goto retry;
1077 case omp_atv_null_fb:
1078 break;
1079 default:
1080 case omp_atv_abort_fb:
1081 gomp_fatal ("Out of memory allocating %lu bytes",
1082 (unsigned long) (size * nmemb));
1083 case omp_atv_allocator_fb:
1084 allocator = allocator_data->fb_data;
1085 goto retry;
b38a4bd1
JJ		 1086 }
1087 return NULL;
1088}
1089
1090ialias (omp_aligned_calloc)
1091
1092void *
1093omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
1094{
1095 return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
1096}
1097
1098void *
1099omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
1100 omp_allocator_handle_t free_allocator)
1101{
1102 struct omp_allocator_data *allocator_data, *free_allocator_data;
1103 size_t new_size, old_size, new_alignment, old_alignment;
1104 void *new_ptr, *ret;
1105 struct omp_mem_header *data;
450b05ce
TB		 1106#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1107 enum gomp_numa_memkind_kind memkind, free_memkind;
17f52a1c 1108#endif
b38a4bd1
JJ		 1109
1110 if (__builtin_expect (ptr == NULL, 0))
1111 return ialias_call (omp_aligned_alloc) (1, size, allocator);
1112
1113 if (__builtin_expect (size == 0, 0))
1114 {
1115 ialias_call (omp_free) (ptr, free_allocator);
1116 return NULL;
1117 }
1118
1119 data = &((struct omp_mem_header *) ptr)[-1];
1120 free_allocator = data->allocator;
1121
1122retry:
1123 new_alignment = sizeof (void *);
1124 if (allocator == omp_null_allocator)
1125 allocator = free_allocator;
1126
64001441 1127 if (!predefined_allocator_p (allocator))
b38a4bd1
JJ		 1128 {
1129 allocator_data = (struct omp_allocator_data *) allocator;
1130 if (new_alignment < allocator_data->alignment)
1131 new_alignment = allocator_data->alignment;
450b05ce 1132#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1133 memkind = allocator_data->memkind;
1134#endif
b38a4bd1
JJ		 1135 }
1136 else
17f52a1c
JJ		 1137 {
1138 allocator_data = NULL;
450b05ce 1139#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 1140 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 1141#endif
1142#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 1143 if (allocator == omp_high_bw_mem_alloc)
1144 memkind = GOMP_MEMKIND_HBW_PREFERRED;
1145 else if (allocator == omp_large_cap_mem_alloc)
1146 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1147 if (memkind)
1148 {
1149 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1150 if (!memkind_data->kinds[memkind])
1151 memkind = GOMP_MEMKIND_NONE;
1152 }
1153#endif
1154 }
64001441 1155 if (!predefined_allocator_p (free_allocator))
17f52a1c
JJ		 1156 {
1157 free_allocator_data = (struct omp_allocator_data *) free_allocator;
450b05ce 1158#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1159 free_memkind = free_allocator_data->memkind;
1160#endif
1161 }
b38a4bd1 1162 else
17f52a1c
JJ		 1163 {
1164 free_allocator_data = NULL;
450b05ce 1165#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 1166 free_memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 1167#endif
1168#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 1169 if (free_allocator == omp_high_bw_mem_alloc)
1170 free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1171 else if (free_allocator == omp_large_cap_mem_alloc)
1172 free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1173 if (free_memkind)
1174 {
1175 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1176 if (!memkind_data->kinds[free_memkind])
1177 free_memkind = GOMP_MEMKIND_NONE;
1178 }
1179#endif
1180 }
b38a4bd1
JJ		 1181 old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1182
1183 new_size = sizeof (struct omp_mem_header);
1184 if (new_alignment > sizeof (void *))
1185 new_size += new_alignment - sizeof (void *);
1186 if (__builtin_add_overflow (size, new_size, &new_size))
1187 goto fail;
1188 old_size = data->size;
e9a19ead
AS		 1189#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1190 if (allocator == omp_low_lat_mem_alloc)
1191 goto fail;
1192#endif
b38a4bd1
JJ		 1193
1194 if (__builtin_expect (allocator_data
1195 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1196 {
1197 uintptr_t used_pool_size;
1198 size_t prev_size = 0;
1199 /* Check if we can use realloc. Don't use it if extra alignment
1200 was used previously or newly, because realloc might return a pointer
1201 with different alignment and then we'd need to memmove the data
1202 again. */
1203 if (free_allocator_data
1204 && free_allocator_data == allocator_data
1205 && new_alignment == sizeof (void *)
1206 && old_alignment == sizeof (struct omp_mem_header))
1207 prev_size = old_size;
1208 if (new_size > prev_size
1209 && new_size - prev_size > allocator_data->pool_size)
1210 goto fail;
1211#ifdef HAVE_SYNC_BUILTINS
1212 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1213 MEMMODEL_RELAXED);
1214 do
1215 {
1216 uintptr_t new_pool_size;
1217 if (new_size > prev_size)
1218 {
1219 if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1220 &new_pool_size)
1221 || new_pool_size > allocator_data->pool_size)
1222 goto fail;
1223 }
1224 else
1225 new_pool_size = used_pool_size + new_size - prev_size;
1226 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1227 &used_pool_size, new_pool_size,
1228 true, MEMMODEL_RELAXED,
1229 MEMMODEL_RELAXED))
1230 break;
1231 }
1232 while (1);
1233#else
1234 gomp_mutex_lock (&allocator_data->lock);
1235 if (new_size > prev_size)
1236 {
1237 if (__builtin_add_overflow (allocator_data->used_pool_size,
1238 new_size - prev_size,
1239 &used_pool_size)
1240 || used_pool_size > allocator_data->pool_size)
1241 {
1242 gomp_mutex_unlock (&allocator_data->lock);
1243 goto fail;
1244 }
1245 }
1246 else
1247 used_pool_size = (allocator_data->used_pool_size
1248 + new_size - prev_size);
1249 allocator_data->used_pool_size = used_pool_size;
1250 gomp_mutex_unlock (&allocator_data->lock);
17f52a1c 1251#endif
450b05ce
TB		 1252#ifdef LIBGOMP_USE_LIBNUMA
1253 if (memkind == GOMP_MEMKIND_LIBNUMA)
1254 {
1255 if (prev_size)
1256 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1257 new_size);
1258 else
1259 new_ptr = libnuma_data->numa_alloc_local (new_size);
1260 }
1261# ifdef LIBGOMP_USE_MEMKIND
1262 else
1263# endif
1264#endif
17f52a1c
JJ		 1265#ifdef LIBGOMP_USE_MEMKIND
1266 if (memkind)
1267 {
1268 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1269 void *kind = *memkind_data->kinds[memkind];
1270 if (prev_size)
1271 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1272 new_size);
1273 else
1274 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1275 }
1276 else
b38a4bd1
JJ		 1277#endif
1278 if (prev_size)
64001441
AS		 1279 {
1280 int was_pinned = (free_allocator_data
1281 ? free_allocator_data->pinned
1282 : free_allocator == ompx_gnu_pinned_mem_alloc);
1283 new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
1284 data->size, new_size, was_pinned,
1285 allocator_data->pinned);
1286 }
b38a4bd1 1287 else
348874f0
AS		 1288 new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
1289 allocator_data->pinned);
b38a4bd1
JJ		 1290 if (new_ptr == NULL)
1291 {
1292#ifdef HAVE_SYNC_BUILTINS
1293 __atomic_add_fetch (&allocator_data->used_pool_size,
1294 prev_size - new_size,
1295 MEMMODEL_RELAXED);
1296#else
1297 gomp_mutex_lock (&allocator_data->lock);
1298 allocator_data->used_pool_size -= new_size - prev_size;
1299 gomp_mutex_unlock (&allocator_data->lock);
1300#endif
1301 goto fail;
1302 }
1303 else if (prev_size)
1304 {
1305 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1306 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1307 ((struct omp_mem_header *) ret)[-1].size = new_size;
1308 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1309 return ret;
1310 }
1311 }
1312 else if (new_alignment == sizeof (void *)
1313 && old_alignment == sizeof (struct omp_mem_header)
450b05ce 1314#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1315 && memkind == free_memkind
1316#endif
b38a4bd1
JJ		 1317 && (free_allocator_data == NULL
1318 || free_allocator_data->pool_size == ~(uintptr_t) 0))
1319 {
450b05ce
TB		 1320#ifdef LIBGOMP_USE_LIBNUMA
1321 if (memkind == GOMP_MEMKIND_LIBNUMA)
1322 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1323# ifdef LIBGOMP_USE_MEMKIND
1324 else
1325# endif
1326#endif
17f52a1c
JJ		 1327#ifdef LIBGOMP_USE_MEMKIND
1328 if (memkind)
1329 {
1330 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1331 void *kind = *memkind_data->kinds[memkind];
1332 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1333 new_size);
1334 }
1335 else
1336#endif
30486fab
AS		 1337 {
1338 omp_memspace_handle_t memspace;
1339 memspace = (allocator_data
1340 ? allocator_data->memspace
64001441
AS		 1341 : predefined_alloc_mapping (allocator));
1342 int was_pinned = (free_allocator_data
1343 ? free_allocator_data->pinned
1344 : free_allocator == ompx_gnu_pinned_mem_alloc);
1345 int pinned = (allocator_data
1346 ? allocator_data->pinned
1347 : allocator == ompx_gnu_pinned_mem_alloc);
348874f0 1348 new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
64001441 1349 was_pinned, pinned);
30486fab 1350 }
b38a4bd1
JJ		 1351 if (new_ptr == NULL)
1352 goto fail;
348874f0 1353
b38a4bd1
JJ		 1354 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1355 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1356 ((struct omp_mem_header *) ret)[-1].size = new_size;
1357 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1358 return ret;
1359 }
1360 else
1361 {
450b05ce
TB		 1362#ifdef LIBGOMP_USE_LIBNUMA
1363 if (memkind == GOMP_MEMKIND_LIBNUMA)
1364 new_ptr = libnuma_data->numa_alloc_local (new_size);
1365# ifdef LIBGOMP_USE_MEMKIND
1366 else
1367# endif
1368#endif
17f52a1c
JJ		 1369#ifdef LIBGOMP_USE_MEMKIND
1370 if (memkind)
1371 {
1372 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1373 void *kind = *memkind_data->kinds[memkind];
1374 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1375 }
1376 else
1377#endif
30486fab
AS		 1378 {
1379 omp_memspace_handle_t memspace;
1380 memspace = (allocator_data
1381 ? allocator_data->memspace
64001441
AS		 1382 : predefined_alloc_mapping (allocator));
1383 int pinned = (allocator_data
1384 ? allocator_data->pinned
1385 : allocator == ompx_gnu_pinned_mem_alloc);
1386 new_ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
30486fab 1387 }
b38a4bd1
JJ		 1388 if (new_ptr == NULL)
1389 goto fail;
1390 }
1391
1392 if (new_alignment > sizeof (void *))
1393 ret = (void *) (((uintptr_t) new_ptr
1394 + sizeof (struct omp_mem_header)
1395 + new_alignment - sizeof (void *))
1396 & ~(new_alignment - 1));
1397 else
1398 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1399 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1400 ((struct omp_mem_header *) ret)[-1].size = new_size;
1401 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1402 if (old_size - old_alignment < size)
1403 size = old_size - old_alignment;
1404 memcpy (ret, ptr, size);
1405 if (__builtin_expect (free_allocator_data
1406 && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1407 {
1408#ifdef HAVE_SYNC_BUILTINS
1409 __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1410 MEMMODEL_RELAXED);
1411#else
1412 gomp_mutex_lock (&free_allocator_data->lock);
1413 free_allocator_data->used_pool_size -= data->size;
1414 gomp_mutex_unlock (&free_allocator_data->lock);
1415#endif
1416 }
450b05ce
TB		 1417#ifdef LIBGOMP_USE_LIBNUMA
1418 if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1419 {
1420 libnuma_data->numa_free (data->ptr, data->size);
1421 return ret;
1422 }
1423# ifdef LIBGOMP_USE_MEMKIND
1424 else
1425# endif
1426#endif
17f52a1c
JJ		 1427#ifdef LIBGOMP_USE_MEMKIND
1428 if (free_memkind)
1429 {
1430 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1431 void *kind = *memkind_data->kinds[free_memkind];
1432 memkind_data->memkind_free (kind, data->ptr);
1433 return ret;
1434 }
1435#endif
30486fab
AS		 1436 {
1437 omp_memspace_handle_t was_memspace;
1438 was_memspace = (free_allocator_data
1439 ? free_allocator_data->memspace
64001441
AS		 1440 : predefined_alloc_mapping (free_allocator));
1441 int was_pinned = (free_allocator_data
1442 ? free_allocator_data->pinned
1443 : free_allocator == ompx_gnu_pinned_mem_alloc);
348874f0 1444 MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
30486fab 1445 }
b38a4bd1
JJ		 1446 return ret;
1447
30486fab
AS		 1448fail:;
1449 int fallback = (allocator_data
1450 ? allocator_data->fallback
64001441
AS		 1451 : (allocator == omp_default_mem_alloc
1452 || allocator == ompx_gnu_pinned_mem_alloc)
30486fab
AS		 1453 ? omp_atv_null_fb
1454 : omp_atv_default_mem_fb);
1455 switch (fallback)
b38a4bd1 1456 {
30486fab
AS		 1457 case omp_atv_default_mem_fb:
1458 allocator = omp_default_mem_alloc;
1459 goto retry;
1460 case omp_atv_null_fb:
1461 break;
1462 default:
1463 case omp_atv_abort_fb:
1464 gomp_fatal ("Out of memory allocating %lu bytes",
1465 (unsigned long) size);
1466 case omp_atv_allocator_fb:
1467 allocator = allocator_data->fb_data;
1468 goto retry;
b38a4bd1
JJ		 1469 }
1470 return NULL;
6fcc3cac 1471}
Mirror of https://gcc.gnu.org/git/gcc.git