]> git.ipfire.org Git - thirdparty/gcc.git/blame - libgomp/allocator.c
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
openmp: Add OpenMP _BitInt support [PR113409]
[thirdparty/gcc.git] / libgomp / allocator.c
CommitLineData
a945c346 1/* Copyright (C) 2020-2024 Free Software Foundation, Inc.
e1071571
JJ		 2 Contributed by Jakub Jelinek <jakub@redhat.com>.
3
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
6
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26/* This file contains wrappers for the system allocation routines. Most
27 places in the OpenMP API do not make any provision for failure, so in
28 general we cannot allow memory allocation to fail. */
29
30#define _GNU_SOURCE
31#include "libgomp.h"
32#include <stdlib.h>
b38a4bd1 33#include <string.h>
450b05ce 34#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 35#include <dlfcn.h>
36#endif
e1071571 37
d4b6d147
TB		 38/* Keeping track whether a Fortran scalar allocatable/pointer has been
39 allocated via 'omp allocators'/'omp allocate'. */
40
41struct fort_alloc_splay_tree_key_s {
42 void *ptr;
43};
44
45typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node;
46typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree;
47typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key;
48
49static inline int
50fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y)
51{
52 if (x->ptr < y->ptr)
53 return -1;
54 if (x->ptr > y->ptr)
55 return 1;
56 return 0;
57}
58#define splay_tree_prefix fort_alloc
59#define splay_tree_static
60#include "splay-tree.h"
61
62#define splay_tree_prefix fort_alloc
63#define splay_tree_static
64#define splay_tree_c
65#include "splay-tree.h"
66
67static struct fort_alloc_splay_tree_s fort_alloc_scalars;
68
69/* Add pointer as being alloced by GOMP_alloc. */
70void
71GOMP_add_alloc (void *ptr)
72{
73 if (ptr == NULL)
74 return;
75 fort_alloc_splay_tree_node item;
76 item = gomp_malloc (sizeof (struct splay_tree_node_s));
77 item->key.ptr = ptr;
78 item->left = NULL;
79 item->right = NULL;
80 fort_alloc_splay_tree_insert (&fort_alloc_scalars, item);
81}
82
83/* Remove pointer, either called by FREE or by REALLOC,
84 either of them can change the allocation status. */
85bool
86GOMP_is_alloc (void *ptr)
87{
88 struct fort_alloc_splay_tree_key_s needle;
89 fort_alloc_splay_tree_node n;
90 needle.ptr = ptr;
91 n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle);
92 if (n)
93 {
94 fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key);
95 free (n);
96 }
97 return n != NULL;
98}
99
100
e1071571
JJ		 101#define omp_max_predefined_alloc omp_thread_mem_alloc
102
30486fab 103/* These macros may be overridden in config/<target>/allocator.c.
348874f0
AS		 104 The defaults (no override) are to return NULL for pinned memory requests
105 and pass through to the regular OS calls otherwise.
30486fab
AS		 106 The following definitions (ab)use comma operators to avoid unused
107 variable errors. */
108#ifndef MEMSPACE_ALLOC
348874f0
AS		 109#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
110 (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
30486fab
AS		 111#endif
112#ifndef MEMSPACE_CALLOC
348874f0
AS		 113#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
114 (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
30486fab
AS		 115#endif
116#ifndef MEMSPACE_REALLOC
348874f0
AS		 117#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
118 ((PIN) || (OLDPIN) ? NULL \
119 : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
30486fab
AS		 120#endif
121#ifndef MEMSPACE_FREE
348874f0
AS		 122#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
123 if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
30486fab 124#endif
e9a19ead 125#ifndef MEMSPACE_VALIDATE
348874f0
AS		 126#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
127 (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1))
e9a19ead 128#endif
30486fab
AS		 129
130/* Map the predefined allocators to the correct memory space.
131 The index to this table is the omp_allocator_handle_t enum value.
132 When the user calls omp_alloc with a predefined allocator this
133 table determines what memory they get. */
134static const omp_memspace_handle_t predefined_alloc_mapping[] = {
135 omp_default_mem_space, /* omp_null_allocator doesn't actually use this. */
136 omp_default_mem_space, /* omp_default_mem_alloc. */
137 omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
138 omp_const_mem_space, /* omp_const_mem_alloc. */
139 omp_high_bw_mem_space, /* omp_high_bw_mem_alloc. */
140 omp_low_lat_mem_space, /* omp_low_lat_mem_alloc. */
141 omp_low_lat_mem_space, /* omp_cgroup_mem_alloc (implementation defined). */
142 omp_low_lat_mem_space, /* omp_pteam_mem_alloc (implementation defined). */
143 omp_low_lat_mem_space, /* omp_thread_mem_alloc (implementation defined). */
144};
145
146#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
147_Static_assert (ARRAY_SIZE (predefined_alloc_mapping)
148 == omp_max_predefined_alloc + 1,
149 "predefined_alloc_mapping must match omp_memspace_handle_t");
150
450b05ce 151enum gomp_numa_memkind_kind
17f52a1c
JJ		 152{
153 GOMP_MEMKIND_NONE = 0,
154#define GOMP_MEMKIND_KINDS \
155 GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \
156 GOMP_MEMKIND_KIND (HBW_PREFERRED), \
157 GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \
158 GOMP_MEMKIND_KIND (DAX_KMEM), \
159 GOMP_MEMKIND_KIND (INTERLEAVE), \
160 GOMP_MEMKIND_KIND (DEFAULT)
161#define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
162 GOMP_MEMKIND_KINDS,
163#undef GOMP_MEMKIND_KIND
450b05ce
TB		 164 GOMP_MEMKIND_COUNT,
165 GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
17f52a1c
JJ		 166};
167
e1071571
JJ		 168struct omp_allocator_data
169{
170 omp_memspace_handle_t memspace;
171 omp_uintptr_t alignment;
172 omp_uintptr_t pool_size;
173 omp_uintptr_t used_pool_size;
174 omp_allocator_handle_t fb_data;
175 unsigned int sync_hint : 8;
176 unsigned int access : 8;
177 unsigned int fallback : 8;
178 unsigned int pinned : 1;
179 unsigned int partition : 7;
450b05ce 180#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 181 unsigned int memkind : 8;
182#endif
e1071571
JJ		 183#ifndef HAVE_SYNC_BUILTINS
184 gomp_mutex_t lock;
185#endif
186};
187
188struct omp_mem_header
189{
190 void *ptr;
191 size_t size;
192 omp_allocator_handle_t allocator;
193 void *pad;
194};
195
450b05ce
TB		 196struct gomp_libnuma_data
197{
198 void *numa_handle;
199 void *(*numa_alloc_local) (size_t);
200 void *(*numa_realloc) (void *, size_t, size_t);
201 void (*numa_free) (void *, size_t);
202};
203
17f52a1c
JJ		 204struct gomp_memkind_data
205{
206 void *memkind_handle;
207 void *(*memkind_malloc) (void *, size_t);
208 void *(*memkind_calloc) (void *, size_t, size_t);
209 void *(*memkind_realloc) (void *, void *, size_t);
210 void (*memkind_free) (void *, void *);
211 int (*memkind_check_available) (void *);
212 void **kinds[GOMP_MEMKIND_COUNT];
213};
214
450b05ce
TB		 215#ifdef LIBGOMP_USE_LIBNUMA
216static struct gomp_libnuma_data *libnuma_data;
217static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
218
219static void
220gomp_init_libnuma (void)
221{
222 void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
223 struct gomp_libnuma_data *data;
224
225 data = calloc (1, sizeof (struct gomp_libnuma_data));
226 if (data == NULL)
227 {
228 if (handle)
229 dlclose (handle);
230 return;
231 }
8f3c4517
TB		 232 if (handle)
233 {
234 int (*numa_available) (void);
235 numa_available
236 = (__typeof (numa_available)) dlsym (handle, "numa_available");
237 if (!numa_available || numa_available () != 0)
238 {
239 dlclose (handle);
240 handle = NULL;
241 }
242 }
450b05ce
TB		 243 if (!handle)
244 {
245 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
246 return;
247 }
248 data->numa_handle = handle;
249 data->numa_alloc_local
250 = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
251 data->numa_realloc
252 = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
253 data->numa_free
254 = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
255 __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
256}
257
258static struct gomp_libnuma_data *
259gomp_get_libnuma (void)
260{
261 struct gomp_libnuma_data *data
262 = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
263 if (data)
264 return data;
265 pthread_once (&libnuma_data_once, gomp_init_libnuma);
266 return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
267}
268#endif
269
17f52a1c
JJ		 270#ifdef LIBGOMP_USE_MEMKIND
271static struct gomp_memkind_data *memkind_data;
272static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
273
274static void
275gomp_init_memkind (void)
276{
1eff4872 277 void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
17f52a1c
JJ		 278 struct gomp_memkind_data *data;
279 int i;
280 static const char *kinds[] = {
281 NULL,
282#define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
283 GOMP_MEMKIND_KINDS
284#undef GOMP_MEMKIND_KIND
285 };
286
287 data = calloc (1, sizeof (struct gomp_memkind_data));
288 if (data == NULL)
289 {
290 if (handle)
291 dlclose (handle);
292 return;
293 }
294 if (!handle)
295 {
296 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
297 return;
298 }
299 data->memkind_handle = handle;
300 data->memkind_malloc
301 = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
302 data->memkind_calloc
303 = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
304 data->memkind_realloc
305 = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
306 data->memkind_free
307 = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
308 data->memkind_check_available
309 = (__typeof (data->memkind_check_available))
310 dlsym (handle, "memkind_check_available");
311 if (data->memkind_malloc
312 && data->memkind_calloc
313 && data->memkind_realloc
314 && data->memkind_free
315 && data->memkind_check_available)
316 for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
317 {
318 data->kinds[i] = (void **) dlsym (handle, kinds[i]);
319 if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
320 data->kinds[i] = NULL;
321 }
322 __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
323}
324
325static struct gomp_memkind_data *
326gomp_get_memkind (void)
327{
328 struct gomp_memkind_data *data
329 = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
330 if (data)
331 return data;
332 pthread_once (&memkind_data_once, gomp_init_memkind);
333 return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
334}
335#endif
336
e1071571
JJ		 337omp_allocator_handle_t
338omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
339 const omp_alloctrait_t traits[])
340{
341 struct omp_allocator_data data
342 = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
17f52a1c 343 omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
450b05ce 344#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 345 GOMP_MEMKIND_NONE
346#endif
347 };
e1071571
JJ		 348 struct omp_allocator_data *ret;
349 int i;
350
351 if (memspace > omp_low_lat_mem_space)
352 return omp_null_allocator;
353 for (i = 0; i < ntraits; i++)
354 switch (traits[i].key)
355 {
356 case omp_atk_sync_hint:
357 switch (traits[i].value)
358 {
359 case omp_atv_default:
360 data.sync_hint = omp_atv_contended;
361 break;
362 case omp_atv_contended:
363 case omp_atv_uncontended:
f7097793 364 case omp_atv_serialized:
e1071571
JJ		 365 case omp_atv_private:
366 data.sync_hint = traits[i].value;
367 break;
368 default:
369 return omp_null_allocator;
370 }
371 break;
372 case omp_atk_alignment:
ea82325a
JJ		 373 if (traits[i].value == omp_atv_default)
374 {
375 data.alignment = 1;
376 break;
377 }
e1071571
JJ		 378 if ((traits[i].value & (traits[i].value - 1)) != 0
379 || !traits[i].value)
380 return omp_null_allocator;
381 data.alignment = traits[i].value;
382 break;
383 case omp_atk_access:
384 switch (traits[i].value)
385 {
386 case omp_atv_default:
387 data.access = omp_atv_all;
388 break;
389 case omp_atv_all:
390 case omp_atv_cgroup:
391 case omp_atv_pteam:
392 case omp_atv_thread:
393 data.access = traits[i].value;
394 break;
395 default:
396 return omp_null_allocator;
397 }
398 break;
399 case omp_atk_pool_size:
ea82325a
JJ		 400 if (traits[i].value == omp_atv_default)
401 data.pool_size = ~(uintptr_t) 0;
402 else
403 data.pool_size = traits[i].value;
e1071571
JJ		 404 break;
405 case omp_atk_fallback:
406 switch (traits[i].value)
407 {
408 case omp_atv_default:
409 data.fallback = omp_atv_default_mem_fb;
410 break;
411 case omp_atv_default_mem_fb:
412 case omp_atv_null_fb:
413 case omp_atv_abort_fb:
414 case omp_atv_allocator_fb:
415 data.fallback = traits[i].value;
416 break;
417 default:
418 return omp_null_allocator;
419 }
420 break;
421 case omp_atk_fb_data:
422 data.fb_data = traits[i].value;
423 break;
424 case omp_atk_pinned:
425 switch (traits[i].value)
426 {
427 case omp_atv_default:
428 case omp_atv_false:
429 data.pinned = omp_atv_false;
430 break;
431 case omp_atv_true:
432 data.pinned = omp_atv_true;
433 break;
434 default:
435 return omp_null_allocator;
436 }
437 break;
438 case omp_atk_partition:
439 switch (traits[i].value)
440 {
441 case omp_atv_default:
442 data.partition = omp_atv_environment;
443 break;
444 case omp_atv_environment:
445 case omp_atv_nearest:
446 case omp_atv_blocked:
447 case omp_atv_interleaved:
448 data.partition = traits[i].value;
449 break;
450 default:
451 return omp_null_allocator;
452 }
453 break;
454 default:
455 return omp_null_allocator;
456 }
457
458 if (data.alignment < sizeof (void *))
459 data.alignment = sizeof (void *);
460
17f52a1c
JJ		 461 switch (memspace)
462 {
17f52a1c 463#ifdef LIBGOMP_USE_MEMKIND
450b05ce 464 case omp_high_bw_mem_space:
17f52a1c
JJ		 465 struct gomp_memkind_data *memkind_data;
466 memkind_data = gomp_get_memkind ();
467 if (data.partition == omp_atv_interleaved
468 && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
469 {
470 data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
471 break;
472 }
473 else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
474 {
475 data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
476 break;
477 }
8c2fc744 478 break;
17f52a1c 479 case omp_large_cap_mem_space:
17f52a1c
JJ		 480 memkind_data = gomp_get_memkind ();
481 if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
482 data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
483 else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
484 data.memkind = GOMP_MEMKIND_DAX_KMEM;
17f52a1c 485 break;
450b05ce 486#endif
17f52a1c
JJ		 487 default:
488#ifdef LIBGOMP_USE_MEMKIND
489 if (data.partition == omp_atv_interleaved)
490 {
491 memkind_data = gomp_get_memkind ();
492 if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
493 data.memkind = GOMP_MEMKIND_INTERLEAVE;
494 }
495#endif
496 break;
497 }
498
450b05ce
TB		 499#ifdef LIBGOMP_USE_LIBNUMA
500 if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
501 {
450b05ce 502 libnuma_data = gomp_get_libnuma ();
407d68da
TB		 503 if (libnuma_data->numa_alloc_local != NULL)
504 data.memkind = GOMP_MEMKIND_LIBNUMA;
450b05ce
TB		 505 }
506#endif
507
e9a19ead 508 /* Reject unsupported memory spaces. */
348874f0 509 if (!MEMSPACE_VALIDATE (data.memspace, data.access, data.pinned))
e9a19ead
AS		 510 return omp_null_allocator;
511
e1071571
JJ		 512 ret = gomp_malloc (sizeof (struct omp_allocator_data));
513 *ret = data;
514#ifndef HAVE_SYNC_BUILTINS
515 gomp_mutex_init (&ret->lock);
516#endif
517 return (omp_allocator_handle_t) ret;
518}
519
520void
521omp_destroy_allocator (omp_allocator_handle_t allocator)
522{
523 if (allocator != omp_null_allocator)
524 {
525#ifndef HAVE_SYNC_BUILTINS
526 gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
527#endif
528 free ((void *) allocator);
529 }
530}
531
fff15bad
TB		 532ialias (omp_init_allocator)
533ialias (omp_destroy_allocator)
534
b38a4bd1 535void *
6fcc3cac
JJ		 536omp_aligned_alloc (size_t alignment, size_t size,
537 omp_allocator_handle_t allocator)
e1071571
JJ		 538{
539 struct omp_allocator_data *allocator_data;
b38a4bd1 540 size_t new_size, new_alignment;
e1071571 541 void *ptr, *ret;
450b05ce
TB		 542#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
543 enum gomp_numa_memkind_kind memkind;
17f52a1c 544#endif
e1071571 545
05e4db63
JJ		 546 if (__builtin_expect (size == 0, 0))
547 return NULL;
548
e1071571 549retry:
b38a4bd1 550 new_alignment = alignment;
e1071571
JJ		 551 if (allocator == omp_null_allocator)
552 {
553 struct gomp_thread *thr = gomp_thread ();
554 if (thr->ts.def_allocator == omp_null_allocator)
555 thr->ts.def_allocator = gomp_def_allocator;
556 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
557 }
558
559 if (allocator > omp_max_predefined_alloc)
560 {
561 allocator_data = (struct omp_allocator_data *) allocator;
b38a4bd1
JJ		 562 if (new_alignment < allocator_data->alignment)
563 new_alignment = allocator_data->alignment;
450b05ce 564#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 565 memkind = allocator_data->memkind;
566#endif
e1071571
JJ		 567 }
568 else
569 {
570 allocator_data = NULL;
b38a4bd1
JJ		 571 if (new_alignment < sizeof (void *))
572 new_alignment = sizeof (void *);
450b05ce 573#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 574 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 575#endif
576#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 577 if (allocator == omp_high_bw_mem_alloc)
578 memkind = GOMP_MEMKIND_HBW_PREFERRED;
579 else if (allocator == omp_large_cap_mem_alloc)
580 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
581 if (memkind)
582 {
583 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
584 if (!memkind_data->kinds[memkind])
585 memkind = GOMP_MEMKIND_NONE;
586 }
587#endif
e1071571
JJ		 588 }
589
590 new_size = sizeof (struct omp_mem_header);
b38a4bd1
JJ		 591 if (new_alignment > sizeof (void *))
592 new_size += new_alignment - sizeof (void *);
e1071571
JJ		 593 if (__builtin_add_overflow (size, new_size, &new_size))
594 goto fail;
e9a19ead
AS		 595#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
596 if (allocator == omp_low_lat_mem_alloc)
597 goto fail;
598#endif
e1071571
JJ		 599
600 if (__builtin_expect (allocator_data
601 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
602 {
603 uintptr_t used_pool_size;
604 if (new_size > allocator_data->pool_size)
605 goto fail;
606#ifdef HAVE_SYNC_BUILTINS
607 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
608 MEMMODEL_RELAXED);
609 do
610 {
611 uintptr_t new_pool_size;
612 if (__builtin_add_overflow (used_pool_size, new_size,
613 &new_pool_size)
614 || new_pool_size > allocator_data->pool_size)
615 goto fail;
616 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
617 &used_pool_size, new_pool_size,
618 true, MEMMODEL_RELAXED,
619 MEMMODEL_RELAXED))
620 break;
621 }
622 while (1);
623#else
624 gomp_mutex_lock (&allocator_data->lock);
625 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
626 &used_pool_size)
627 || used_pool_size > allocator_data->pool_size)
628 {
629 gomp_mutex_unlock (&allocator_data->lock);
630 goto fail;
631 }
632 allocator_data->used_pool_size = used_pool_size;
633 gomp_mutex_unlock (&allocator_data->lock);
634#endif
450b05ce
TB		 635#ifdef LIBGOMP_USE_LIBNUMA
636 if (memkind == GOMP_MEMKIND_LIBNUMA)
637 ptr = libnuma_data->numa_alloc_local (new_size);
638# ifdef LIBGOMP_USE_MEMKIND
639 else
640# endif
641#endif
17f52a1c
JJ		 642#ifdef LIBGOMP_USE_MEMKIND
643 if (memkind)
644 {
645 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
646 void *kind = *memkind_data->kinds[memkind];
647 ptr = memkind_data->memkind_malloc (kind, new_size);
648 }
649 else
650#endif
348874f0
AS		 651 ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
652 allocator_data->pinned);
e1071571
JJ		 653 if (ptr == NULL)
654 {
655#ifdef HAVE_SYNC_BUILTINS
656 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
657 MEMMODEL_RELAXED);
658#else
659 gomp_mutex_lock (&allocator_data->lock);
660 allocator_data->used_pool_size -= new_size;
661 gomp_mutex_unlock (&allocator_data->lock);
662#endif
663 goto fail;
664 }
665 }
666 else
667 {
450b05ce
TB		 668#ifdef LIBGOMP_USE_LIBNUMA
669 if (memkind == GOMP_MEMKIND_LIBNUMA)
670 ptr = libnuma_data->numa_alloc_local (new_size);
671# ifdef LIBGOMP_USE_MEMKIND
672 else
673# endif
674#endif
17f52a1c
JJ		 675#ifdef LIBGOMP_USE_MEMKIND
676 if (memkind)
677 {
678 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
679 void *kind = *memkind_data->kinds[memkind];
680 ptr = memkind_data->memkind_malloc (kind, new_size);
681 }
682 else
683#endif
30486fab
AS		 684 {
685 omp_memspace_handle_t memspace;
686 memspace = (allocator_data
687 ? allocator_data->memspace
688 : predefined_alloc_mapping[allocator]);
348874f0
AS		 689 ptr = MEMSPACE_ALLOC (memspace, new_size,
690 allocator_data && allocator_data->pinned);
30486fab 691 }
e1071571
JJ		 692 if (ptr == NULL)
693 goto fail;
694 }
695
b38a4bd1 696 if (new_alignment > sizeof (void *))
e1071571
JJ		 697 ret = (void *) (((uintptr_t) ptr
698 + sizeof (struct omp_mem_header)
b38a4bd1
JJ		 699 + new_alignment - sizeof (void *))
700 & ~(new_alignment - 1));
e1071571
JJ		 701 else
702 ret = (char *) ptr + sizeof (struct omp_mem_header);
703 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
704 ((struct omp_mem_header *) ret)[-1].size = new_size;
705 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
706 return ret;
707
30486fab
AS		 708fail:;
709 int fallback = (allocator_data
710 ? allocator_data->fallback
711 : allocator == omp_default_mem_alloc
712 ? omp_atv_null_fb
713 : omp_atv_default_mem_fb);
714 switch (fallback)
e1071571 715 {
30486fab
AS		 716 case omp_atv_default_mem_fb:
717 allocator = omp_default_mem_alloc;
718 goto retry;
719 case omp_atv_null_fb:
720 break;
721 default:
722 case omp_atv_abort_fb:
723 gomp_fatal ("Out of memory allocating %lu bytes",
724 (unsigned long) size);
725 case omp_atv_allocator_fb:
726 allocator = allocator_data->fb_data;
727 goto retry;
e1071571
JJ		 728 }
729 return NULL;
730}
731
b38a4bd1
JJ		 732ialias (omp_aligned_alloc)
733
6fcc3cac
JJ		 734void *
735omp_alloc (size_t size, omp_allocator_handle_t allocator)
736{
b38a4bd1 737 return ialias_call (omp_aligned_alloc) (1, size, allocator);
6fcc3cac
JJ		 738}
739
740/* Like omp_aligned_alloc, but apply on top of that:
741 "For allocations that arise from this ... the null_fb value of the
742 fallback allocator trait behaves as if the abort_fb had been specified." */
743
744void *
745GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
746{
b38a4bd1
JJ		 747 void *ret
748 = ialias_call (omp_aligned_alloc) (alignment, size,
749 (omp_allocator_handle_t) allocator);
6fcc3cac
JJ		 750 if (__builtin_expect (ret == NULL, 0) && size)
751 gomp_fatal ("Out of memory allocating %lu bytes",
752 (unsigned long) size);
753 return ret;
754}
755
e1071571
JJ		 756void
757omp_free (void *ptr, omp_allocator_handle_t allocator)
758{
759 struct omp_mem_header *data;
30486fab 760 omp_memspace_handle_t memspace = omp_default_mem_space;
348874f0 761 int pinned = false;
e1071571
JJ		 762
763 if (ptr == NULL)
764 return;
765 (void) allocator;
766 data = &((struct omp_mem_header *) ptr)[-1];
767 if (data->allocator > omp_max_predefined_alloc)
768 {
769 struct omp_allocator_data *allocator_data
770 = (struct omp_allocator_data *) (data->allocator);
771 if (allocator_data->pool_size < ~(uintptr_t) 0)
772 {
773#ifdef HAVE_SYNC_BUILTINS
774 __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
775 MEMMODEL_RELAXED);
776#else
777 gomp_mutex_lock (&allocator_data->lock);
23438370 778 allocator_data->used_pool_size -= data->size;
e1071571
JJ		 779 gomp_mutex_unlock (&allocator_data->lock);
780#endif
781 }
450b05ce
TB		 782#ifdef LIBGOMP_USE_LIBNUMA
783 if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
784 {
785 libnuma_data->numa_free (data->ptr, data->size);
786 return;
787 }
788# ifdef LIBGOMP_USE_MEMKIND
789 else
790# endif
791#endif
17f52a1c
JJ		 792#ifdef LIBGOMP_USE_MEMKIND
793 if (allocator_data->memkind)
794 {
795 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
796 void *kind = *memkind_data->kinds[allocator_data->memkind];
797 memkind_data->memkind_free (kind, data->ptr);
798 return;
799 }
800#endif
30486fab
AS		 801
802 memspace = allocator_data->memspace;
348874f0 803 pinned = allocator_data->pinned;
e1071571 804 }
17f52a1c
JJ		 805 else
806 {
30486fab 807#ifdef LIBGOMP_USE_MEMKIND
450b05ce 808 enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
17f52a1c
JJ		 809 if (data->allocator == omp_high_bw_mem_alloc)
810 memkind = GOMP_MEMKIND_HBW_PREFERRED;
811 else if (data->allocator == omp_large_cap_mem_alloc)
812 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
813 if (memkind)
814 {
815 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
816 if (memkind_data->kinds[memkind])
817 {
818 void *kind = *memkind_data->kinds[memkind];
819 memkind_data->memkind_free (kind, data->ptr);
820 return;
821 }
822 }
17f52a1c 823#endif
30486fab
AS		 824
825 memspace = predefined_alloc_mapping[data->allocator];
826 }
827
348874f0 828 MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
e1071571 829}
6fcc3cac
JJ		 830
831ialias (omp_free)
832
833void
834GOMP_free (void *ptr, uintptr_t allocator)
835{
b38a4bd1
JJ		 836 return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
837}
838
839void *
840omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
841 omp_allocator_handle_t allocator)
842{
843 struct omp_allocator_data *allocator_data;
844 size_t new_size, size_temp, new_alignment;
845 void *ptr, *ret;
450b05ce
TB		 846#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
847 enum gomp_numa_memkind_kind memkind;
17f52a1c 848#endif
b38a4bd1
JJ		 849
850 if (__builtin_expect (size == 0 || nmemb == 0, 0))
851 return NULL;
852
853retry:
854 new_alignment = alignment;
855 if (allocator == omp_null_allocator)
856 {
857 struct gomp_thread *thr = gomp_thread ();
858 if (thr->ts.def_allocator == omp_null_allocator)
859 thr->ts.def_allocator = gomp_def_allocator;
860 allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
861 }
862
863 if (allocator > omp_max_predefined_alloc)
864 {
865 allocator_data = (struct omp_allocator_data *) allocator;
866 if (new_alignment < allocator_data->alignment)
867 new_alignment = allocator_data->alignment;
450b05ce 868#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 869 memkind = allocator_data->memkind;
870#endif
b38a4bd1
JJ		 871 }
872 else
873 {
874 allocator_data = NULL;
875 if (new_alignment < sizeof (void *))
876 new_alignment = sizeof (void *);
450b05ce 877#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 878 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 879#endif
880#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 881 if (allocator == omp_high_bw_mem_alloc)
882 memkind = GOMP_MEMKIND_HBW_PREFERRED;
883 else if (allocator == omp_large_cap_mem_alloc)
884 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
885 if (memkind)
886 {
887 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
888 if (!memkind_data->kinds[memkind])
889 memkind = GOMP_MEMKIND_NONE;
890 }
891#endif
b38a4bd1
JJ		 892 }
893
894 new_size = sizeof (struct omp_mem_header);
895 if (new_alignment > sizeof (void *))
896 new_size += new_alignment - sizeof (void *);
897 if (__builtin_mul_overflow (size, nmemb, &size_temp))
898 goto fail;
899 if (__builtin_add_overflow (size_temp, new_size, &new_size))
900 goto fail;
e9a19ead
AS		 901#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
902 if (allocator == omp_low_lat_mem_alloc)
903 goto fail;
904#endif
b38a4bd1
JJ		 905
906 if (__builtin_expect (allocator_data
907 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
908 {
909 uintptr_t used_pool_size;
910 if (new_size > allocator_data->pool_size)
911 goto fail;
912#ifdef HAVE_SYNC_BUILTINS
913 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
914 MEMMODEL_RELAXED);
915 do
916 {
917 uintptr_t new_pool_size;
918 if (__builtin_add_overflow (used_pool_size, new_size,
919 &new_pool_size)
920 || new_pool_size > allocator_data->pool_size)
921 goto fail;
922 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
923 &used_pool_size, new_pool_size,
924 true, MEMMODEL_RELAXED,
925 MEMMODEL_RELAXED))
926 break;
927 }
928 while (1);
929#else
930 gomp_mutex_lock (&allocator_data->lock);
931 if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
932 &used_pool_size)
933 || used_pool_size > allocator_data->pool_size)
934 {
935 gomp_mutex_unlock (&allocator_data->lock);
936 goto fail;
937 }
938 allocator_data->used_pool_size = used_pool_size;
939 gomp_mutex_unlock (&allocator_data->lock);
940#endif
450b05ce
TB		 941#ifdef LIBGOMP_USE_LIBNUMA
942 if (memkind == GOMP_MEMKIND_LIBNUMA)
943 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
944 memory that is initialized to zero. */
945 ptr = libnuma_data->numa_alloc_local (new_size);
946# ifdef LIBGOMP_USE_MEMKIND
947 else
948# endif
949#endif
17f52a1c
JJ		 950#ifdef LIBGOMP_USE_MEMKIND
951 if (memkind)
952 {
953 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
954 void *kind = *memkind_data->kinds[memkind];
955 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
956 }
957 else
958#endif
348874f0
AS		 959 ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size,
960 allocator_data->pinned);
b38a4bd1
JJ		 961 if (ptr == NULL)
962 {
963#ifdef HAVE_SYNC_BUILTINS
964 __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
965 MEMMODEL_RELAXED);
966#else
967 gomp_mutex_lock (&allocator_data->lock);
968 allocator_data->used_pool_size -= new_size;
969 gomp_mutex_unlock (&allocator_data->lock);
970#endif
971 goto fail;
972 }
973 }
974 else
975 {
450b05ce
TB		 976#ifdef LIBGOMP_USE_LIBNUMA
977 if (memkind == GOMP_MEMKIND_LIBNUMA)
978 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
979 memory that is initialized to zero. */
980 ptr = libnuma_data->numa_alloc_local (new_size);
981# ifdef LIBGOMP_USE_MEMKIND
982 else
983# endif
984#endif
17f52a1c
JJ		 985#ifdef LIBGOMP_USE_MEMKIND
986 if (memkind)
987 {
988 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
989 void *kind = *memkind_data->kinds[memkind];
990 ptr = memkind_data->memkind_calloc (kind, 1, new_size);
991 }
992 else
993#endif
30486fab
AS		 994 {
995 omp_memspace_handle_t memspace;
996 memspace = (allocator_data
997 ? allocator_data->memspace
998 : predefined_alloc_mapping[allocator]);
348874f0
AS		 999 ptr = MEMSPACE_CALLOC (memspace, new_size,
1000 allocator_data && allocator_data->pinned);
30486fab 1001 }
b38a4bd1
JJ		 1002 if (ptr == NULL)
1003 goto fail;
1004 }
1005
1006 if (new_alignment > sizeof (void *))
1007 ret = (void *) (((uintptr_t) ptr
1008 + sizeof (struct omp_mem_header)
1009 + new_alignment - sizeof (void *))
1010 & ~(new_alignment - 1));
1011 else
1012 ret = (char *) ptr + sizeof (struct omp_mem_header);
1013 ((struct omp_mem_header *) ret)[-1].ptr = ptr;
1014 ((struct omp_mem_header *) ret)[-1].size = new_size;
1015 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1016 return ret;
1017
30486fab
AS		 1018fail:;
1019 int fallback = (allocator_data
1020 ? allocator_data->fallback
1021 : allocator == omp_default_mem_alloc
1022 ? omp_atv_null_fb
1023 : omp_atv_default_mem_fb);
1024 switch (fallback)
b38a4bd1 1025 {
30486fab
AS		 1026 case omp_atv_default_mem_fb:
1027 allocator = omp_default_mem_alloc;
1028 goto retry;
1029 case omp_atv_null_fb:
1030 break;
1031 default:
1032 case omp_atv_abort_fb:
1033 gomp_fatal ("Out of memory allocating %lu bytes",
1034 (unsigned long) (size * nmemb));
1035 case omp_atv_allocator_fb:
1036 allocator = allocator_data->fb_data;
1037 goto retry;
b38a4bd1
JJ		 1038 }
1039 return NULL;
1040}
1041
1042ialias (omp_aligned_calloc)
1043
1044void *
1045omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
1046{
1047 return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
1048}
1049
1050void *
1051omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
1052 omp_allocator_handle_t free_allocator)
1053{
1054 struct omp_allocator_data *allocator_data, *free_allocator_data;
1055 size_t new_size, old_size, new_alignment, old_alignment;
1056 void *new_ptr, *ret;
1057 struct omp_mem_header *data;
450b05ce
TB		 1058#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1059 enum gomp_numa_memkind_kind memkind, free_memkind;
17f52a1c 1060#endif
b38a4bd1
JJ		 1061
1062 if (__builtin_expect (ptr == NULL, 0))
1063 return ialias_call (omp_aligned_alloc) (1, size, allocator);
1064
1065 if (__builtin_expect (size == 0, 0))
1066 {
1067 ialias_call (omp_free) (ptr, free_allocator);
1068 return NULL;
1069 }
1070
1071 data = &((struct omp_mem_header *) ptr)[-1];
1072 free_allocator = data->allocator;
1073
1074retry:
1075 new_alignment = sizeof (void *);
1076 if (allocator == omp_null_allocator)
1077 allocator = free_allocator;
1078
1079 if (allocator > omp_max_predefined_alloc)
1080 {
1081 allocator_data = (struct omp_allocator_data *) allocator;
1082 if (new_alignment < allocator_data->alignment)
1083 new_alignment = allocator_data->alignment;
450b05ce 1084#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1085 memkind = allocator_data->memkind;
1086#endif
b38a4bd1
JJ		 1087 }
1088 else
17f52a1c
JJ		 1089 {
1090 allocator_data = NULL;
450b05ce 1091#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 1092 memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 1093#endif
1094#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 1095 if (allocator == omp_high_bw_mem_alloc)
1096 memkind = GOMP_MEMKIND_HBW_PREFERRED;
1097 else if (allocator == omp_large_cap_mem_alloc)
1098 memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1099 if (memkind)
1100 {
1101 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1102 if (!memkind_data->kinds[memkind])
1103 memkind = GOMP_MEMKIND_NONE;
1104 }
1105#endif
1106 }
b38a4bd1 1107 if (free_allocator > omp_max_predefined_alloc)
17f52a1c
JJ		 1108 {
1109 free_allocator_data = (struct omp_allocator_data *) free_allocator;
450b05ce 1110#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1111 free_memkind = free_allocator_data->memkind;
1112#endif
1113 }
b38a4bd1 1114 else
17f52a1c
JJ		 1115 {
1116 free_allocator_data = NULL;
450b05ce 1117#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c 1118 free_memkind = GOMP_MEMKIND_NONE;
450b05ce
TB		 1119#endif
1120#ifdef LIBGOMP_USE_MEMKIND
17f52a1c
JJ		 1121 if (free_allocator == omp_high_bw_mem_alloc)
1122 free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1123 else if (free_allocator == omp_large_cap_mem_alloc)
1124 free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1125 if (free_memkind)
1126 {
1127 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1128 if (!memkind_data->kinds[free_memkind])
1129 free_memkind = GOMP_MEMKIND_NONE;
1130 }
1131#endif
1132 }
b38a4bd1
JJ		 1133 old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1134
1135 new_size = sizeof (struct omp_mem_header);
1136 if (new_alignment > sizeof (void *))
1137 new_size += new_alignment - sizeof (void *);
1138 if (__builtin_add_overflow (size, new_size, &new_size))
1139 goto fail;
1140 old_size = data->size;
e9a19ead
AS		 1141#ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1142 if (allocator == omp_low_lat_mem_alloc)
1143 goto fail;
1144#endif
b38a4bd1
JJ		 1145
1146 if (__builtin_expect (allocator_data
1147 && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1148 {
1149 uintptr_t used_pool_size;
1150 size_t prev_size = 0;
1151 /* Check if we can use realloc. Don't use it if extra alignment
1152 was used previously or newly, because realloc might return a pointer
1153 with different alignment and then we'd need to memmove the data
1154 again. */
1155 if (free_allocator_data
1156 && free_allocator_data == allocator_data
1157 && new_alignment == sizeof (void *)
1158 && old_alignment == sizeof (struct omp_mem_header))
1159 prev_size = old_size;
1160 if (new_size > prev_size
1161 && new_size - prev_size > allocator_data->pool_size)
1162 goto fail;
1163#ifdef HAVE_SYNC_BUILTINS
1164 used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1165 MEMMODEL_RELAXED);
1166 do
1167 {
1168 uintptr_t new_pool_size;
1169 if (new_size > prev_size)
1170 {
1171 if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1172 &new_pool_size)
1173 || new_pool_size > allocator_data->pool_size)
1174 goto fail;
1175 }
1176 else
1177 new_pool_size = used_pool_size + new_size - prev_size;
1178 if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1179 &used_pool_size, new_pool_size,
1180 true, MEMMODEL_RELAXED,
1181 MEMMODEL_RELAXED))
1182 break;
1183 }
1184 while (1);
1185#else
1186 gomp_mutex_lock (&allocator_data->lock);
1187 if (new_size > prev_size)
1188 {
1189 if (__builtin_add_overflow (allocator_data->used_pool_size,
1190 new_size - prev_size,
1191 &used_pool_size)
1192 || used_pool_size > allocator_data->pool_size)
1193 {
1194 gomp_mutex_unlock (&allocator_data->lock);
1195 goto fail;
1196 }
1197 }
1198 else
1199 used_pool_size = (allocator_data->used_pool_size
1200 + new_size - prev_size);
1201 allocator_data->used_pool_size = used_pool_size;
1202 gomp_mutex_unlock (&allocator_data->lock);
17f52a1c 1203#endif
450b05ce
TB		 1204#ifdef LIBGOMP_USE_LIBNUMA
1205 if (memkind == GOMP_MEMKIND_LIBNUMA)
1206 {
1207 if (prev_size)
1208 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1209 new_size);
1210 else
1211 new_ptr = libnuma_data->numa_alloc_local (new_size);
1212 }
1213# ifdef LIBGOMP_USE_MEMKIND
1214 else
1215# endif
1216#endif
17f52a1c
JJ		 1217#ifdef LIBGOMP_USE_MEMKIND
1218 if (memkind)
1219 {
1220 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1221 void *kind = *memkind_data->kinds[memkind];
1222 if (prev_size)
1223 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1224 new_size);
1225 else
1226 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1227 }
1228 else
b38a4bd1
JJ		 1229#endif
1230 if (prev_size)
30486fab 1231 new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
348874f0
AS		 1232 data->size, new_size,
1233 (free_allocator_data
1234 && free_allocator_data->pinned),
1235 allocator_data->pinned);
b38a4bd1 1236 else
348874f0
AS		 1237 new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
1238 allocator_data->pinned);
b38a4bd1
JJ		 1239 if (new_ptr == NULL)
1240 {
1241#ifdef HAVE_SYNC_BUILTINS
1242 __atomic_add_fetch (&allocator_data->used_pool_size,
1243 prev_size - new_size,
1244 MEMMODEL_RELAXED);
1245#else
1246 gomp_mutex_lock (&allocator_data->lock);
1247 allocator_data->used_pool_size -= new_size - prev_size;
1248 gomp_mutex_unlock (&allocator_data->lock);
1249#endif
1250 goto fail;
1251 }
1252 else if (prev_size)
1253 {
1254 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1255 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1256 ((struct omp_mem_header *) ret)[-1].size = new_size;
1257 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1258 return ret;
1259 }
1260 }
1261 else if (new_alignment == sizeof (void *)
1262 && old_alignment == sizeof (struct omp_mem_header)
450b05ce 1263#if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
17f52a1c
JJ		 1264 && memkind == free_memkind
1265#endif
b38a4bd1
JJ		 1266 && (free_allocator_data == NULL
1267 || free_allocator_data->pool_size == ~(uintptr_t) 0))
1268 {
450b05ce
TB		 1269#ifdef LIBGOMP_USE_LIBNUMA
1270 if (memkind == GOMP_MEMKIND_LIBNUMA)
1271 new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1272# ifdef LIBGOMP_USE_MEMKIND
1273 else
1274# endif
1275#endif
17f52a1c
JJ		 1276#ifdef LIBGOMP_USE_MEMKIND
1277 if (memkind)
1278 {
1279 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1280 void *kind = *memkind_data->kinds[memkind];
1281 new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1282 new_size);
1283 }
1284 else
1285#endif
30486fab
AS		 1286 {
1287 omp_memspace_handle_t memspace;
1288 memspace = (allocator_data
1289 ? allocator_data->memspace
1290 : predefined_alloc_mapping[allocator]);
348874f0
AS		 1291 new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
1292 (free_allocator_data
1293 && free_allocator_data->pinned),
1294 allocator_data && allocator_data->pinned);
30486fab 1295 }
b38a4bd1
JJ		 1296 if (new_ptr == NULL)
1297 goto fail;
348874f0 1298
b38a4bd1
JJ		 1299 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1300 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1301 ((struct omp_mem_header *) ret)[-1].size = new_size;
1302 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1303 return ret;
1304 }
1305 else
1306 {
450b05ce
TB		 1307#ifdef LIBGOMP_USE_LIBNUMA
1308 if (memkind == GOMP_MEMKIND_LIBNUMA)
1309 new_ptr = libnuma_data->numa_alloc_local (new_size);
1310# ifdef LIBGOMP_USE_MEMKIND
1311 else
1312# endif
1313#endif
17f52a1c
JJ		 1314#ifdef LIBGOMP_USE_MEMKIND
1315 if (memkind)
1316 {
1317 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1318 void *kind = *memkind_data->kinds[memkind];
1319 new_ptr = memkind_data->memkind_malloc (kind, new_size);
1320 }
1321 else
1322#endif
30486fab
AS		 1323 {
1324 omp_memspace_handle_t memspace;
1325 memspace = (allocator_data
1326 ? allocator_data->memspace
1327 : predefined_alloc_mapping[allocator]);
348874f0
AS		 1328 new_ptr = MEMSPACE_ALLOC (memspace, new_size,
1329 allocator_data && allocator_data->pinned);
30486fab 1330 }
b38a4bd1
JJ		 1331 if (new_ptr == NULL)
1332 goto fail;
1333 }
1334
1335 if (new_alignment > sizeof (void *))
1336 ret = (void *) (((uintptr_t) new_ptr
1337 + sizeof (struct omp_mem_header)
1338 + new_alignment - sizeof (void *))
1339 & ~(new_alignment - 1));
1340 else
1341 ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1342 ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1343 ((struct omp_mem_header *) ret)[-1].size = new_size;
1344 ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1345 if (old_size - old_alignment < size)
1346 size = old_size - old_alignment;
1347 memcpy (ret, ptr, size);
1348 if (__builtin_expect (free_allocator_data
1349 && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1350 {
1351#ifdef HAVE_SYNC_BUILTINS
1352 __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1353 MEMMODEL_RELAXED);
1354#else
1355 gomp_mutex_lock (&free_allocator_data->lock);
1356 free_allocator_data->used_pool_size -= data->size;
1357 gomp_mutex_unlock (&free_allocator_data->lock);
1358#endif
1359 }
450b05ce
TB		 1360#ifdef LIBGOMP_USE_LIBNUMA
1361 if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1362 {
1363 libnuma_data->numa_free (data->ptr, data->size);
1364 return ret;
1365 }
1366# ifdef LIBGOMP_USE_MEMKIND
1367 else
1368# endif
1369#endif
17f52a1c
JJ		 1370#ifdef LIBGOMP_USE_MEMKIND
1371 if (free_memkind)
1372 {
1373 struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1374 void *kind = *memkind_data->kinds[free_memkind];
1375 memkind_data->memkind_free (kind, data->ptr);
1376 return ret;
1377 }
1378#endif
30486fab
AS		 1379 {
1380 omp_memspace_handle_t was_memspace;
1381 was_memspace = (free_allocator_data
1382 ? free_allocator_data->memspace
1383 : predefined_alloc_mapping[free_allocator]);
348874f0
AS		 1384 int was_pinned = (free_allocator_data && free_allocator_data->pinned);
1385 MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
30486fab 1386 }
b38a4bd1
JJ		 1387 return ret;
1388
30486fab
AS		 1389fail:;
1390 int fallback = (allocator_data
1391 ? allocator_data->fallback
1392 : allocator == omp_default_mem_alloc
1393 ? omp_atv_null_fb
1394 : omp_atv_default_mem_fb);
1395 switch (fallback)
b38a4bd1 1396 {
30486fab
AS		 1397 case omp_atv_default_mem_fb:
1398 allocator = omp_default_mem_alloc;
1399 goto retry;
1400 case omp_atv_null_fb:
1401 break;
1402 default:
1403 case omp_atv_abort_fb:
1404 gomp_fatal ("Out of memory allocating %lu bytes",
1405 (unsigned long) size);
1406 case omp_atv_allocator_fb:
1407 allocator = allocator_data->fb_data;
1408 goto retry;
b38a4bd1
JJ		 1409 }
1410 return NULL;
6fcc3cac 1411}
Mirror of https://gcc.gnu.org/git/gcc.git