libgomp/allocator.c

   1 /* Copyright (C) 2020-2023 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
  35 #include <dlfcn.h>
  36 #endif
  37
  38 /* Keeping track whether a Fortran scalar allocatable/pointer has been
  39    allocated via 'omp allocators'/'omp allocate'.  */
  40
  41 struct fort_alloc_splay_tree_key_s {
  42   void *ptr;
  43 };
  44
  45 typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node;
  46 typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree;
  47 typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key;
  48
  49 static inline int
  50 fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y)
  51 {
  52   if (x->ptr < y->ptr)
  53     return -1;
  54   if (x->ptr > y->ptr)
  55     return 1;
  56   return 0;
  57 }
  58 #define splay_tree_prefix fort_alloc
  59 #define splay_tree_static
  60 #include "splay-tree.h"
  61
  62 #define splay_tree_prefix fort_alloc
  63 #define splay_tree_static
  64 #define splay_tree_c
  65 #include "splay-tree.h"
  66
  67 static struct fort_alloc_splay_tree_s fort_alloc_scalars;
  68
  69 /* Add pointer as being alloced by GOMP_alloc.  */
  70 void
  71 GOMP_add_alloc (void *ptr)
  72 {
  73   if (ptr == NULL)
  74     return;
  75   fort_alloc_splay_tree_node item;
  76   item = gomp_malloc (sizeof (struct splay_tree_node_s));
  77   item->key.ptr = ptr;
  78   item->left = NULL;
  79   item->right = NULL;
  80   fort_alloc_splay_tree_insert (&fort_alloc_scalars, item);
  81 }
  82
  83 /* Remove pointer, either called by FREE or by REALLOC,
  84    either of them can change the allocation status.  */
  85 bool
  86 GOMP_is_alloc (void *ptr)
  87 {
  88   struct fort_alloc_splay_tree_key_s needle;
  89   fort_alloc_splay_tree_node n;
  90   needle.ptr = ptr;
  91   n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle);
  92   if (n)
  93     {
  94       fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key);
  95       free (n);
  96     }
  97   return n != NULL;
  98 }
  99
 100
 101 #define omp_max_predefined_alloc omp_thread_mem_alloc
 102
 103 /* These macros may be overridden in config/<target>/allocator.c.
 104    The following definitions (ab)use comma operators to avoid unused
 105    variable errors.  */
 106 #ifndef MEMSPACE_ALLOC
 107 #define MEMSPACE_ALLOC(MEMSPACE, SIZE) \
 108   malloc (((void)(MEMSPACE), (SIZE)))
 109 #endif
 110 #ifndef MEMSPACE_CALLOC
 111 #define MEMSPACE_CALLOC(MEMSPACE, SIZE) \
 112   calloc (1, (((void)(MEMSPACE), (SIZE))))
 113 #endif
 114 #ifndef MEMSPACE_REALLOC
 115 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \
 116   realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE))))
 117 #endif
 118 #ifndef MEMSPACE_FREE
 119 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \
 120   free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
 121 #endif
 122 #ifndef MEMSPACE_VALIDATE
 123 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
 124   (((void)(MEMSPACE), (void)(ACCESS), 1))
 125 #endif
 126
 127 /* Map the predefined allocators to the correct memory space.
 128    The index to this table is the omp_allocator_handle_t enum value.
 129    When the user calls omp_alloc with a predefined allocator this
 130    table determines what memory they get.  */
 131 static const omp_memspace_handle_t predefined_alloc_mapping[] = {
 132   omp_default_mem_space,   /* omp_null_allocator doesn't actually use this. */
 133   omp_default_mem_space,   /* omp_default_mem_alloc. */
 134   omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
 135   omp_const_mem_space,     /* omp_const_mem_alloc. */
 136   omp_high_bw_mem_space,   /* omp_high_bw_mem_alloc. */
 137   omp_low_lat_mem_space,   /* omp_low_lat_mem_alloc. */
 138   omp_low_lat_mem_space,   /* omp_cgroup_mem_alloc (implementation defined). */
 139   omp_low_lat_mem_space,   /* omp_pteam_mem_alloc (implementation defined). */
 140   omp_low_lat_mem_space,   /* omp_thread_mem_alloc (implementation defined). */
 141 };
 142
 143 #define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
 144 _Static_assert (ARRAY_SIZE (predefined_alloc_mapping)
 145                 == omp_max_predefined_alloc + 1,
 146                 "predefined_alloc_mapping must match omp_memspace_handle_t");
 147
 148 enum gomp_numa_memkind_kind
 149 {
 150   GOMP_MEMKIND_NONE = 0,
 151 #define GOMP_MEMKIND_KINDS \
 152   GOMP_MEMKIND_KIND (HBW_INTERLEAVE),           \
 153   GOMP_MEMKIND_KIND (HBW_PREFERRED),            \
 154   GOMP_MEMKIND_KIND (DAX_KMEM_ALL),             \
 155   GOMP_MEMKIND_KIND (DAX_KMEM),                 \
 156   GOMP_MEMKIND_KIND (INTERLEAVE),               \
 157   GOMP_MEMKIND_KIND (DEFAULT)
 158 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
 159   GOMP_MEMKIND_KINDS,
 160 #undef GOMP_MEMKIND_KIND
 161   GOMP_MEMKIND_COUNT,
 162   GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
 163 };
 164
 165 struct omp_allocator_data
 166 {
 167   omp_memspace_handle_t memspace;
 168   omp_uintptr_t alignment;
 169   omp_uintptr_t pool_size;
 170   omp_uintptr_t used_pool_size;
 171   omp_allocator_handle_t fb_data;
 172   unsigned int sync_hint : 8;
 173   unsigned int access : 8;
 174   unsigned int fallback : 8;
 175   unsigned int pinned : 1;
 176   unsigned int partition : 7;
 177 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 178   unsigned int memkind : 8;
 179 #endif
 180 #ifndef HAVE_SYNC_BUILTINS
 181   gomp_mutex_t lock;
 182 #endif
 183 };
 184
 185 struct omp_mem_header
 186 {
 187   void *ptr;
 188   size_t size;
 189   omp_allocator_handle_t allocator;
 190   void *pad;
 191 };
 192
 193 struct gomp_libnuma_data
 194 {
 195   void *numa_handle;
 196   void *(*numa_alloc_local) (size_t);
 197   void *(*numa_realloc) (void *, size_t, size_t);
 198   void (*numa_free) (void *, size_t);
 199 };
 200
 201 struct gomp_memkind_data
 202 {
 203   void *memkind_handle;
 204   void *(*memkind_malloc) (void *, size_t);
 205   void *(*memkind_calloc) (void *, size_t, size_t);
 206   void *(*memkind_realloc) (void *, void *, size_t);
 207   void (*memkind_free) (void *, void *);
 208   int (*memkind_check_available) (void *);
 209   void **kinds[GOMP_MEMKIND_COUNT];
 210 };
 211
 212 #ifdef LIBGOMP_USE_LIBNUMA
 213 static struct gomp_libnuma_data *libnuma_data;
 214 static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
 215
 216 static void
 217 gomp_init_libnuma (void)
 218 {
 219   void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
 220   struct gomp_libnuma_data *data;
 221
 222   data = calloc (1, sizeof (struct gomp_libnuma_data));
 223   if (data == NULL)
 224     {
 225       if (handle)
 226         dlclose (handle);
 227       return;
 228     }
 229   if (handle)
 230     {
 231       int (*numa_available) (void);
 232       numa_available
 233         = (__typeof (numa_available)) dlsym (handle, "numa_available");
 234       if (!numa_available || numa_available () != 0)
 235         {
 236           dlclose (handle);
 237           handle = NULL;
 238         }
 239     }
 240   if (!handle)
 241     {
 242       __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 243       return;
 244     }
 245   data->numa_handle = handle;
 246   data->numa_alloc_local
 247     = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
 248   data->numa_realloc
 249     = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
 250   data->numa_free
 251     = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
 252   __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 253 }
 254
 255 static struct gomp_libnuma_data *
 256 gomp_get_libnuma (void)
 257 {
 258   struct gomp_libnuma_data *data
 259     = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 260   if (data)
 261     return data;
 262   pthread_once (&libnuma_data_once, gomp_init_libnuma);
 263   return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 264 }
 265 #endif
 266
 267 #ifdef LIBGOMP_USE_MEMKIND
 268 static struct gomp_memkind_data *memkind_data;
 269 static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
 270
 271 static void
 272 gomp_init_memkind (void)
 273 {
 274   void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
 275   struct gomp_memkind_data *data;
 276   int i;
 277   static const char *kinds[] = {
 278     NULL,
 279 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
 280     GOMP_MEMKIND_KINDS
 281 #undef GOMP_MEMKIND_KIND
 282   };
 283
 284   data = calloc (1, sizeof (struct gomp_memkind_data));
 285   if (data == NULL)
 286     {
 287       if (handle)
 288         dlclose (handle);
 289       return;
 290     }
 291   if (!handle)
 292     {
 293       __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 294       return;
 295     }
 296   data->memkind_handle = handle;
 297   data->memkind_malloc
 298     = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
 299   data->memkind_calloc
 300     = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
 301   data->memkind_realloc
 302     = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
 303   data->memkind_free
 304     = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
 305   data->memkind_check_available
 306     = (__typeof (data->memkind_check_available))
 307       dlsym (handle, "memkind_check_available");
 308   if (data->memkind_malloc
 309       && data->memkind_calloc
 310       && data->memkind_realloc
 311       && data->memkind_free
 312       && data->memkind_check_available)
 313     for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
 314       {
 315         data->kinds[i] = (void **) dlsym (handle, kinds[i]);
 316         if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
 317           data->kinds[i] = NULL;
 318       }
 319   __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 320 }
 321
 322 static struct gomp_memkind_data *
 323 gomp_get_memkind (void)
 324 {
 325   struct gomp_memkind_data *data
 326     = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 327   if (data)
 328     return data;
 329   pthread_once (&memkind_data_once, gomp_init_memkind);
 330   return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 331 }
 332 #endif
 333
 334 omp_allocator_handle_t
 335 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
 336                     const omp_alloctrait_t traits[])
 337 {
 338   struct omp_allocator_data data
 339     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
 340         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
 341 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 342         GOMP_MEMKIND_NONE
 343 #endif
 344       };
 345   struct omp_allocator_data *ret;
 346   int i;
 347
 348   if (memspace > omp_low_lat_mem_space)
 349     return omp_null_allocator;
 350   for (i = 0; i < ntraits; i++)
 351     switch (traits[i].key)
 352       {
 353       case omp_atk_sync_hint:
 354         switch (traits[i].value)
 355           {
 356           case omp_atv_default:
 357             data.sync_hint = omp_atv_contended;
 358             break;
 359           case omp_atv_contended:
 360           case omp_atv_uncontended:
 361           case omp_atv_serialized:
 362           case omp_atv_private:
 363             data.sync_hint = traits[i].value;
 364             break;
 365           default:
 366             return omp_null_allocator;
 367           }
 368         break;
 369       case omp_atk_alignment:
 370         if (traits[i].value == omp_atv_default)
 371           {
 372             data.alignment = 1;
 373             break;
 374           }
 375         if ((traits[i].value & (traits[i].value - 1)) != 0
 376             || !traits[i].value)
 377           return omp_null_allocator;
 378         data.alignment = traits[i].value;
 379         break;
 380       case omp_atk_access:
 381         switch (traits[i].value)
 382           {
 383           case omp_atv_default:
 384             data.access = omp_atv_all;
 385             break;
 386           case omp_atv_all:
 387           case omp_atv_cgroup:
 388           case omp_atv_pteam:
 389           case omp_atv_thread:
 390             data.access = traits[i].value;
 391             break;
 392           default:
 393             return omp_null_allocator;
 394           }
 395         break;
 396       case omp_atk_pool_size:
 397         if (traits[i].value == omp_atv_default)
 398           data.pool_size = ~(uintptr_t) 0;
 399         else
 400           data.pool_size = traits[i].value;
 401         break;
 402       case omp_atk_fallback:
 403         switch (traits[i].value)
 404           {
 405           case omp_atv_default:
 406             data.fallback = omp_atv_default_mem_fb;
 407             break;
 408           case omp_atv_default_mem_fb:
 409           case omp_atv_null_fb:
 410           case omp_atv_abort_fb:
 411           case omp_atv_allocator_fb:
 412             data.fallback = traits[i].value;
 413             break;
 414           default:
 415             return omp_null_allocator;
 416           }
 417         break;
 418       case omp_atk_fb_data:
 419         data.fb_data = traits[i].value;
 420         break;
 421       case omp_atk_pinned:
 422         switch (traits[i].value)
 423           {
 424           case omp_atv_default:
 425           case omp_atv_false:
 426             data.pinned = omp_atv_false;
 427             break;
 428           case omp_atv_true:
 429             data.pinned = omp_atv_true;
 430             break;
 431           default:
 432             return omp_null_allocator;
 433           }
 434         break;
 435       case omp_atk_partition:
 436         switch (traits[i].value)
 437           {
 438           case omp_atv_default:
 439             data.partition = omp_atv_environment;
 440             break;
 441           case omp_atv_environment:
 442           case omp_atv_nearest:
 443           case omp_atv_blocked:
 444           case omp_atv_interleaved:
 445             data.partition = traits[i].value;
 446             break;
 447           default:
 448             return omp_null_allocator;
 449           }
 450         break;
 451       default:
 452         return omp_null_allocator;
 453       }
 454
 455   if (data.alignment < sizeof (void *))
 456     data.alignment = sizeof (void *);
 457
 458   switch (memspace)
 459     {
 460 #ifdef LIBGOMP_USE_MEMKIND
 461     case omp_high_bw_mem_space:
 462       struct gomp_memkind_data *memkind_data;
 463       memkind_data = gomp_get_memkind ();
 464       if (data.partition == omp_atv_interleaved
 465           && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
 466         {
 467           data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
 468           break;
 469         }
 470       else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
 471         {
 472           data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
 473           break;
 474         }
 475       break;
 476     case omp_large_cap_mem_space:
 477       memkind_data = gomp_get_memkind ();
 478       if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
 479         data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 480       else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
 481         data.memkind = GOMP_MEMKIND_DAX_KMEM;
 482       break;
 483 #endif
 484     default:
 485 #ifdef LIBGOMP_USE_MEMKIND
 486       if (data.partition == omp_atv_interleaved)
 487         {
 488           memkind_data = gomp_get_memkind ();
 489           if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
 490             data.memkind = GOMP_MEMKIND_INTERLEAVE;
 491         }
 492 #endif
 493       break;
 494     }
 495
 496 #ifdef LIBGOMP_USE_LIBNUMA
 497   if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
 498     {
 499       libnuma_data = gomp_get_libnuma ();
 500       if (libnuma_data->numa_alloc_local != NULL)
 501         data.memkind = GOMP_MEMKIND_LIBNUMA;
 502     }
 503 #endif
 504
 505   /* No support for this so far.  */
 506   if (data.pinned)
 507     return omp_null_allocator;
 508
 509   /* Reject unsupported memory spaces.  */
 510   if (!MEMSPACE_VALIDATE (data.memspace, data.access))
 511     return omp_null_allocator;
 512
 513   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 514   *ret = data;
 515 #ifndef HAVE_SYNC_BUILTINS
 516   gomp_mutex_init (&ret->lock);
 517 #endif
 518   return (omp_allocator_handle_t) ret;
 519 }
 520
 521 void
 522 omp_destroy_allocator (omp_allocator_handle_t allocator)
 523 {
 524   if (allocator != omp_null_allocator)
 525     {
 526 #ifndef HAVE_SYNC_BUILTINS
 527       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 528 #endif
 529       free ((void *) allocator);
 530     }
 531 }
 532
 533 ialias (omp_init_allocator)
 534 ialias (omp_destroy_allocator)
 535
 536 void *
 537 omp_aligned_alloc (size_t alignment, size_t size,
 538                    omp_allocator_handle_t allocator)
 539 {
 540   struct omp_allocator_data *allocator_data;
 541   size_t new_size, new_alignment;
 542   void *ptr, *ret;
 543 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 544   enum gomp_numa_memkind_kind memkind;
 545 #endif
 546
 547   if (__builtin_expect (size == 0, 0))
 548     return NULL;
 549
 550 retry:
 551   new_alignment = alignment;
 552   if (allocator == omp_null_allocator)
 553     {
 554       struct gomp_thread *thr = gomp_thread ();
 555       if (thr->ts.def_allocator == omp_null_allocator)
 556         thr->ts.def_allocator = gomp_def_allocator;
 557       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 558     }
 559
 560   if (allocator > omp_max_predefined_alloc)
 561     {
 562       allocator_data = (struct omp_allocator_data *) allocator;
 563       if (new_alignment < allocator_data->alignment)
 564         new_alignment = allocator_data->alignment;
 565 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 566       memkind = allocator_data->memkind;
 567 #endif
 568     }
 569   else
 570     {
 571       allocator_data = NULL;
 572       if (new_alignment < sizeof (void *))
 573         new_alignment = sizeof (void *);
 574 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 575       memkind = GOMP_MEMKIND_NONE;
 576 #endif
 577 #ifdef LIBGOMP_USE_MEMKIND
 578       if (allocator == omp_high_bw_mem_alloc)
 579         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 580       else if (allocator == omp_large_cap_mem_alloc)
 581         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 582       if (memkind)
 583         {
 584           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 585           if (!memkind_data->kinds[memkind])
 586             memkind = GOMP_MEMKIND_NONE;
 587         }
 588 #endif
 589     }
 590
 591   new_size = sizeof (struct omp_mem_header);
 592   if (new_alignment > sizeof (void *))
 593     new_size += new_alignment - sizeof (void *);
 594   if (__builtin_add_overflow (size, new_size, &new_size))
 595     goto fail;
 596 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 597   if (allocator == omp_low_lat_mem_alloc)
 598     goto fail;
 599 #endif
 600
 601   if (__builtin_expect (allocator_data
 602                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 603     {
 604       uintptr_t used_pool_size;
 605       if (new_size > allocator_data->pool_size)
 606         goto fail;
 607 #ifdef HAVE_SYNC_BUILTINS
 608       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 609                                         MEMMODEL_RELAXED);
 610       do
 611         {
 612           uintptr_t new_pool_size;
 613           if (__builtin_add_overflow (used_pool_size, new_size,
 614                                       &new_pool_size)
 615               || new_pool_size > allocator_data->pool_size)
 616             goto fail;
 617           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 618                                            &used_pool_size, new_pool_size,
 619                                            true, MEMMODEL_RELAXED,
 620                                            MEMMODEL_RELAXED))
 621             break;
 622         }
 623       while (1);
 624 #else
 625       gomp_mutex_lock (&allocator_data->lock);
 626       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 627                                   &used_pool_size)
 628           || used_pool_size > allocator_data->pool_size)
 629         {
 630           gomp_mutex_unlock (&allocator_data->lock);
 631           goto fail;
 632         }
 633       allocator_data->used_pool_size = used_pool_size;
 634       gomp_mutex_unlock (&allocator_data->lock);
 635 #endif
 636 #ifdef LIBGOMP_USE_LIBNUMA
 637       if (memkind == GOMP_MEMKIND_LIBNUMA)
 638         ptr = libnuma_data->numa_alloc_local (new_size);
 639 # ifdef LIBGOMP_USE_MEMKIND
 640       else
 641 # endif
 642 #endif
 643 #ifdef LIBGOMP_USE_MEMKIND
 644       if (memkind)
 645         {
 646           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 647           void *kind = *memkind_data->kinds[memkind];
 648           ptr = memkind_data->memkind_malloc (kind, new_size);
 649         }
 650       else
 651 #endif
 652         ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
 653       if (ptr == NULL)
 654         {
 655 #ifdef HAVE_SYNC_BUILTINS
 656           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 657                               MEMMODEL_RELAXED);
 658 #else
 659           gomp_mutex_lock (&allocator_data->lock);
 660           allocator_data->used_pool_size -= new_size;
 661           gomp_mutex_unlock (&allocator_data->lock);
 662 #endif
 663           goto fail;
 664         }
 665     }
 666   else
 667     {
 668 #ifdef LIBGOMP_USE_LIBNUMA
 669       if (memkind == GOMP_MEMKIND_LIBNUMA)
 670         ptr = libnuma_data->numa_alloc_local (new_size);
 671 # ifdef LIBGOMP_USE_MEMKIND
 672       else
 673 # endif
 674 #endif
 675 #ifdef LIBGOMP_USE_MEMKIND
 676       if (memkind)
 677         {
 678           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 679           void *kind = *memkind_data->kinds[memkind];
 680           ptr = memkind_data->memkind_malloc (kind, new_size);
 681         }
 682       else
 683 #endif
 684         {
 685           omp_memspace_handle_t memspace;
 686           memspace = (allocator_data
 687                       ? allocator_data->memspace
 688                       : predefined_alloc_mapping[allocator]);
 689           ptr = MEMSPACE_ALLOC (memspace, new_size);
 690         }
 691       if (ptr == NULL)
 692         goto fail;
 693     }
 694
 695   if (new_alignment > sizeof (void *))
 696     ret = (void *) (((uintptr_t) ptr
 697                      + sizeof (struct omp_mem_header)
 698                      + new_alignment - sizeof (void *))
 699                     & ~(new_alignment - 1));
 700   else
 701     ret = (char *) ptr + sizeof (struct omp_mem_header);
 702   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 703   ((struct omp_mem_header *) ret)[-1].size = new_size;
 704   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 705   return ret;
 706
 707 fail:;
 708   int fallback = (allocator_data
 709                   ? allocator_data->fallback
 710                   : allocator == omp_default_mem_alloc
 711                   ? omp_atv_null_fb
 712                   : omp_atv_default_mem_fb);
 713   switch (fallback)
 714     {
 715     case omp_atv_default_mem_fb:
 716       allocator = omp_default_mem_alloc;
 717       goto retry;
 718     case omp_atv_null_fb:
 719       break;
 720     default:
 721     case omp_atv_abort_fb:
 722       gomp_fatal ("Out of memory allocating %lu bytes",
 723                   (unsigned long) size);
 724     case omp_atv_allocator_fb:
 725       allocator = allocator_data->fb_data;
 726       goto retry;
 727     }
 728   return NULL;
 729 }
 730
 731 ialias (omp_aligned_alloc)
 732
 733 void *
 734 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 735 {
 736   return ialias_call (omp_aligned_alloc) (1, size, allocator);
 737 }
 738
 739 /* Like omp_aligned_alloc, but apply on top of that:
 740    "For allocations that arise from this ... the null_fb value of the
 741    fallback allocator trait behaves as if the abort_fb had been specified."  */
 742
 743 void *
 744 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 745 {
 746   void *ret
 747     = ialias_call (omp_aligned_alloc) (alignment, size,
 748                                        (omp_allocator_handle_t) allocator);
 749   if (__builtin_expect (ret == NULL, 0) && size)
 750     gomp_fatal ("Out of memory allocating %lu bytes",
 751                 (unsigned long) size);
 752   return ret;
 753 }
 754
 755 void
 756 omp_free (void *ptr, omp_allocator_handle_t allocator)
 757 {
 758   struct omp_mem_header *data;
 759   omp_memspace_handle_t memspace = omp_default_mem_space;
 760
 761   if (ptr == NULL)
 762     return;
 763   (void) allocator;
 764   data = &((struct omp_mem_header *) ptr)[-1];
 765   if (data->allocator > omp_max_predefined_alloc)
 766     {
 767       struct omp_allocator_data *allocator_data
 768         = (struct omp_allocator_data *) (data->allocator);
 769       if (allocator_data->pool_size < ~(uintptr_t) 0)
 770         {
 771 #ifdef HAVE_SYNC_BUILTINS
 772           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 773                               MEMMODEL_RELAXED);
 774 #else
 775           gomp_mutex_lock (&allocator_data->lock);
 776           allocator_data->used_pool_size -= data->size;
 777           gomp_mutex_unlock (&allocator_data->lock);
 778 #endif
 779         }
 780 #ifdef LIBGOMP_USE_LIBNUMA
 781       if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
 782         {
 783           libnuma_data->numa_free (data->ptr, data->size);
 784           return;
 785         }
 786 # ifdef LIBGOMP_USE_MEMKIND
 787       else
 788 # endif
 789 #endif
 790 #ifdef LIBGOMP_USE_MEMKIND
 791       if (allocator_data->memkind)
 792         {
 793           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 794           void *kind = *memkind_data->kinds[allocator_data->memkind];
 795           memkind_data->memkind_free (kind, data->ptr);
 796           return;
 797         }
 798 #endif
 799
 800       memspace = allocator_data->memspace;
 801     }
 802   else
 803     {
 804 #ifdef LIBGOMP_USE_MEMKIND
 805       enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
 806       if (data->allocator == omp_high_bw_mem_alloc)
 807         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 808       else if (data->allocator == omp_large_cap_mem_alloc)
 809         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 810       if (memkind)
 811         {
 812           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 813           if (memkind_data->kinds[memkind])
 814             {
 815               void *kind = *memkind_data->kinds[memkind];
 816               memkind_data->memkind_free (kind, data->ptr);
 817               return;
 818             }
 819         }
 820 #endif
 821
 822       memspace = predefined_alloc_mapping[data->allocator];
 823     }
 824
 825   MEMSPACE_FREE (memspace, data->ptr, data->size);
 826 }
 827
 828 ialias (omp_free)
 829
 830 void
 831 GOMP_free (void *ptr, uintptr_t allocator)
 832 {
 833   return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
 834 }
 835
 836 void *
 837 omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
 838                     omp_allocator_handle_t allocator)
 839 {
 840   struct omp_allocator_data *allocator_data;
 841   size_t new_size, size_temp, new_alignment;
 842   void *ptr, *ret;
 843 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 844   enum gomp_numa_memkind_kind memkind;
 845 #endif
 846
 847   if (__builtin_expect (size == 0 || nmemb == 0, 0))
 848     return NULL;
 849
 850 retry:
 851   new_alignment = alignment;
 852   if (allocator == omp_null_allocator)
 853     {
 854       struct gomp_thread *thr = gomp_thread ();
 855       if (thr->ts.def_allocator == omp_null_allocator)
 856         thr->ts.def_allocator = gomp_def_allocator;
 857       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 858     }
 859
 860   if (allocator > omp_max_predefined_alloc)
 861     {
 862       allocator_data = (struct omp_allocator_data *) allocator;
 863       if (new_alignment < allocator_data->alignment)
 864         new_alignment = allocator_data->alignment;
 865 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 866       memkind = allocator_data->memkind;
 867 #endif
 868     }
 869   else
 870     {
 871       allocator_data = NULL;
 872       if (new_alignment < sizeof (void *))
 873         new_alignment = sizeof (void *);
 874 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 875       memkind = GOMP_MEMKIND_NONE;
 876 #endif
 877 #ifdef LIBGOMP_USE_MEMKIND
 878       if (allocator == omp_high_bw_mem_alloc)
 879         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 880       else if (allocator == omp_large_cap_mem_alloc)
 881         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 882       if (memkind)
 883         {
 884           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 885           if (!memkind_data->kinds[memkind])
 886             memkind = GOMP_MEMKIND_NONE;
 887         }
 888 #endif
 889     }
 890
 891   new_size = sizeof (struct omp_mem_header);
 892   if (new_alignment > sizeof (void *))
 893     new_size += new_alignment - sizeof (void *);
 894   if (__builtin_mul_overflow (size, nmemb, &size_temp))
 895     goto fail;
 896   if (__builtin_add_overflow (size_temp, new_size, &new_size))
 897     goto fail;
 898 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 899   if (allocator == omp_low_lat_mem_alloc)
 900     goto fail;
 901 #endif
 902
 903   if (__builtin_expect (allocator_data
 904                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 905     {
 906       uintptr_t used_pool_size;
 907       if (new_size > allocator_data->pool_size)
 908         goto fail;
 909 #ifdef HAVE_SYNC_BUILTINS
 910       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 911                                         MEMMODEL_RELAXED);
 912       do
 913         {
 914           uintptr_t new_pool_size;
 915           if (__builtin_add_overflow (used_pool_size, new_size,
 916                                       &new_pool_size)
 917               || new_pool_size > allocator_data->pool_size)
 918             goto fail;
 919           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 920                                            &used_pool_size, new_pool_size,
 921                                            true, MEMMODEL_RELAXED,
 922                                            MEMMODEL_RELAXED))
 923             break;
 924         }
 925       while (1);
 926 #else
 927       gomp_mutex_lock (&allocator_data->lock);
 928       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 929                                   &used_pool_size)
 930           || used_pool_size > allocator_data->pool_size)
 931         {
 932           gomp_mutex_unlock (&allocator_data->lock);
 933           goto fail;
 934         }
 935       allocator_data->used_pool_size = used_pool_size;
 936       gomp_mutex_unlock (&allocator_data->lock);
 937 #endif
 938 #ifdef LIBGOMP_USE_LIBNUMA
 939       if (memkind == GOMP_MEMKIND_LIBNUMA)
 940         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 941            memory that is initialized to zero.  */
 942         ptr = libnuma_data->numa_alloc_local (new_size);
 943 # ifdef LIBGOMP_USE_MEMKIND
 944       else
 945 # endif
 946 #endif
 947 #ifdef LIBGOMP_USE_MEMKIND
 948       if (memkind)
 949         {
 950           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 951           void *kind = *memkind_data->kinds[memkind];
 952           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 953         }
 954       else
 955 #endif
 956         ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size);
 957       if (ptr == NULL)
 958         {
 959 #ifdef HAVE_SYNC_BUILTINS
 960           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 961                               MEMMODEL_RELAXED);
 962 #else
 963           gomp_mutex_lock (&allocator_data->lock);
 964           allocator_data->used_pool_size -= new_size;
 965           gomp_mutex_unlock (&allocator_data->lock);
 966 #endif
 967           goto fail;
 968         }
 969     }
 970   else
 971     {
 972 #ifdef LIBGOMP_USE_LIBNUMA
 973       if (memkind == GOMP_MEMKIND_LIBNUMA)
 974         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 975            memory that is initialized to zero.  */
 976         ptr = libnuma_data->numa_alloc_local (new_size);
 977 # ifdef LIBGOMP_USE_MEMKIND
 978       else
 979 # endif
 980 #endif
 981 #ifdef LIBGOMP_USE_MEMKIND
 982       if (memkind)
 983         {
 984           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 985           void *kind = *memkind_data->kinds[memkind];
 986           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 987         }
 988       else
 989 #endif
 990         {
 991           omp_memspace_handle_t memspace;
 992           memspace = (allocator_data
 993                       ? allocator_data->memspace
 994                       : predefined_alloc_mapping[allocator]);
 995           ptr = MEMSPACE_CALLOC (memspace, new_size);
 996         }
 997       if (ptr == NULL)
 998         goto fail;
 999     }
1000
1001   if (new_alignment > sizeof (void *))
1002     ret = (void *) (((uintptr_t) ptr
1003                      + sizeof (struct omp_mem_header)
1004                      + new_alignment - sizeof (void *))
1005                     & ~(new_alignment - 1));
1006   else
1007     ret = (char *) ptr + sizeof (struct omp_mem_header);
1008   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
1009   ((struct omp_mem_header *) ret)[-1].size = new_size;
1010   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1011   return ret;
1012
1013 fail:;
1014   int fallback = (allocator_data
1015                   ? allocator_data->fallback
1016                   : allocator == omp_default_mem_alloc
1017                   ? omp_atv_null_fb
1018                   : omp_atv_default_mem_fb);
1019   switch (fallback)
1020     {
1021     case omp_atv_default_mem_fb:
1022       allocator = omp_default_mem_alloc;
1023       goto retry;
1024     case omp_atv_null_fb:
1025       break;
1026     default:
1027     case omp_atv_abort_fb:
1028       gomp_fatal ("Out of memory allocating %lu bytes",
1029                   (unsigned long) (size * nmemb));
1030     case omp_atv_allocator_fb:
1031       allocator = allocator_data->fb_data;
1032       goto retry;
1033     }
1034   return NULL;
1035 }
1036
1037 ialias (omp_aligned_calloc)
1038
1039 void *
1040 omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
1041 {
1042   return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
1043 }
1044
1045 void *
1046 omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
1047              omp_allocator_handle_t free_allocator)
1048 {
1049   struct omp_allocator_data *allocator_data, *free_allocator_data;
1050   size_t new_size, old_size, new_alignment, old_alignment;
1051   void *new_ptr, *ret;
1052   struct omp_mem_header *data;
1053 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1054   enum gomp_numa_memkind_kind memkind, free_memkind;
1055 #endif
1056
1057   if (__builtin_expect (ptr == NULL, 0))
1058     return ialias_call (omp_aligned_alloc) (1, size, allocator);
1059
1060   if (__builtin_expect (size == 0, 0))
1061     {
1062       ialias_call (omp_free) (ptr, free_allocator);
1063       return NULL;
1064     }
1065
1066   data = &((struct omp_mem_header *) ptr)[-1];
1067   free_allocator = data->allocator;
1068
1069 retry:
1070   new_alignment = sizeof (void *);
1071   if (allocator == omp_null_allocator)
1072     allocator = free_allocator;
1073
1074   if (allocator > omp_max_predefined_alloc)
1075     {
1076       allocator_data = (struct omp_allocator_data *) allocator;
1077       if (new_alignment < allocator_data->alignment)
1078         new_alignment = allocator_data->alignment;
1079 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1080       memkind = allocator_data->memkind;
1081 #endif
1082     }
1083   else
1084     {
1085       allocator_data = NULL;
1086 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1087       memkind = GOMP_MEMKIND_NONE;
1088 #endif
1089 #ifdef LIBGOMP_USE_MEMKIND
1090       if (allocator == omp_high_bw_mem_alloc)
1091         memkind = GOMP_MEMKIND_HBW_PREFERRED;
1092       else if (allocator == omp_large_cap_mem_alloc)
1093         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1094       if (memkind)
1095         {
1096           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1097           if (!memkind_data->kinds[memkind])
1098             memkind = GOMP_MEMKIND_NONE;
1099         }
1100 #endif
1101     }
1102   if (free_allocator > omp_max_predefined_alloc)
1103     {
1104       free_allocator_data = (struct omp_allocator_data *) free_allocator;
1105 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1106       free_memkind = free_allocator_data->memkind;
1107 #endif
1108     }
1109   else
1110     {
1111       free_allocator_data = NULL;
1112 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1113       free_memkind = GOMP_MEMKIND_NONE;
1114 #endif
1115 #ifdef LIBGOMP_USE_MEMKIND
1116       if (free_allocator == omp_high_bw_mem_alloc)
1117         free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1118       else if (free_allocator == omp_large_cap_mem_alloc)
1119         free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1120       if (free_memkind)
1121         {
1122           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1123           if (!memkind_data->kinds[free_memkind])
1124             free_memkind = GOMP_MEMKIND_NONE;
1125         }
1126 #endif
1127     }
1128   old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1129
1130   new_size = sizeof (struct omp_mem_header);
1131   if (new_alignment > sizeof (void *))
1132     new_size += new_alignment - sizeof (void *);
1133   if (__builtin_add_overflow (size, new_size, &new_size))
1134     goto fail;
1135   old_size = data->size;
1136 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1137   if (allocator == omp_low_lat_mem_alloc)
1138     goto fail;
1139 #endif
1140
1141   if (__builtin_expect (allocator_data
1142                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1143     {
1144       uintptr_t used_pool_size;
1145       size_t prev_size = 0;
1146       /* Check if we can use realloc.  Don't use it if extra alignment
1147          was used previously or newly, because realloc might return a pointer
1148          with different alignment and then we'd need to memmove the data
1149          again.  */
1150       if (free_allocator_data
1151           && free_allocator_data == allocator_data
1152           && new_alignment == sizeof (void *)
1153           && old_alignment == sizeof (struct omp_mem_header))
1154         prev_size = old_size;
1155       if (new_size > prev_size
1156           && new_size - prev_size > allocator_data->pool_size)
1157         goto fail;
1158 #ifdef HAVE_SYNC_BUILTINS
1159       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1160                                         MEMMODEL_RELAXED);
1161       do
1162         {
1163           uintptr_t new_pool_size;
1164           if (new_size > prev_size)
1165             {
1166               if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1167                                           &new_pool_size)
1168                   || new_pool_size > allocator_data->pool_size)
1169                 goto fail;
1170             }
1171           else
1172             new_pool_size = used_pool_size + new_size - prev_size;
1173           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1174                                            &used_pool_size, new_pool_size,
1175                                            true, MEMMODEL_RELAXED,
1176                                            MEMMODEL_RELAXED))
1177             break;
1178         }
1179       while (1);
1180 #else
1181       gomp_mutex_lock (&allocator_data->lock);
1182       if (new_size > prev_size)
1183         {
1184           if (__builtin_add_overflow (allocator_data->used_pool_size,
1185                                       new_size - prev_size,
1186                                       &used_pool_size)
1187               || used_pool_size > allocator_data->pool_size)
1188             {
1189               gomp_mutex_unlock (&allocator_data->lock);
1190               goto fail;
1191             }
1192         }
1193       else
1194         used_pool_size = (allocator_data->used_pool_size
1195                           + new_size - prev_size);
1196       allocator_data->used_pool_size = used_pool_size;
1197       gomp_mutex_unlock (&allocator_data->lock);
1198 #endif
1199 #ifdef LIBGOMP_USE_LIBNUMA
1200       if (memkind == GOMP_MEMKIND_LIBNUMA)
1201         {
1202           if (prev_size)
1203             new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1204                                                   new_size);
1205           else
1206             new_ptr = libnuma_data->numa_alloc_local (new_size);
1207         }
1208 # ifdef LIBGOMP_USE_MEMKIND
1209       else
1210 # endif
1211 #endif
1212 #ifdef LIBGOMP_USE_MEMKIND
1213       if (memkind)
1214         {
1215           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1216           void *kind = *memkind_data->kinds[memkind];
1217           if (prev_size)
1218             new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1219                                                      new_size);
1220           else
1221             new_ptr = memkind_data->memkind_malloc (kind, new_size);
1222         }
1223       else
1224 #endif
1225       if (prev_size)
1226         new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
1227                                     data->size, new_size);
1228       else
1229         new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
1230       if (new_ptr == NULL)
1231         {
1232 #ifdef HAVE_SYNC_BUILTINS
1233           __atomic_add_fetch (&allocator_data->used_pool_size,
1234                               prev_size - new_size,
1235                               MEMMODEL_RELAXED);
1236 #else
1237           gomp_mutex_lock (&allocator_data->lock);
1238           allocator_data->used_pool_size -= new_size - prev_size;
1239           gomp_mutex_unlock (&allocator_data->lock);
1240 #endif
1241           goto fail;
1242         }
1243       else if (prev_size)
1244         {
1245           ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1246           ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1247           ((struct omp_mem_header *) ret)[-1].size = new_size;
1248           ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1249           return ret;
1250         }
1251     }
1252   else if (new_alignment == sizeof (void *)
1253            && old_alignment == sizeof (struct omp_mem_header)
1254 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1255            && memkind == free_memkind
1256 #endif
1257            && (free_allocator_data == NULL
1258                || free_allocator_data->pool_size == ~(uintptr_t) 0))
1259     {
1260 #ifdef LIBGOMP_USE_LIBNUMA
1261       if (memkind == GOMP_MEMKIND_LIBNUMA)
1262         new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1263 # ifdef LIBGOMP_USE_MEMKIND
1264       else
1265 # endif
1266 #endif
1267 #ifdef LIBGOMP_USE_MEMKIND
1268       if (memkind)
1269         {
1270           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1271           void *kind = *memkind_data->kinds[memkind];
1272           new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1273                                                    new_size);
1274         }
1275       else
1276 #endif
1277         {
1278           omp_memspace_handle_t memspace;
1279           memspace = (allocator_data
1280                       ? allocator_data->memspace
1281                       : predefined_alloc_mapping[allocator]);
1282           new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size);
1283         }
1284       if (new_ptr == NULL)
1285         goto fail;
1286       ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1287       ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1288       ((struct omp_mem_header *) ret)[-1].size = new_size;
1289       ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1290       return ret;
1291     }
1292   else
1293     {
1294 #ifdef LIBGOMP_USE_LIBNUMA
1295       if (memkind == GOMP_MEMKIND_LIBNUMA)
1296         new_ptr = libnuma_data->numa_alloc_local (new_size);
1297 # ifdef LIBGOMP_USE_MEMKIND
1298       else
1299 # endif
1300 #endif
1301 #ifdef LIBGOMP_USE_MEMKIND
1302       if (memkind)
1303         {
1304           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1305           void *kind = *memkind_data->kinds[memkind];
1306           new_ptr = memkind_data->memkind_malloc (kind, new_size);
1307         }
1308       else
1309 #endif
1310         {
1311           omp_memspace_handle_t memspace;
1312           memspace = (allocator_data
1313                       ? allocator_data->memspace
1314                       : predefined_alloc_mapping[allocator]);
1315           new_ptr = MEMSPACE_ALLOC (memspace, new_size);
1316         }
1317       if (new_ptr == NULL)
1318         goto fail;
1319     }
1320
1321   if (new_alignment > sizeof (void *))
1322     ret = (void *) (((uintptr_t) new_ptr
1323                      + sizeof (struct omp_mem_header)
1324                      + new_alignment - sizeof (void *))
1325                     & ~(new_alignment - 1));
1326   else
1327     ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1328   ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1329   ((struct omp_mem_header *) ret)[-1].size = new_size;
1330   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1331   if (old_size - old_alignment < size)
1332     size = old_size - old_alignment;
1333   memcpy (ret, ptr, size);
1334   if (__builtin_expect (free_allocator_data
1335                         && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1336     {
1337 #ifdef HAVE_SYNC_BUILTINS
1338       __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1339                           MEMMODEL_RELAXED);
1340 #else
1341       gomp_mutex_lock (&free_allocator_data->lock);
1342       free_allocator_data->used_pool_size -= data->size;
1343       gomp_mutex_unlock (&free_allocator_data->lock);
1344 #endif
1345     }
1346 #ifdef LIBGOMP_USE_LIBNUMA
1347   if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1348     {
1349       libnuma_data->numa_free (data->ptr, data->size);
1350       return ret;
1351     }
1352 # ifdef LIBGOMP_USE_MEMKIND
1353   else
1354 # endif
1355 #endif
1356 #ifdef LIBGOMP_USE_MEMKIND
1357   if (free_memkind)
1358     {
1359       struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1360       void *kind = *memkind_data->kinds[free_memkind];
1361       memkind_data->memkind_free (kind, data->ptr);
1362       return ret;
1363     }
1364 #endif
1365   {
1366     omp_memspace_handle_t was_memspace;
1367     was_memspace = (free_allocator_data
1368                     ? free_allocator_data->memspace
1369                     : predefined_alloc_mapping[free_allocator]);
1370     MEMSPACE_FREE (was_memspace, data->ptr, data->size);
1371   }
1372   return ret;
1373
1374 fail:;
1375   int fallback = (allocator_data
1376                   ? allocator_data->fallback
1377                   : allocator == omp_default_mem_alloc
1378                   ? omp_atv_null_fb
1379                   : omp_atv_default_mem_fb);
1380   switch (fallback)
1381     {
1382     case omp_atv_default_mem_fb:
1383       allocator = omp_default_mem_alloc;
1384       goto retry;
1385     case omp_atv_null_fb:
1386       break;
1387     default:
1388     case omp_atv_abort_fb:
1389       gomp_fatal ("Out of memory allocating %lu bytes",
1390                   (unsigned long) size);
1391     case omp_atv_allocator_fb:
1392       allocator = allocator_data->fb_data;
1393       goto retry;
1394     }
1395   return NULL;
1396 }