libgomp/allocator.c

   1 /* Copyright (C) 2020-2024 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
  35 #include <dlfcn.h>
  36 #endif
  37
  38 /* Keeping track whether a Fortran scalar allocatable/pointer has been
  39    allocated via 'omp allocators'/'omp allocate'.  */
  40
  41 struct fort_alloc_splay_tree_key_s {
  42   void *ptr;
  43 };
  44
  45 typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node;
  46 typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree;
  47 typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key;
  48
  49 static inline int
  50 fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y)
  51 {
  52   if (x->ptr < y->ptr)
  53     return -1;
  54   if (x->ptr > y->ptr)
  55     return 1;
  56   return 0;
  57 }
  58 #define splay_tree_prefix fort_alloc
  59 #define splay_tree_static
  60 #include "splay-tree.h"
  61
  62 #define splay_tree_prefix fort_alloc
  63 #define splay_tree_static
  64 #define splay_tree_c
  65 #include "splay-tree.h"
  66
  67 static struct fort_alloc_splay_tree_s fort_alloc_scalars;
  68
  69 /* Add pointer as being alloced by GOMP_alloc.  */
  70 void
  71 GOMP_add_alloc (void *ptr)
  72 {
  73   if (ptr == NULL)
  74     return;
  75   fort_alloc_splay_tree_node item;
  76   item = gomp_malloc (sizeof (struct splay_tree_node_s));
  77   item->key.ptr = ptr;
  78   item->left = NULL;
  79   item->right = NULL;
  80   fort_alloc_splay_tree_insert (&fort_alloc_scalars, item);
  81 }
  82
  83 /* Remove pointer, either called by FREE or by REALLOC,
  84    either of them can change the allocation status.  */
  85 bool
  86 GOMP_is_alloc (void *ptr)
  87 {
  88   struct fort_alloc_splay_tree_key_s needle;
  89   fort_alloc_splay_tree_node n;
  90   needle.ptr = ptr;
  91   n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle);
  92   if (n)
  93     {
  94       fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key);
  95       free (n);
  96     }
  97   return n != NULL;
  98 }
  99
 100
 101 #define omp_max_predefined_alloc omp_thread_mem_alloc
 102
 103 /* These macros may be overridden in config/<target>/allocator.c.
 104    The defaults (no override) are to return NULL for pinned memory requests
 105    and pass through to the regular OS calls otherwise.
 106    The following definitions (ab)use comma operators to avoid unused
 107    variable errors.  */
 108 #ifndef MEMSPACE_ALLOC
 109 #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
 110   (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
 111 #endif
 112 #ifndef MEMSPACE_CALLOC
 113 #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
 114   (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
 115 #endif
 116 #ifndef MEMSPACE_REALLOC
 117 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
 118    ((PIN) || (OLDPIN) ? NULL \
 119    : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
 120 #endif
 121 #ifndef MEMSPACE_FREE
 122 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
 123   if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
 124 #endif
 125 #ifndef MEMSPACE_VALIDATE
 126 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
 127   (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1))
 128 #endif
 129
 130 /* Map the predefined allocators to the correct memory space.
 131    The index to this table is the omp_allocator_handle_t enum value.
 132    When the user calls omp_alloc with a predefined allocator this
 133    table determines what memory they get.  */
 134 static const omp_memspace_handle_t predefined_alloc_mapping[] = {
 135   omp_default_mem_space,   /* omp_null_allocator doesn't actually use this. */
 136   omp_default_mem_space,   /* omp_default_mem_alloc. */
 137   omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
 138   omp_const_mem_space,     /* omp_const_mem_alloc. */
 139   omp_high_bw_mem_space,   /* omp_high_bw_mem_alloc. */
 140   omp_low_lat_mem_space,   /* omp_low_lat_mem_alloc. */
 141   omp_low_lat_mem_space,   /* omp_cgroup_mem_alloc (implementation defined). */
 142   omp_low_lat_mem_space,   /* omp_pteam_mem_alloc (implementation defined). */
 143   omp_low_lat_mem_space,   /* omp_thread_mem_alloc (implementation defined). */
 144 };
 145
 146 #define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
 147 _Static_assert (ARRAY_SIZE (predefined_alloc_mapping)
 148                 == omp_max_predefined_alloc + 1,
 149                 "predefined_alloc_mapping must match omp_memspace_handle_t");
 150
 151 enum gomp_numa_memkind_kind
 152 {
 153   GOMP_MEMKIND_NONE = 0,
 154 #define GOMP_MEMKIND_KINDS \
 155   GOMP_MEMKIND_KIND (HBW_INTERLEAVE),           \
 156   GOMP_MEMKIND_KIND (HBW_PREFERRED),            \
 157   GOMP_MEMKIND_KIND (DAX_KMEM_ALL),             \
 158   GOMP_MEMKIND_KIND (DAX_KMEM),                 \
 159   GOMP_MEMKIND_KIND (INTERLEAVE),               \
 160   GOMP_MEMKIND_KIND (DEFAULT)
 161 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
 162   GOMP_MEMKIND_KINDS,
 163 #undef GOMP_MEMKIND_KIND
 164   GOMP_MEMKIND_COUNT,
 165   GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
 166 };
 167
 168 struct omp_allocator_data
 169 {
 170   omp_memspace_handle_t memspace;
 171   omp_uintptr_t alignment;
 172   omp_uintptr_t pool_size;
 173   omp_uintptr_t used_pool_size;
 174   omp_allocator_handle_t fb_data;
 175   unsigned int sync_hint : 8;
 176   unsigned int access : 8;
 177   unsigned int fallback : 8;
 178   unsigned int pinned : 1;
 179   unsigned int partition : 7;
 180 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 181   unsigned int memkind : 8;
 182 #endif
 183 #ifndef HAVE_SYNC_BUILTINS
 184   gomp_mutex_t lock;
 185 #endif
 186 };
 187
 188 struct omp_mem_header
 189 {
 190   void *ptr;
 191   size_t size;
 192   omp_allocator_handle_t allocator;
 193   void *pad;
 194 };
 195
 196 struct gomp_libnuma_data
 197 {
 198   void *numa_handle;
 199   void *(*numa_alloc_local) (size_t);
 200   void *(*numa_realloc) (void *, size_t, size_t);
 201   void (*numa_free) (void *, size_t);
 202 };
 203
 204 struct gomp_memkind_data
 205 {
 206   void *memkind_handle;
 207   void *(*memkind_malloc) (void *, size_t);
 208   void *(*memkind_calloc) (void *, size_t, size_t);
 209   void *(*memkind_realloc) (void *, void *, size_t);
 210   void (*memkind_free) (void *, void *);
 211   int (*memkind_check_available) (void *);
 212   void **kinds[GOMP_MEMKIND_COUNT];
 213 };
 214
 215 #ifdef LIBGOMP_USE_LIBNUMA
 216 static struct gomp_libnuma_data *libnuma_data;
 217 static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
 218
 219 static void
 220 gomp_init_libnuma (void)
 221 {
 222   void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
 223   struct gomp_libnuma_data *data;
 224
 225   data = calloc (1, sizeof (struct gomp_libnuma_data));
 226   if (data == NULL)
 227     {
 228       if (handle)
 229         dlclose (handle);
 230       return;
 231     }
 232   if (handle)
 233     {
 234       int (*numa_available) (void);
 235       numa_available
 236         = (__typeof (numa_available)) dlsym (handle, "numa_available");
 237       if (!numa_available || numa_available () != 0)
 238         {
 239           dlclose (handle);
 240           handle = NULL;
 241         }
 242     }
 243   if (!handle)
 244     {
 245       __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 246       return;
 247     }
 248   data->numa_handle = handle;
 249   data->numa_alloc_local
 250     = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
 251   data->numa_realloc
 252     = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
 253   data->numa_free
 254     = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
 255   __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 256 }
 257
 258 static struct gomp_libnuma_data *
 259 gomp_get_libnuma (void)
 260 {
 261   struct gomp_libnuma_data *data
 262     = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 263   if (data)
 264     return data;
 265   pthread_once (&libnuma_data_once, gomp_init_libnuma);
 266   return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 267 }
 268 #endif
 269
 270 #ifdef LIBGOMP_USE_MEMKIND
 271 static struct gomp_memkind_data *memkind_data;
 272 static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
 273
 274 static void
 275 gomp_init_memkind (void)
 276 {
 277   void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
 278   struct gomp_memkind_data *data;
 279   int i;
 280   static const char *kinds[] = {
 281     NULL,
 282 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
 283     GOMP_MEMKIND_KINDS
 284 #undef GOMP_MEMKIND_KIND
 285   };
 286
 287   data = calloc (1, sizeof (struct gomp_memkind_data));
 288   if (data == NULL)
 289     {
 290       if (handle)
 291         dlclose (handle);
 292       return;
 293     }
 294   if (!handle)
 295     {
 296       __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 297       return;
 298     }
 299   data->memkind_handle = handle;
 300   data->memkind_malloc
 301     = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
 302   data->memkind_calloc
 303     = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
 304   data->memkind_realloc
 305     = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
 306   data->memkind_free
 307     = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
 308   data->memkind_check_available
 309     = (__typeof (data->memkind_check_available))
 310       dlsym (handle, "memkind_check_available");
 311   if (data->memkind_malloc
 312       && data->memkind_calloc
 313       && data->memkind_realloc
 314       && data->memkind_free
 315       && data->memkind_check_available)
 316     for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
 317       {
 318         data->kinds[i] = (void **) dlsym (handle, kinds[i]);
 319         if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
 320           data->kinds[i] = NULL;
 321       }
 322   __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 323 }
 324
 325 static struct gomp_memkind_data *
 326 gomp_get_memkind (void)
 327 {
 328   struct gomp_memkind_data *data
 329     = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 330   if (data)
 331     return data;
 332   pthread_once (&memkind_data_once, gomp_init_memkind);
 333   return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 334 }
 335 #endif
 336
 337 omp_allocator_handle_t
 338 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
 339                     const omp_alloctrait_t traits[])
 340 {
 341   struct omp_allocator_data data
 342     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
 343         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
 344 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 345         GOMP_MEMKIND_NONE
 346 #endif
 347       };
 348   struct omp_allocator_data *ret;
 349   int i;
 350
 351   if (memspace > omp_low_lat_mem_space)
 352     return omp_null_allocator;
 353   for (i = 0; i < ntraits; i++)
 354     switch (traits[i].key)
 355       {
 356       case omp_atk_sync_hint:
 357         switch (traits[i].value)
 358           {
 359           case omp_atv_default:
 360             data.sync_hint = omp_atv_contended;
 361             break;
 362           case omp_atv_contended:
 363           case omp_atv_uncontended:
 364           case omp_atv_serialized:
 365           case omp_atv_private:
 366             data.sync_hint = traits[i].value;
 367             break;
 368           default:
 369             return omp_null_allocator;
 370           }
 371         break;
 372       case omp_atk_alignment:
 373         if (traits[i].value == omp_atv_default)
 374           {
 375             data.alignment = 1;
 376             break;
 377           }
 378         if ((traits[i].value & (traits[i].value - 1)) != 0
 379             || !traits[i].value)
 380           return omp_null_allocator;
 381         data.alignment = traits[i].value;
 382         break;
 383       case omp_atk_access:
 384         switch (traits[i].value)
 385           {
 386           case omp_atv_default:
 387             data.access = omp_atv_all;
 388             break;
 389           case omp_atv_all:
 390           case omp_atv_cgroup:
 391           case omp_atv_pteam:
 392           case omp_atv_thread:
 393             data.access = traits[i].value;
 394             break;
 395           default:
 396             return omp_null_allocator;
 397           }
 398         break;
 399       case omp_atk_pool_size:
 400         if (traits[i].value == omp_atv_default)
 401           data.pool_size = ~(uintptr_t) 0;
 402         else
 403           data.pool_size = traits[i].value;
 404         break;
 405       case omp_atk_fallback:
 406         switch (traits[i].value)
 407           {
 408           case omp_atv_default:
 409             data.fallback = omp_atv_default_mem_fb;
 410             break;
 411           case omp_atv_default_mem_fb:
 412           case omp_atv_null_fb:
 413           case omp_atv_abort_fb:
 414           case omp_atv_allocator_fb:
 415             data.fallback = traits[i].value;
 416             break;
 417           default:
 418             return omp_null_allocator;
 419           }
 420         break;
 421       case omp_atk_fb_data:
 422         data.fb_data = traits[i].value;
 423         break;
 424       case omp_atk_pinned:
 425         switch (traits[i].value)
 426           {
 427           case omp_atv_default:
 428           case omp_atv_false:
 429             data.pinned = omp_atv_false;
 430             break;
 431           case omp_atv_true:
 432             data.pinned = omp_atv_true;
 433             break;
 434           default:
 435             return omp_null_allocator;
 436           }
 437         break;
 438       case omp_atk_partition:
 439         switch (traits[i].value)
 440           {
 441           case omp_atv_default:
 442             data.partition = omp_atv_environment;
 443             break;
 444           case omp_atv_environment:
 445           case omp_atv_nearest:
 446           case omp_atv_blocked:
 447           case omp_atv_interleaved:
 448             data.partition = traits[i].value;
 449             break;
 450           default:
 451             return omp_null_allocator;
 452           }
 453         break;
 454       default:
 455         return omp_null_allocator;
 456       }
 457
 458   if (data.alignment < sizeof (void *))
 459     data.alignment = sizeof (void *);
 460
 461   switch (memspace)
 462     {
 463 #ifdef LIBGOMP_USE_MEMKIND
 464     case omp_high_bw_mem_space:
 465       struct gomp_memkind_data *memkind_data;
 466       memkind_data = gomp_get_memkind ();
 467       if (data.partition == omp_atv_interleaved
 468           && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
 469         {
 470           data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
 471           break;
 472         }
 473       else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
 474         {
 475           data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
 476           break;
 477         }
 478       break;
 479     case omp_large_cap_mem_space:
 480       memkind_data = gomp_get_memkind ();
 481       if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
 482         data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 483       else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
 484         data.memkind = GOMP_MEMKIND_DAX_KMEM;
 485       break;
 486 #endif
 487     default:
 488 #ifdef LIBGOMP_USE_MEMKIND
 489       if (data.partition == omp_atv_interleaved)
 490         {
 491           memkind_data = gomp_get_memkind ();
 492           if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
 493             data.memkind = GOMP_MEMKIND_INTERLEAVE;
 494         }
 495 #endif
 496       break;
 497     }
 498
 499 #ifdef LIBGOMP_USE_LIBNUMA
 500   if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
 501     {
 502       libnuma_data = gomp_get_libnuma ();
 503       if (libnuma_data->numa_alloc_local != NULL)
 504         data.memkind = GOMP_MEMKIND_LIBNUMA;
 505     }
 506 #endif
 507
 508   /* Reject unsupported memory spaces.  */
 509   if (!MEMSPACE_VALIDATE (data.memspace, data.access, data.pinned))
 510     return omp_null_allocator;
 511
 512   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 513   *ret = data;
 514 #ifndef HAVE_SYNC_BUILTINS
 515   gomp_mutex_init (&ret->lock);
 516 #endif
 517   return (omp_allocator_handle_t) ret;
 518 }
 519
 520 void
 521 omp_destroy_allocator (omp_allocator_handle_t allocator)
 522 {
 523   if (allocator != omp_null_allocator)
 524     {
 525 #ifndef HAVE_SYNC_BUILTINS
 526       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 527 #endif
 528       free ((void *) allocator);
 529     }
 530 }
 531
 532 ialias (omp_init_allocator)
 533 ialias (omp_destroy_allocator)
 534
 535 void *
 536 omp_aligned_alloc (size_t alignment, size_t size,
 537                    omp_allocator_handle_t allocator)
 538 {
 539   struct omp_allocator_data *allocator_data;
 540   size_t new_size, new_alignment;
 541   void *ptr, *ret;
 542 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 543   enum gomp_numa_memkind_kind memkind;
 544 #endif
 545
 546   if (__builtin_expect (size == 0, 0))
 547     return NULL;
 548
 549 retry:
 550   new_alignment = alignment;
 551   if (allocator == omp_null_allocator)
 552     {
 553       struct gomp_thread *thr = gomp_thread ();
 554       if (thr->ts.def_allocator == omp_null_allocator)
 555         thr->ts.def_allocator = gomp_def_allocator;
 556       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 557     }
 558
 559   if (allocator > omp_max_predefined_alloc)
 560     {
 561       allocator_data = (struct omp_allocator_data *) allocator;
 562       if (new_alignment < allocator_data->alignment)
 563         new_alignment = allocator_data->alignment;
 564 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 565       memkind = allocator_data->memkind;
 566 #endif
 567     }
 568   else
 569     {
 570       allocator_data = NULL;
 571       if (new_alignment < sizeof (void *))
 572         new_alignment = sizeof (void *);
 573 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 574       memkind = GOMP_MEMKIND_NONE;
 575 #endif
 576 #ifdef LIBGOMP_USE_MEMKIND
 577       if (allocator == omp_high_bw_mem_alloc)
 578         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 579       else if (allocator == omp_large_cap_mem_alloc)
 580         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 581       if (memkind)
 582         {
 583           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 584           if (!memkind_data->kinds[memkind])
 585             memkind = GOMP_MEMKIND_NONE;
 586         }
 587 #endif
 588     }
 589
 590   new_size = sizeof (struct omp_mem_header);
 591   if (new_alignment > sizeof (void *))
 592     new_size += new_alignment - sizeof (void *);
 593   if (__builtin_add_overflow (size, new_size, &new_size))
 594     goto fail;
 595 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 596   if (allocator == omp_low_lat_mem_alloc)
 597     goto fail;
 598 #endif
 599
 600   if (__builtin_expect (allocator_data
 601                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 602     {
 603       uintptr_t used_pool_size;
 604       if (new_size > allocator_data->pool_size)
 605         goto fail;
 606 #ifdef HAVE_SYNC_BUILTINS
 607       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 608                                         MEMMODEL_RELAXED);
 609       do
 610         {
 611           uintptr_t new_pool_size;
 612           if (__builtin_add_overflow (used_pool_size, new_size,
 613                                       &new_pool_size)
 614               || new_pool_size > allocator_data->pool_size)
 615             goto fail;
 616           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 617                                            &used_pool_size, new_pool_size,
 618                                            true, MEMMODEL_RELAXED,
 619                                            MEMMODEL_RELAXED))
 620             break;
 621         }
 622       while (1);
 623 #else
 624       gomp_mutex_lock (&allocator_data->lock);
 625       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 626                                   &used_pool_size)
 627           || used_pool_size > allocator_data->pool_size)
 628         {
 629           gomp_mutex_unlock (&allocator_data->lock);
 630           goto fail;
 631         }
 632       allocator_data->used_pool_size = used_pool_size;
 633       gomp_mutex_unlock (&allocator_data->lock);
 634 #endif
 635 #ifdef LIBGOMP_USE_LIBNUMA
 636       if (memkind == GOMP_MEMKIND_LIBNUMA)
 637         ptr = libnuma_data->numa_alloc_local (new_size);
 638 # ifdef LIBGOMP_USE_MEMKIND
 639       else
 640 # endif
 641 #endif
 642 #ifdef LIBGOMP_USE_MEMKIND
 643       if (memkind)
 644         {
 645           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 646           void *kind = *memkind_data->kinds[memkind];
 647           ptr = memkind_data->memkind_malloc (kind, new_size);
 648         }
 649       else
 650 #endif
 651         ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
 652                               allocator_data->pinned);
 653       if (ptr == NULL)
 654         {
 655 #ifdef HAVE_SYNC_BUILTINS
 656           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 657                               MEMMODEL_RELAXED);
 658 #else
 659           gomp_mutex_lock (&allocator_data->lock);
 660           allocator_data->used_pool_size -= new_size;
 661           gomp_mutex_unlock (&allocator_data->lock);
 662 #endif
 663           goto fail;
 664         }
 665     }
 666   else
 667     {
 668 #ifdef LIBGOMP_USE_LIBNUMA
 669       if (memkind == GOMP_MEMKIND_LIBNUMA)
 670         ptr = libnuma_data->numa_alloc_local (new_size);
 671 # ifdef LIBGOMP_USE_MEMKIND
 672       else
 673 # endif
 674 #endif
 675 #ifdef LIBGOMP_USE_MEMKIND
 676       if (memkind)
 677         {
 678           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 679           void *kind = *memkind_data->kinds[memkind];
 680           ptr = memkind_data->memkind_malloc (kind, new_size);
 681         }
 682       else
 683 #endif
 684         {
 685           omp_memspace_handle_t memspace;
 686           memspace = (allocator_data
 687                       ? allocator_data->memspace
 688                       : predefined_alloc_mapping[allocator]);
 689           ptr = MEMSPACE_ALLOC (memspace, new_size,
 690                                 allocator_data && allocator_data->pinned);
 691         }
 692       if (ptr == NULL)
 693         goto fail;
 694     }
 695
 696   if (new_alignment > sizeof (void *))
 697     ret = (void *) (((uintptr_t) ptr
 698                      + sizeof (struct omp_mem_header)
 699                      + new_alignment - sizeof (void *))
 700                     & ~(new_alignment - 1));
 701   else
 702     ret = (char *) ptr + sizeof (struct omp_mem_header);
 703   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 704   ((struct omp_mem_header *) ret)[-1].size = new_size;
 705   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 706   return ret;
 707
 708 fail:;
 709   int fallback = (allocator_data
 710                   ? allocator_data->fallback
 711                   : allocator == omp_default_mem_alloc
 712                   ? omp_atv_null_fb
 713                   : omp_atv_default_mem_fb);
 714   switch (fallback)
 715     {
 716     case omp_atv_default_mem_fb:
 717       allocator = omp_default_mem_alloc;
 718       goto retry;
 719     case omp_atv_null_fb:
 720       break;
 721     default:
 722     case omp_atv_abort_fb:
 723       gomp_fatal ("Out of memory allocating %lu bytes",
 724                   (unsigned long) size);
 725     case omp_atv_allocator_fb:
 726       allocator = allocator_data->fb_data;
 727       goto retry;
 728     }
 729   return NULL;
 730 }
 731
 732 ialias (omp_aligned_alloc)
 733
 734 void *
 735 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 736 {
 737   return ialias_call (omp_aligned_alloc) (1, size, allocator);
 738 }
 739
 740 /* Like omp_aligned_alloc, but apply on top of that:
 741    "For allocations that arise from this ... the null_fb value of the
 742    fallback allocator trait behaves as if the abort_fb had been specified."  */
 743
 744 void *
 745 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 746 {
 747   void *ret
 748     = ialias_call (omp_aligned_alloc) (alignment, size,
 749                                        (omp_allocator_handle_t) allocator);
 750   if (__builtin_expect (ret == NULL, 0) && size)
 751     gomp_fatal ("Out of memory allocating %lu bytes",
 752                 (unsigned long) size);
 753   return ret;
 754 }
 755
 756 void
 757 omp_free (void *ptr, omp_allocator_handle_t allocator)
 758 {
 759   struct omp_mem_header *data;
 760   omp_memspace_handle_t memspace = omp_default_mem_space;
 761   int pinned = false;
 762
 763   if (ptr == NULL)
 764     return;
 765   (void) allocator;
 766   data = &((struct omp_mem_header *) ptr)[-1];
 767   if (data->allocator > omp_max_predefined_alloc)
 768     {
 769       struct omp_allocator_data *allocator_data
 770         = (struct omp_allocator_data *) (data->allocator);
 771       if (allocator_data->pool_size < ~(uintptr_t) 0)
 772         {
 773 #ifdef HAVE_SYNC_BUILTINS
 774           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 775                               MEMMODEL_RELAXED);
 776 #else
 777           gomp_mutex_lock (&allocator_data->lock);
 778           allocator_data->used_pool_size -= data->size;
 779           gomp_mutex_unlock (&allocator_data->lock);
 780 #endif
 781         }
 782 #ifdef LIBGOMP_USE_LIBNUMA
 783       if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
 784         {
 785           libnuma_data->numa_free (data->ptr, data->size);
 786           return;
 787         }
 788 # ifdef LIBGOMP_USE_MEMKIND
 789       else
 790 # endif
 791 #endif
 792 #ifdef LIBGOMP_USE_MEMKIND
 793       if (allocator_data->memkind)
 794         {
 795           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 796           void *kind = *memkind_data->kinds[allocator_data->memkind];
 797           memkind_data->memkind_free (kind, data->ptr);
 798           return;
 799         }
 800 #endif
 801
 802       memspace = allocator_data->memspace;
 803       pinned = allocator_data->pinned;
 804     }
 805   else
 806     {
 807 #ifdef LIBGOMP_USE_MEMKIND
 808       enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
 809       if (data->allocator == omp_high_bw_mem_alloc)
 810         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 811       else if (data->allocator == omp_large_cap_mem_alloc)
 812         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 813       if (memkind)
 814         {
 815           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 816           if (memkind_data->kinds[memkind])
 817             {
 818               void *kind = *memkind_data->kinds[memkind];
 819               memkind_data->memkind_free (kind, data->ptr);
 820               return;
 821             }
 822         }
 823 #endif
 824
 825       memspace = predefined_alloc_mapping[data->allocator];
 826     }
 827
 828   MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
 829 }
 830
 831 ialias (omp_free)
 832
 833 void
 834 GOMP_free (void *ptr, uintptr_t allocator)
 835 {
 836   return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
 837 }
 838
 839 void *
 840 omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
 841                     omp_allocator_handle_t allocator)
 842 {
 843   struct omp_allocator_data *allocator_data;
 844   size_t new_size, size_temp, new_alignment;
 845   void *ptr, *ret;
 846 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 847   enum gomp_numa_memkind_kind memkind;
 848 #endif
 849
 850   if (__builtin_expect (size == 0 || nmemb == 0, 0))
 851     return NULL;
 852
 853 retry:
 854   new_alignment = alignment;
 855   if (allocator == omp_null_allocator)
 856     {
 857       struct gomp_thread *thr = gomp_thread ();
 858       if (thr->ts.def_allocator == omp_null_allocator)
 859         thr->ts.def_allocator = gomp_def_allocator;
 860       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 861     }
 862
 863   if (allocator > omp_max_predefined_alloc)
 864     {
 865       allocator_data = (struct omp_allocator_data *) allocator;
 866       if (new_alignment < allocator_data->alignment)
 867         new_alignment = allocator_data->alignment;
 868 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 869       memkind = allocator_data->memkind;
 870 #endif
 871     }
 872   else
 873     {
 874       allocator_data = NULL;
 875       if (new_alignment < sizeof (void *))
 876         new_alignment = sizeof (void *);
 877 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 878       memkind = GOMP_MEMKIND_NONE;
 879 #endif
 880 #ifdef LIBGOMP_USE_MEMKIND
 881       if (allocator == omp_high_bw_mem_alloc)
 882         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 883       else if (allocator == omp_large_cap_mem_alloc)
 884         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 885       if (memkind)
 886         {
 887           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 888           if (!memkind_data->kinds[memkind])
 889             memkind = GOMP_MEMKIND_NONE;
 890         }
 891 #endif
 892     }
 893
 894   new_size = sizeof (struct omp_mem_header);
 895   if (new_alignment > sizeof (void *))
 896     new_size += new_alignment - sizeof (void *);
 897   if (__builtin_mul_overflow (size, nmemb, &size_temp))
 898     goto fail;
 899   if (__builtin_add_overflow (size_temp, new_size, &new_size))
 900     goto fail;
 901 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 902   if (allocator == omp_low_lat_mem_alloc)
 903     goto fail;
 904 #endif
 905
 906   if (__builtin_expect (allocator_data
 907                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 908     {
 909       uintptr_t used_pool_size;
 910       if (new_size > allocator_data->pool_size)
 911         goto fail;
 912 #ifdef HAVE_SYNC_BUILTINS
 913       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 914                                         MEMMODEL_RELAXED);
 915       do
 916         {
 917           uintptr_t new_pool_size;
 918           if (__builtin_add_overflow (used_pool_size, new_size,
 919                                       &new_pool_size)
 920               || new_pool_size > allocator_data->pool_size)
 921             goto fail;
 922           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 923                                            &used_pool_size, new_pool_size,
 924                                            true, MEMMODEL_RELAXED,
 925                                            MEMMODEL_RELAXED))
 926             break;
 927         }
 928       while (1);
 929 #else
 930       gomp_mutex_lock (&allocator_data->lock);
 931       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 932                                   &used_pool_size)
 933           || used_pool_size > allocator_data->pool_size)
 934         {
 935           gomp_mutex_unlock (&allocator_data->lock);
 936           goto fail;
 937         }
 938       allocator_data->used_pool_size = used_pool_size;
 939       gomp_mutex_unlock (&allocator_data->lock);
 940 #endif
 941 #ifdef LIBGOMP_USE_LIBNUMA
 942       if (memkind == GOMP_MEMKIND_LIBNUMA)
 943         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 944            memory that is initialized to zero.  */
 945         ptr = libnuma_data->numa_alloc_local (new_size);
 946 # ifdef LIBGOMP_USE_MEMKIND
 947       else
 948 # endif
 949 #endif
 950 #ifdef LIBGOMP_USE_MEMKIND
 951       if (memkind)
 952         {
 953           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 954           void *kind = *memkind_data->kinds[memkind];
 955           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 956         }
 957       else
 958 #endif
 959         ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size,
 960                                allocator_data->pinned);
 961       if (ptr == NULL)
 962         {
 963 #ifdef HAVE_SYNC_BUILTINS
 964           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 965                               MEMMODEL_RELAXED);
 966 #else
 967           gomp_mutex_lock (&allocator_data->lock);
 968           allocator_data->used_pool_size -= new_size;
 969           gomp_mutex_unlock (&allocator_data->lock);
 970 #endif
 971           goto fail;
 972         }
 973     }
 974   else
 975     {
 976 #ifdef LIBGOMP_USE_LIBNUMA
 977       if (memkind == GOMP_MEMKIND_LIBNUMA)
 978         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 979            memory that is initialized to zero.  */
 980         ptr = libnuma_data->numa_alloc_local (new_size);
 981 # ifdef LIBGOMP_USE_MEMKIND
 982       else
 983 # endif
 984 #endif
 985 #ifdef LIBGOMP_USE_MEMKIND
 986       if (memkind)
 987         {
 988           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 989           void *kind = *memkind_data->kinds[memkind];
 990           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 991         }
 992       else
 993 #endif
 994         {
 995           omp_memspace_handle_t memspace;
 996           memspace = (allocator_data
 997                       ? allocator_data->memspace
 998                       : predefined_alloc_mapping[allocator]);
 999           ptr = MEMSPACE_CALLOC (memspace, new_size,
1000                                  allocator_data && allocator_data->pinned);
1001         }
1002       if (ptr == NULL)
1003         goto fail;
1004     }
1005
1006   if (new_alignment > sizeof (void *))
1007     ret = (void *) (((uintptr_t) ptr
1008                      + sizeof (struct omp_mem_header)
1009                      + new_alignment - sizeof (void *))
1010                     & ~(new_alignment - 1));
1011   else
1012     ret = (char *) ptr + sizeof (struct omp_mem_header);
1013   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
1014   ((struct omp_mem_header *) ret)[-1].size = new_size;
1015   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1016   return ret;
1017
1018 fail:;
1019   int fallback = (allocator_data
1020                   ? allocator_data->fallback
1021                   : allocator == omp_default_mem_alloc
1022                   ? omp_atv_null_fb
1023                   : omp_atv_default_mem_fb);
1024   switch (fallback)
1025     {
1026     case omp_atv_default_mem_fb:
1027       allocator = omp_default_mem_alloc;
1028       goto retry;
1029     case omp_atv_null_fb:
1030       break;
1031     default:
1032     case omp_atv_abort_fb:
1033       gomp_fatal ("Out of memory allocating %lu bytes",
1034                   (unsigned long) (size * nmemb));
1035     case omp_atv_allocator_fb:
1036       allocator = allocator_data->fb_data;
1037       goto retry;
1038     }
1039   return NULL;
1040 }
1041
1042 ialias (omp_aligned_calloc)
1043
1044 void *
1045 omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
1046 {
1047   return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
1048 }
1049
1050 void *
1051 omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
1052              omp_allocator_handle_t free_allocator)
1053 {
1054   struct omp_allocator_data *allocator_data, *free_allocator_data;
1055   size_t new_size, old_size, new_alignment, old_alignment;
1056   void *new_ptr, *ret;
1057   struct omp_mem_header *data;
1058 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1059   enum gomp_numa_memkind_kind memkind, free_memkind;
1060 #endif
1061
1062   if (__builtin_expect (ptr == NULL, 0))
1063     return ialias_call (omp_aligned_alloc) (1, size, allocator);
1064
1065   if (__builtin_expect (size == 0, 0))
1066     {
1067       ialias_call (omp_free) (ptr, free_allocator);
1068       return NULL;
1069     }
1070
1071   data = &((struct omp_mem_header *) ptr)[-1];
1072   free_allocator = data->allocator;
1073
1074 retry:
1075   new_alignment = sizeof (void *);
1076   if (allocator == omp_null_allocator)
1077     allocator = free_allocator;
1078
1079   if (allocator > omp_max_predefined_alloc)
1080     {
1081       allocator_data = (struct omp_allocator_data *) allocator;
1082       if (new_alignment < allocator_data->alignment)
1083         new_alignment = allocator_data->alignment;
1084 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1085       memkind = allocator_data->memkind;
1086 #endif
1087     }
1088   else
1089     {
1090       allocator_data = NULL;
1091 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1092       memkind = GOMP_MEMKIND_NONE;
1093 #endif
1094 #ifdef LIBGOMP_USE_MEMKIND
1095       if (allocator == omp_high_bw_mem_alloc)
1096         memkind = GOMP_MEMKIND_HBW_PREFERRED;
1097       else if (allocator == omp_large_cap_mem_alloc)
1098         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1099       if (memkind)
1100         {
1101           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1102           if (!memkind_data->kinds[memkind])
1103             memkind = GOMP_MEMKIND_NONE;
1104         }
1105 #endif
1106     }
1107   if (free_allocator > omp_max_predefined_alloc)
1108     {
1109       free_allocator_data = (struct omp_allocator_data *) free_allocator;
1110 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1111       free_memkind = free_allocator_data->memkind;
1112 #endif
1113     }
1114   else
1115     {
1116       free_allocator_data = NULL;
1117 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1118       free_memkind = GOMP_MEMKIND_NONE;
1119 #endif
1120 #ifdef LIBGOMP_USE_MEMKIND
1121       if (free_allocator == omp_high_bw_mem_alloc)
1122         free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1123       else if (free_allocator == omp_large_cap_mem_alloc)
1124         free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1125       if (free_memkind)
1126         {
1127           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1128           if (!memkind_data->kinds[free_memkind])
1129             free_memkind = GOMP_MEMKIND_NONE;
1130         }
1131 #endif
1132     }
1133   old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1134
1135   new_size = sizeof (struct omp_mem_header);
1136   if (new_alignment > sizeof (void *))
1137     new_size += new_alignment - sizeof (void *);
1138   if (__builtin_add_overflow (size, new_size, &new_size))
1139     goto fail;
1140   old_size = data->size;
1141 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1142   if (allocator == omp_low_lat_mem_alloc)
1143     goto fail;
1144 #endif
1145
1146   if (__builtin_expect (allocator_data
1147                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1148     {
1149       uintptr_t used_pool_size;
1150       size_t prev_size = 0;
1151       /* Check if we can use realloc.  Don't use it if extra alignment
1152          was used previously or newly, because realloc might return a pointer
1153          with different alignment and then we'd need to memmove the data
1154          again.  */
1155       if (free_allocator_data
1156           && free_allocator_data == allocator_data
1157           && new_alignment == sizeof (void *)
1158           && old_alignment == sizeof (struct omp_mem_header))
1159         prev_size = old_size;
1160       if (new_size > prev_size
1161           && new_size - prev_size > allocator_data->pool_size)
1162         goto fail;
1163 #ifdef HAVE_SYNC_BUILTINS
1164       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1165                                         MEMMODEL_RELAXED);
1166       do
1167         {
1168           uintptr_t new_pool_size;
1169           if (new_size > prev_size)
1170             {
1171               if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1172                                           &new_pool_size)
1173                   || new_pool_size > allocator_data->pool_size)
1174                 goto fail;
1175             }
1176           else
1177             new_pool_size = used_pool_size + new_size - prev_size;
1178           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1179                                            &used_pool_size, new_pool_size,
1180                                            true, MEMMODEL_RELAXED,
1181                                            MEMMODEL_RELAXED))
1182             break;
1183         }
1184       while (1);
1185 #else
1186       gomp_mutex_lock (&allocator_data->lock);
1187       if (new_size > prev_size)
1188         {
1189           if (__builtin_add_overflow (allocator_data->used_pool_size,
1190                                       new_size - prev_size,
1191                                       &used_pool_size)
1192               || used_pool_size > allocator_data->pool_size)
1193             {
1194               gomp_mutex_unlock (&allocator_data->lock);
1195               goto fail;
1196             }
1197         }
1198       else
1199         used_pool_size = (allocator_data->used_pool_size
1200                           + new_size - prev_size);
1201       allocator_data->used_pool_size = used_pool_size;
1202       gomp_mutex_unlock (&allocator_data->lock);
1203 #endif
1204 #ifdef LIBGOMP_USE_LIBNUMA
1205       if (memkind == GOMP_MEMKIND_LIBNUMA)
1206         {
1207           if (prev_size)
1208             new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1209                                                   new_size);
1210           else
1211             new_ptr = libnuma_data->numa_alloc_local (new_size);
1212         }
1213 # ifdef LIBGOMP_USE_MEMKIND
1214       else
1215 # endif
1216 #endif
1217 #ifdef LIBGOMP_USE_MEMKIND
1218       if (memkind)
1219         {
1220           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1221           void *kind = *memkind_data->kinds[memkind];
1222           if (prev_size)
1223             new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1224                                                      new_size);
1225           else
1226             new_ptr = memkind_data->memkind_malloc (kind, new_size);
1227         }
1228       else
1229 #endif
1230       if (prev_size)
1231         new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
1232                                     data->size, new_size,
1233                                     (free_allocator_data
1234                                      && free_allocator_data->pinned),
1235                                     allocator_data->pinned);
1236       else
1237         new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
1238                                   allocator_data->pinned);
1239       if (new_ptr == NULL)
1240         {
1241 #ifdef HAVE_SYNC_BUILTINS
1242           __atomic_add_fetch (&allocator_data->used_pool_size,
1243                               prev_size - new_size,
1244                               MEMMODEL_RELAXED);
1245 #else
1246           gomp_mutex_lock (&allocator_data->lock);
1247           allocator_data->used_pool_size -= new_size - prev_size;
1248           gomp_mutex_unlock (&allocator_data->lock);
1249 #endif
1250           goto fail;
1251         }
1252       else if (prev_size)
1253         {
1254           ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1255           ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1256           ((struct omp_mem_header *) ret)[-1].size = new_size;
1257           ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1258           return ret;
1259         }
1260     }
1261   else if (new_alignment == sizeof (void *)
1262            && old_alignment == sizeof (struct omp_mem_header)
1263 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1264            && memkind == free_memkind
1265 #endif
1266            && (free_allocator_data == NULL
1267                || free_allocator_data->pool_size == ~(uintptr_t) 0))
1268     {
1269 #ifdef LIBGOMP_USE_LIBNUMA
1270       if (memkind == GOMP_MEMKIND_LIBNUMA)
1271         new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1272 # ifdef LIBGOMP_USE_MEMKIND
1273       else
1274 # endif
1275 #endif
1276 #ifdef LIBGOMP_USE_MEMKIND
1277       if (memkind)
1278         {
1279           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1280           void *kind = *memkind_data->kinds[memkind];
1281           new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1282                                                    new_size);
1283         }
1284       else
1285 #endif
1286         {
1287           omp_memspace_handle_t memspace;
1288           memspace = (allocator_data
1289                       ? allocator_data->memspace
1290                       : predefined_alloc_mapping[allocator]);
1291           new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
1292                                       (free_allocator_data
1293                                        && free_allocator_data->pinned),
1294                                       allocator_data && allocator_data->pinned);
1295         }
1296       if (new_ptr == NULL)
1297         goto fail;
1298
1299       ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1300       ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1301       ((struct omp_mem_header *) ret)[-1].size = new_size;
1302       ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1303       return ret;
1304     }
1305   else
1306     {
1307 #ifdef LIBGOMP_USE_LIBNUMA
1308       if (memkind == GOMP_MEMKIND_LIBNUMA)
1309         new_ptr = libnuma_data->numa_alloc_local (new_size);
1310 # ifdef LIBGOMP_USE_MEMKIND
1311       else
1312 # endif
1313 #endif
1314 #ifdef LIBGOMP_USE_MEMKIND
1315       if (memkind)
1316         {
1317           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1318           void *kind = *memkind_data->kinds[memkind];
1319           new_ptr = memkind_data->memkind_malloc (kind, new_size);
1320         }
1321       else
1322 #endif
1323         {
1324           omp_memspace_handle_t memspace;
1325           memspace = (allocator_data
1326                       ? allocator_data->memspace
1327                       : predefined_alloc_mapping[allocator]);
1328           new_ptr = MEMSPACE_ALLOC (memspace, new_size,
1329                                     allocator_data && allocator_data->pinned);
1330         }
1331       if (new_ptr == NULL)
1332         goto fail;
1333     }
1334
1335   if (new_alignment > sizeof (void *))
1336     ret = (void *) (((uintptr_t) new_ptr
1337                      + sizeof (struct omp_mem_header)
1338                      + new_alignment - sizeof (void *))
1339                     & ~(new_alignment - 1));
1340   else
1341     ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1342   ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1343   ((struct omp_mem_header *) ret)[-1].size = new_size;
1344   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1345   if (old_size - old_alignment < size)
1346     size = old_size - old_alignment;
1347   memcpy (ret, ptr, size);
1348   if (__builtin_expect (free_allocator_data
1349                         && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1350     {
1351 #ifdef HAVE_SYNC_BUILTINS
1352       __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1353                           MEMMODEL_RELAXED);
1354 #else
1355       gomp_mutex_lock (&free_allocator_data->lock);
1356       free_allocator_data->used_pool_size -= data->size;
1357       gomp_mutex_unlock (&free_allocator_data->lock);
1358 #endif
1359     }
1360 #ifdef LIBGOMP_USE_LIBNUMA
1361   if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1362     {
1363       libnuma_data->numa_free (data->ptr, data->size);
1364       return ret;
1365     }
1366 # ifdef LIBGOMP_USE_MEMKIND
1367   else
1368 # endif
1369 #endif
1370 #ifdef LIBGOMP_USE_MEMKIND
1371   if (free_memkind)
1372     {
1373       struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1374       void *kind = *memkind_data->kinds[free_memkind];
1375       memkind_data->memkind_free (kind, data->ptr);
1376       return ret;
1377     }
1378 #endif
1379   {
1380     omp_memspace_handle_t was_memspace;
1381     was_memspace = (free_allocator_data
1382                     ? free_allocator_data->memspace
1383                     : predefined_alloc_mapping[free_allocator]);
1384     int was_pinned = (free_allocator_data && free_allocator_data->pinned);
1385     MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
1386   }
1387   return ret;
1388
1389 fail:;
1390   int fallback = (allocator_data
1391                   ? allocator_data->fallback
1392                   : allocator == omp_default_mem_alloc
1393                   ? omp_atv_null_fb
1394                   : omp_atv_default_mem_fb);
1395   switch (fallback)
1396     {
1397     case omp_atv_default_mem_fb:
1398       allocator = omp_default_mem_alloc;
1399       goto retry;
1400     case omp_atv_null_fb:
1401       break;
1402     default:
1403     case omp_atv_abort_fb:
1404       gomp_fatal ("Out of memory allocating %lu bytes",
1405                   (unsigned long) size);
1406     case omp_atv_allocator_fb:
1407       allocator = allocator_data->fb_data;
1408       goto retry;
1409     }
1410   return NULL;
1411 }