libgomp/allocator.c

   1 /* Copyright (C) 2020-2022 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #ifdef LIBGOMP_USE_MEMKIND
  35 #include <dlfcn.h>
  36 #endif
  37
  38 #define omp_max_predefined_alloc omp_thread_mem_alloc
  39
  40 enum gomp_memkind_kind
  41 {
  42   GOMP_MEMKIND_NONE = 0,
  43 #define GOMP_MEMKIND_KINDS \
  44   GOMP_MEMKIND_KIND (HBW_INTERLEAVE),           \
  45   GOMP_MEMKIND_KIND (HBW_PREFERRED),            \
  46   GOMP_MEMKIND_KIND (DAX_KMEM_ALL),             \
  47   GOMP_MEMKIND_KIND (DAX_KMEM),                 \
  48   GOMP_MEMKIND_KIND (INTERLEAVE),               \
  49   GOMP_MEMKIND_KIND (DEFAULT)
  50 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
  51   GOMP_MEMKIND_KINDS,
  52 #undef GOMP_MEMKIND_KIND
  53   GOMP_MEMKIND_COUNT
  54 };
  55
  56 struct omp_allocator_data
  57 {
  58   omp_memspace_handle_t memspace;
  59   omp_uintptr_t alignment;
  60   omp_uintptr_t pool_size;
  61   omp_uintptr_t used_pool_size;
  62   omp_allocator_handle_t fb_data;
  63   unsigned int sync_hint : 8;
  64   unsigned int access : 8;
  65   unsigned int fallback : 8;
  66   unsigned int pinned : 1;
  67   unsigned int partition : 7;
  68 #ifdef LIBGOMP_USE_MEMKIND
  69   unsigned int memkind : 8;
  70 #endif
  71 #ifndef HAVE_SYNC_BUILTINS
  72   gomp_mutex_t lock;
  73 #endif
  74 };
  75
  76 struct omp_mem_header
  77 {
  78   void *ptr;
  79   size_t size;
  80   omp_allocator_handle_t allocator;
  81   void *pad;
  82 };
  83
  84 struct gomp_memkind_data
  85 {
  86   void *memkind_handle;
  87   void *(*memkind_malloc) (void *, size_t);
  88   void *(*memkind_calloc) (void *, size_t, size_t);
  89   void *(*memkind_realloc) (void *, void *, size_t);
  90   void (*memkind_free) (void *, void *);
  91   int (*memkind_check_available) (void *);
  92   void **kinds[GOMP_MEMKIND_COUNT];
  93 };
  94
  95 #ifdef LIBGOMP_USE_MEMKIND
  96 static struct gomp_memkind_data *memkind_data;
  97 static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
  98
  99 static void
 100 gomp_init_memkind (void)
 101 {
 102   void *handle = dlopen ("libmemkind.so", RTLD_LAZY);
 103   struct gomp_memkind_data *data;
 104   int i;
 105   static const char *kinds[] = {
 106     NULL,
 107 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
 108     GOMP_MEMKIND_KINDS
 109 #undef GOMP_MEMKIND_KIND
 110   };
 111
 112   data = calloc (1, sizeof (struct gomp_memkind_data));
 113   if (data == NULL)
 114     {
 115       if (handle)
 116         dlclose (handle);
 117       return;
 118     }
 119   if (!handle)
 120     {
 121       __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 122       return;
 123     }
 124   data->memkind_handle = handle;
 125   data->memkind_malloc
 126     = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
 127   data->memkind_calloc
 128     = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
 129   data->memkind_realloc
 130     = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
 131   data->memkind_free
 132     = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
 133   data->memkind_check_available
 134     = (__typeof (data->memkind_check_available))
 135       dlsym (handle, "memkind_check_available");
 136   if (data->memkind_malloc
 137       && data->memkind_calloc
 138       && data->memkind_realloc
 139       && data->memkind_free
 140       && data->memkind_check_available)
 141     for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
 142       {
 143         data->kinds[i] = (void **) dlsym (handle, kinds[i]);
 144         if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
 145           data->kinds[i] = NULL;
 146       }
 147   __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 148 }
 149
 150 static struct gomp_memkind_data *
 151 gomp_get_memkind (void)
 152 {
 153   struct gomp_memkind_data *data
 154     = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 155   if (data)
 156     return data;
 157   pthread_once (&memkind_data_once, gomp_init_memkind);
 158   return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 159 }
 160 #endif
 161
 162 omp_allocator_handle_t
 163 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
 164                     const omp_alloctrait_t traits[])
 165 {
 166   struct omp_allocator_data data
 167     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
 168         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
 169 #ifdef LIBGOMP_USE_MEMKIND
 170         GOMP_MEMKIND_NONE
 171 #endif
 172       };
 173   struct omp_allocator_data *ret;
 174   int i;
 175
 176   if (memspace > omp_low_lat_mem_space)
 177     return omp_null_allocator;
 178   for (i = 0; i < ntraits; i++)
 179     switch (traits[i].key)
 180       {
 181       case omp_atk_sync_hint:
 182         switch (traits[i].value)
 183           {
 184           case omp_atv_default:
 185             data.sync_hint = omp_atv_contended;
 186             break;
 187           case omp_atv_contended:
 188           case omp_atv_uncontended:
 189           case omp_atv_serialized:
 190           case omp_atv_private:
 191             data.sync_hint = traits[i].value;
 192             break;
 193           default:
 194             return omp_null_allocator;
 195           }
 196         break;
 197       case omp_atk_alignment:
 198         if (traits[i].value == omp_atv_default)
 199           {
 200             data.alignment = 1;
 201             break;
 202           }
 203         if ((traits[i].value & (traits[i].value - 1)) != 0
 204             || !traits[i].value)
 205           return omp_null_allocator;
 206         data.alignment = traits[i].value;
 207         break;
 208       case omp_atk_access:
 209         switch (traits[i].value)
 210           {
 211           case omp_atv_default:
 212             data.access = omp_atv_all;
 213             break;
 214           case omp_atv_all:
 215           case omp_atv_cgroup:
 216           case omp_atv_pteam:
 217           case omp_atv_thread:
 218             data.access = traits[i].value;
 219             break;
 220           default:
 221             return omp_null_allocator;
 222           }
 223         break;
 224       case omp_atk_pool_size:
 225         if (traits[i].value == omp_atv_default)
 226           data.pool_size = ~(uintptr_t) 0;
 227         else
 228           data.pool_size = traits[i].value;
 229         break;
 230       case omp_atk_fallback:
 231         switch (traits[i].value)
 232           {
 233           case omp_atv_default:
 234             data.fallback = omp_atv_default_mem_fb;
 235             break;
 236           case omp_atv_default_mem_fb:
 237           case omp_atv_null_fb:
 238           case omp_atv_abort_fb:
 239           case omp_atv_allocator_fb:
 240             data.fallback = traits[i].value;
 241             break;
 242           default:
 243             return omp_null_allocator;
 244           }
 245         break;
 246       case omp_atk_fb_data:
 247         data.fb_data = traits[i].value;
 248         break;
 249       case omp_atk_pinned:
 250         switch (traits[i].value)
 251           {
 252           case omp_atv_default:
 253           case omp_atv_false:
 254             data.pinned = omp_atv_false;
 255             break;
 256           case omp_atv_true:
 257             data.pinned = omp_atv_true;
 258             break;
 259           default:
 260             return omp_null_allocator;
 261           }
 262         break;
 263       case omp_atk_partition:
 264         switch (traits[i].value)
 265           {
 266           case omp_atv_default:
 267             data.partition = omp_atv_environment;
 268             break;
 269           case omp_atv_environment:
 270           case omp_atv_nearest:
 271           case omp_atv_blocked:
 272           case omp_atv_interleaved:
 273             data.partition = traits[i].value;
 274             break;
 275           default:
 276             return omp_null_allocator;
 277           }
 278         break;
 279       default:
 280         return omp_null_allocator;
 281       }
 282
 283   if (data.alignment < sizeof (void *))
 284     data.alignment = sizeof (void *);
 285
 286   switch (memspace)
 287     {
 288     case omp_high_bw_mem_space:
 289 #ifdef LIBGOMP_USE_MEMKIND
 290       struct gomp_memkind_data *memkind_data;
 291       memkind_data = gomp_get_memkind ();
 292       if (data.partition == omp_atv_interleaved
 293           && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
 294         {
 295           data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
 296           break;
 297         }
 298       else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
 299         {
 300           data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
 301           break;
 302         }
 303 #endif
 304       return omp_null_allocator;
 305     case omp_large_cap_mem_space:
 306 #ifdef LIBGOMP_USE_MEMKIND
 307       memkind_data = gomp_get_memkind ();
 308       if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
 309         data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 310       else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
 311         data.memkind = GOMP_MEMKIND_DAX_KMEM;
 312 #endif
 313       break;
 314     default:
 315 #ifdef LIBGOMP_USE_MEMKIND
 316       if (data.partition == omp_atv_interleaved)
 317         {
 318           memkind_data = gomp_get_memkind ();
 319           if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
 320             data.memkind = GOMP_MEMKIND_INTERLEAVE;
 321         }
 322 #endif
 323       break;
 324     }
 325
 326   /* No support for this so far.  */
 327   if (data.pinned)
 328     return omp_null_allocator;
 329
 330   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 331   *ret = data;
 332 #ifndef HAVE_SYNC_BUILTINS
 333   gomp_mutex_init (&ret->lock);
 334 #endif
 335   return (omp_allocator_handle_t) ret;
 336 }
 337
 338 void
 339 omp_destroy_allocator (omp_allocator_handle_t allocator)
 340 {
 341   if (allocator != omp_null_allocator)
 342     {
 343 #ifndef HAVE_SYNC_BUILTINS
 344       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 345 #endif
 346       free ((void *) allocator);
 347     }
 348 }
 349
 350 ialias (omp_init_allocator)
 351 ialias (omp_destroy_allocator)
 352
 353 void *
 354 omp_aligned_alloc (size_t alignment, size_t size,
 355                    omp_allocator_handle_t allocator)
 356 {
 357   struct omp_allocator_data *allocator_data;
 358   size_t new_size, new_alignment;
 359   void *ptr, *ret;
 360 #ifdef LIBGOMP_USE_MEMKIND
 361   enum gomp_memkind_kind memkind;
 362 #endif
 363
 364   if (__builtin_expect (size == 0, 0))
 365     return NULL;
 366
 367 retry:
 368   new_alignment = alignment;
 369   if (allocator == omp_null_allocator)
 370     {
 371       struct gomp_thread *thr = gomp_thread ();
 372       if (thr->ts.def_allocator == omp_null_allocator)
 373         thr->ts.def_allocator = gomp_def_allocator;
 374       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 375     }
 376
 377   if (allocator > omp_max_predefined_alloc)
 378     {
 379       allocator_data = (struct omp_allocator_data *) allocator;
 380       if (new_alignment < allocator_data->alignment)
 381         new_alignment = allocator_data->alignment;
 382 #ifdef LIBGOMP_USE_MEMKIND
 383       memkind = allocator_data->memkind;
 384 #endif
 385     }
 386   else
 387     {
 388       allocator_data = NULL;
 389       if (new_alignment < sizeof (void *))
 390         new_alignment = sizeof (void *);
 391 #ifdef LIBGOMP_USE_MEMKIND
 392       memkind = GOMP_MEMKIND_NONE;
 393       if (allocator == omp_high_bw_mem_alloc)
 394         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 395       else if (allocator == omp_large_cap_mem_alloc)
 396         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 397       if (memkind)
 398         {
 399           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 400           if (!memkind_data->kinds[memkind])
 401             memkind = GOMP_MEMKIND_NONE;
 402         }
 403 #endif
 404     }
 405
 406   new_size = sizeof (struct omp_mem_header);
 407   if (new_alignment > sizeof (void *))
 408     new_size += new_alignment - sizeof (void *);
 409   if (__builtin_add_overflow (size, new_size, &new_size))
 410     goto fail;
 411
 412   if (__builtin_expect (allocator_data
 413                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 414     {
 415       uintptr_t used_pool_size;
 416       if (new_size > allocator_data->pool_size)
 417         goto fail;
 418 #ifdef HAVE_SYNC_BUILTINS
 419       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 420                                         MEMMODEL_RELAXED);
 421       do
 422         {
 423           uintptr_t new_pool_size;
 424           if (__builtin_add_overflow (used_pool_size, new_size,
 425                                       &new_pool_size)
 426               || new_pool_size > allocator_data->pool_size)
 427             goto fail;
 428           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 429                                            &used_pool_size, new_pool_size,
 430                                            true, MEMMODEL_RELAXED,
 431                                            MEMMODEL_RELAXED))
 432             break;
 433         }
 434       while (1);
 435 #else
 436       gomp_mutex_lock (&allocator_data->lock);
 437       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 438                                   &used_pool_size)
 439           || used_pool_size > allocator_data->pool_size)
 440         {
 441           gomp_mutex_unlock (&allocator_data->lock);
 442           goto fail;
 443         }
 444       allocator_data->used_pool_size = used_pool_size;
 445       gomp_mutex_unlock (&allocator_data->lock);
 446 #endif
 447 #ifdef LIBGOMP_USE_MEMKIND
 448       if (memkind)
 449         {
 450           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 451           void *kind = *memkind_data->kinds[memkind];
 452           ptr = memkind_data->memkind_malloc (kind, new_size);
 453         }
 454       else
 455 #endif
 456         ptr = malloc (new_size);
 457       if (ptr == NULL)
 458         {
 459 #ifdef HAVE_SYNC_BUILTINS
 460           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 461                               MEMMODEL_RELAXED);
 462 #else
 463           gomp_mutex_lock (&allocator_data->lock);
 464           allocator_data->used_pool_size -= new_size;
 465           gomp_mutex_unlock (&allocator_data->lock);
 466 #endif
 467           goto fail;
 468         }
 469     }
 470   else
 471     {
 472 #ifdef LIBGOMP_USE_MEMKIND
 473       if (memkind)
 474         {
 475           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 476           void *kind = *memkind_data->kinds[memkind];
 477           ptr = memkind_data->memkind_malloc (kind, new_size);
 478         }
 479       else
 480 #endif
 481         ptr = malloc (new_size);
 482       if (ptr == NULL)
 483         goto fail;
 484     }
 485
 486   if (new_alignment > sizeof (void *))
 487     ret = (void *) (((uintptr_t) ptr
 488                      + sizeof (struct omp_mem_header)
 489                      + new_alignment - sizeof (void *))
 490                     & ~(new_alignment - 1));
 491   else
 492     ret = (char *) ptr + sizeof (struct omp_mem_header);
 493   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 494   ((struct omp_mem_header *) ret)[-1].size = new_size;
 495   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 496   return ret;
 497
 498 fail:
 499   if (allocator_data)
 500     {
 501       switch (allocator_data->fallback)
 502         {
 503         case omp_atv_default_mem_fb:
 504           if ((new_alignment > sizeof (void *) && new_alignment > alignment)
 505 #ifdef LIBGOMP_USE_MEMKIND
 506               || memkind
 507 #endif
 508               || (allocator_data
 509                   && allocator_data->pool_size < ~(uintptr_t) 0))
 510             {
 511               allocator = omp_default_mem_alloc;
 512               goto retry;
 513             }
 514           /* Otherwise, we've already performed default mem allocation
 515              and if that failed, it won't succeed again (unless it was
 516              intermittent.  Return NULL then, as that is the fallback.  */
 517           break;
 518         case omp_atv_null_fb:
 519           break;
 520         default:
 521         case omp_atv_abort_fb:
 522           gomp_fatal ("Out of memory allocating %lu bytes",
 523                       (unsigned long) size);
 524         case omp_atv_allocator_fb:
 525           allocator = allocator_data->fb_data;
 526           goto retry;
 527         }
 528     }
 529   return NULL;
 530 }
 531
 532 ialias (omp_aligned_alloc)
 533
 534 void *
 535 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 536 {
 537   return ialias_call (omp_aligned_alloc) (1, size, allocator);
 538 }
 539
 540 /* Like omp_aligned_alloc, but apply on top of that:
 541    "For allocations that arise from this ... the null_fb value of the
 542    fallback allocator trait behaves as if the abort_fb had been specified."  */
 543
 544 void *
 545 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 546 {
 547   void *ret
 548     = ialias_call (omp_aligned_alloc) (alignment, size,
 549                                        (omp_allocator_handle_t) allocator);
 550   if (__builtin_expect (ret == NULL, 0) && size)
 551     gomp_fatal ("Out of memory allocating %lu bytes",
 552                 (unsigned long) size);
 553   return ret;
 554 }
 555
 556 void
 557 omp_free (void *ptr, omp_allocator_handle_t allocator)
 558 {
 559   struct omp_mem_header *data;
 560
 561   if (ptr == NULL)
 562     return;
 563   (void) allocator;
 564   data = &((struct omp_mem_header *) ptr)[-1];
 565   if (data->allocator > omp_max_predefined_alloc)
 566     {
 567       struct omp_allocator_data *allocator_data
 568         = (struct omp_allocator_data *) (data->allocator);
 569       if (allocator_data->pool_size < ~(uintptr_t) 0)
 570         {
 571 #ifdef HAVE_SYNC_BUILTINS
 572           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 573                               MEMMODEL_RELAXED);
 574 #else
 575           gomp_mutex_lock (&allocator_data->lock);
 576           allocator_data->used_pool_size -= data->size;
 577           gomp_mutex_unlock (&allocator_data->lock);
 578 #endif
 579         }
 580 #ifdef LIBGOMP_USE_MEMKIND
 581       if (allocator_data->memkind)
 582         {
 583           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 584           void *kind = *memkind_data->kinds[allocator_data->memkind];
 585           memkind_data->memkind_free (kind, data->ptr);
 586           return;
 587         }
 588 #endif
 589     }
 590 #ifdef LIBGOMP_USE_MEMKIND
 591   else
 592     {
 593       enum gomp_memkind_kind memkind = GOMP_MEMKIND_NONE;
 594       if (data->allocator == omp_high_bw_mem_alloc)
 595         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 596       else if (data->allocator == omp_large_cap_mem_alloc)
 597         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 598       if (memkind)
 599         {
 600           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 601           if (memkind_data->kinds[memkind])
 602             {
 603               void *kind = *memkind_data->kinds[memkind];
 604               memkind_data->memkind_free (kind, data->ptr);
 605               return;
 606             }
 607         }
 608     }
 609 #endif
 610   free (data->ptr);
 611 }
 612
 613 ialias (omp_free)
 614
 615 void
 616 GOMP_free (void *ptr, uintptr_t allocator)
 617 {
 618   return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
 619 }
 620
 621 void *
 622 omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
 623                     omp_allocator_handle_t allocator)
 624 {
 625   struct omp_allocator_data *allocator_data;
 626   size_t new_size, size_temp, new_alignment;
 627   void *ptr, *ret;
 628 #ifdef LIBGOMP_USE_MEMKIND
 629   enum gomp_memkind_kind memkind;
 630 #endif
 631
 632   if (__builtin_expect (size == 0 || nmemb == 0, 0))
 633     return NULL;
 634
 635 retry:
 636   new_alignment = alignment;
 637   if (allocator == omp_null_allocator)
 638     {
 639       struct gomp_thread *thr = gomp_thread ();
 640       if (thr->ts.def_allocator == omp_null_allocator)
 641         thr->ts.def_allocator = gomp_def_allocator;
 642       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 643     }
 644
 645   if (allocator > omp_max_predefined_alloc)
 646     {
 647       allocator_data = (struct omp_allocator_data *) allocator;
 648       if (new_alignment < allocator_data->alignment)
 649         new_alignment = allocator_data->alignment;
 650 #ifdef LIBGOMP_USE_MEMKIND
 651       memkind = allocator_data->memkind;
 652 #endif
 653     }
 654   else
 655     {
 656       allocator_data = NULL;
 657       if (new_alignment < sizeof (void *))
 658         new_alignment = sizeof (void *);
 659 #ifdef LIBGOMP_USE_MEMKIND
 660       memkind = GOMP_MEMKIND_NONE;
 661       if (allocator == omp_high_bw_mem_alloc)
 662         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 663       else if (allocator == omp_large_cap_mem_alloc)
 664         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 665       if (memkind)
 666         {
 667           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 668           if (!memkind_data->kinds[memkind])
 669             memkind = GOMP_MEMKIND_NONE;
 670         }
 671 #endif
 672     }
 673
 674   new_size = sizeof (struct omp_mem_header);
 675   if (new_alignment > sizeof (void *))
 676     new_size += new_alignment - sizeof (void *);
 677   if (__builtin_mul_overflow (size, nmemb, &size_temp))
 678     goto fail;
 679   if (__builtin_add_overflow (size_temp, new_size, &new_size))
 680     goto fail;
 681
 682   if (__builtin_expect (allocator_data
 683                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 684     {
 685       uintptr_t used_pool_size;
 686       if (new_size > allocator_data->pool_size)
 687         goto fail;
 688 #ifdef HAVE_SYNC_BUILTINS
 689       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 690                                         MEMMODEL_RELAXED);
 691       do
 692         {
 693           uintptr_t new_pool_size;
 694           if (__builtin_add_overflow (used_pool_size, new_size,
 695                                       &new_pool_size)
 696               || new_pool_size > allocator_data->pool_size)
 697             goto fail;
 698           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 699                                            &used_pool_size, new_pool_size,
 700                                            true, MEMMODEL_RELAXED,
 701                                            MEMMODEL_RELAXED))
 702             break;
 703         }
 704       while (1);
 705 #else
 706       gomp_mutex_lock (&allocator_data->lock);
 707       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 708                                   &used_pool_size)
 709           || used_pool_size > allocator_data->pool_size)
 710         {
 711           gomp_mutex_unlock (&allocator_data->lock);
 712           goto fail;
 713         }
 714       allocator_data->used_pool_size = used_pool_size;
 715       gomp_mutex_unlock (&allocator_data->lock);
 716 #endif
 717 #ifdef LIBGOMP_USE_MEMKIND
 718       if (memkind)
 719         {
 720           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 721           void *kind = *memkind_data->kinds[memkind];
 722           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 723         }
 724       else
 725 #endif
 726         ptr = calloc (1, new_size);
 727       if (ptr == NULL)
 728         {
 729 #ifdef HAVE_SYNC_BUILTINS
 730           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 731                               MEMMODEL_RELAXED);
 732 #else
 733           gomp_mutex_lock (&allocator_data->lock);
 734           allocator_data->used_pool_size -= new_size;
 735           gomp_mutex_unlock (&allocator_data->lock);
 736 #endif
 737           goto fail;
 738         }
 739     }
 740   else
 741     {
 742 #ifdef LIBGOMP_USE_MEMKIND
 743       if (memkind)
 744         {
 745           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 746           void *kind = *memkind_data->kinds[memkind];
 747           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 748         }
 749       else
 750 #endif
 751         ptr = calloc (1, new_size);
 752       if (ptr == NULL)
 753         goto fail;
 754     }
 755
 756   if (new_alignment > sizeof (void *))
 757     ret = (void *) (((uintptr_t) ptr
 758                      + sizeof (struct omp_mem_header)
 759                      + new_alignment - sizeof (void *))
 760                     & ~(new_alignment - 1));
 761   else
 762     ret = (char *) ptr + sizeof (struct omp_mem_header);
 763   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 764   ((struct omp_mem_header *) ret)[-1].size = new_size;
 765   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 766   return ret;
 767
 768 fail:
 769   if (allocator_data)
 770     {
 771       switch (allocator_data->fallback)
 772         {
 773         case omp_atv_default_mem_fb:
 774           if ((new_alignment > sizeof (void *) && new_alignment > alignment)
 775 #ifdef LIBGOMP_USE_MEMKIND
 776               || memkind
 777 #endif
 778               || (allocator_data
 779                   && allocator_data->pool_size < ~(uintptr_t) 0))
 780             {
 781               allocator = omp_default_mem_alloc;
 782               goto retry;
 783             }
 784           /* Otherwise, we've already performed default mem allocation
 785              and if that failed, it won't succeed again (unless it was
 786              intermittent.  Return NULL then, as that is the fallback.  */
 787           break;
 788         case omp_atv_null_fb:
 789           break;
 790         default:
 791         case omp_atv_abort_fb:
 792           gomp_fatal ("Out of memory allocating %lu bytes",
 793                       (unsigned long) (size * nmemb));
 794         case omp_atv_allocator_fb:
 795           allocator = allocator_data->fb_data;
 796           goto retry;
 797         }
 798     }
 799   return NULL;
 800 }
 801
 802 ialias (omp_aligned_calloc)
 803
 804 void *
 805 omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
 806 {
 807   return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
 808 }
 809
 810 void *
 811 omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
 812              omp_allocator_handle_t free_allocator)
 813 {
 814   struct omp_allocator_data *allocator_data, *free_allocator_data;
 815   size_t new_size, old_size, new_alignment, old_alignment;
 816   void *new_ptr, *ret;
 817   struct omp_mem_header *data;
 818 #ifdef LIBGOMP_USE_MEMKIND
 819   enum gomp_memkind_kind memkind, free_memkind;
 820 #endif
 821
 822   if (__builtin_expect (ptr == NULL, 0))
 823     return ialias_call (omp_aligned_alloc) (1, size, allocator);
 824
 825   if (__builtin_expect (size == 0, 0))
 826     {
 827       ialias_call (omp_free) (ptr, free_allocator);
 828       return NULL;
 829     }
 830
 831   data = &((struct omp_mem_header *) ptr)[-1];
 832   free_allocator = data->allocator;
 833
 834 retry:
 835   new_alignment = sizeof (void *);
 836   if (allocator == omp_null_allocator)
 837     allocator = free_allocator;
 838
 839   if (allocator > omp_max_predefined_alloc)
 840     {
 841       allocator_data = (struct omp_allocator_data *) allocator;
 842       if (new_alignment < allocator_data->alignment)
 843         new_alignment = allocator_data->alignment;
 844 #ifdef LIBGOMP_USE_MEMKIND
 845       memkind = allocator_data->memkind;
 846 #endif
 847     }
 848   else
 849     {
 850       allocator_data = NULL;
 851 #ifdef LIBGOMP_USE_MEMKIND
 852       memkind = GOMP_MEMKIND_NONE;
 853       if (allocator == omp_high_bw_mem_alloc)
 854         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 855       else if (allocator == omp_large_cap_mem_alloc)
 856         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 857       if (memkind)
 858         {
 859           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 860           if (!memkind_data->kinds[memkind])
 861             memkind = GOMP_MEMKIND_NONE;
 862         }
 863 #endif
 864     }
 865   if (free_allocator > omp_max_predefined_alloc)
 866     {
 867       free_allocator_data = (struct omp_allocator_data *) free_allocator;
 868 #ifdef LIBGOMP_USE_MEMKIND
 869       free_memkind = free_allocator_data->memkind;
 870 #endif
 871     }
 872   else
 873     {
 874       free_allocator_data = NULL;
 875 #ifdef LIBGOMP_USE_MEMKIND
 876       free_memkind = GOMP_MEMKIND_NONE;
 877       if (free_allocator == omp_high_bw_mem_alloc)
 878         free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
 879       else if (free_allocator == omp_large_cap_mem_alloc)
 880         free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 881       if (free_memkind)
 882         {
 883           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 884           if (!memkind_data->kinds[free_memkind])
 885             free_memkind = GOMP_MEMKIND_NONE;
 886         }
 887 #endif
 888     }
 889   old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
 890
 891   new_size = sizeof (struct omp_mem_header);
 892   if (new_alignment > sizeof (void *))
 893     new_size += new_alignment - sizeof (void *);
 894   if (__builtin_add_overflow (size, new_size, &new_size))
 895     goto fail;
 896   old_size = data->size;
 897
 898   if (__builtin_expect (allocator_data
 899                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 900     {
 901       uintptr_t used_pool_size;
 902       size_t prev_size = 0;
 903       /* Check if we can use realloc.  Don't use it if extra alignment
 904          was used previously or newly, because realloc might return a pointer
 905          with different alignment and then we'd need to memmove the data
 906          again.  */
 907       if (free_allocator_data
 908           && free_allocator_data == allocator_data
 909           && new_alignment == sizeof (void *)
 910           && old_alignment == sizeof (struct omp_mem_header))
 911         prev_size = old_size;
 912       if (new_size > prev_size
 913           && new_size - prev_size > allocator_data->pool_size)
 914         goto fail;
 915 #ifdef HAVE_SYNC_BUILTINS
 916       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 917                                         MEMMODEL_RELAXED);
 918       do
 919         {
 920           uintptr_t new_pool_size;
 921           if (new_size > prev_size)
 922             {
 923               if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
 924                                           &new_pool_size)
 925                   || new_pool_size > allocator_data->pool_size)
 926                 goto fail;
 927             }
 928           else
 929             new_pool_size = used_pool_size + new_size - prev_size;
 930           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 931                                            &used_pool_size, new_pool_size,
 932                                            true, MEMMODEL_RELAXED,
 933                                            MEMMODEL_RELAXED))
 934             break;
 935         }
 936       while (1);
 937 #else
 938       gomp_mutex_lock (&allocator_data->lock);
 939       if (new_size > prev_size)
 940         {
 941           if (__builtin_add_overflow (allocator_data->used_pool_size,
 942                                       new_size - prev_size,
 943                                       &used_pool_size)
 944               || used_pool_size > allocator_data->pool_size)
 945             {
 946               gomp_mutex_unlock (&allocator_data->lock);
 947               goto fail;
 948             }
 949         }
 950       else
 951         used_pool_size = (allocator_data->used_pool_size
 952                           + new_size - prev_size);
 953       allocator_data->used_pool_size = used_pool_size;
 954       gomp_mutex_unlock (&allocator_data->lock);
 955 #endif
 956 #ifdef LIBGOMP_USE_MEMKIND
 957       if (memkind)
 958         {
 959           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 960           void *kind = *memkind_data->kinds[memkind];
 961           if (prev_size)
 962             new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
 963                                                      new_size);
 964           else
 965             new_ptr = memkind_data->memkind_malloc (kind, new_size);
 966         }
 967       else
 968 #endif
 969       if (prev_size)
 970         new_ptr = realloc (data->ptr, new_size);
 971       else
 972         new_ptr = malloc (new_size);
 973       if (new_ptr == NULL)
 974         {
 975 #ifdef HAVE_SYNC_BUILTINS
 976           __atomic_add_fetch (&allocator_data->used_pool_size,
 977                               prev_size - new_size,
 978                               MEMMODEL_RELAXED);
 979 #else
 980           gomp_mutex_lock (&allocator_data->lock);
 981           allocator_data->used_pool_size -= new_size - prev_size;
 982           gomp_mutex_unlock (&allocator_data->lock);
 983 #endif
 984           goto fail;
 985         }
 986       else if (prev_size)
 987         {
 988           ret = (char *) new_ptr + sizeof (struct omp_mem_header);
 989           ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
 990           ((struct omp_mem_header *) ret)[-1].size = new_size;
 991           ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 992           return ret;
 993         }
 994     }
 995   else if (new_alignment == sizeof (void *)
 996            && old_alignment == sizeof (struct omp_mem_header)
 997 #ifdef LIBGOMP_USE_MEMKIND
 998            && memkind == free_memkind
 999 #endif
1000            && (free_allocator_data == NULL
1001                || free_allocator_data->pool_size == ~(uintptr_t) 0))
1002     {
1003 #ifdef LIBGOMP_USE_MEMKIND
1004       if (memkind)
1005         {
1006           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1007           void *kind = *memkind_data->kinds[memkind];
1008           new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1009                                                    new_size);
1010         }
1011       else
1012 #endif
1013         new_ptr = realloc (data->ptr, new_size);
1014       if (new_ptr == NULL)
1015         goto fail;
1016       ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1017       ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1018       ((struct omp_mem_header *) ret)[-1].size = new_size;
1019       ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1020       return ret;
1021     }
1022   else
1023     {
1024 #ifdef LIBGOMP_USE_MEMKIND
1025       if (memkind)
1026         {
1027           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1028           void *kind = *memkind_data->kinds[memkind];
1029           new_ptr = memkind_data->memkind_malloc (kind, new_size);
1030         }
1031       else
1032 #endif
1033         new_ptr = malloc (new_size);
1034       if (new_ptr == NULL)
1035         goto fail;
1036     }
1037
1038   if (new_alignment > sizeof (void *))
1039     ret = (void *) (((uintptr_t) new_ptr
1040                      + sizeof (struct omp_mem_header)
1041                      + new_alignment - sizeof (void *))
1042                     & ~(new_alignment - 1));
1043   else
1044     ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1045   ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1046   ((struct omp_mem_header *) ret)[-1].size = new_size;
1047   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1048   if (old_size - old_alignment < size)
1049     size = old_size - old_alignment;
1050   memcpy (ret, ptr, size);
1051   if (__builtin_expect (free_allocator_data
1052                         && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1053     {
1054 #ifdef HAVE_SYNC_BUILTINS
1055       __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1056                           MEMMODEL_RELAXED);
1057 #else
1058       gomp_mutex_lock (&free_allocator_data->lock);
1059       free_allocator_data->used_pool_size -= data->size;
1060       gomp_mutex_unlock (&free_allocator_data->lock);
1061 #endif
1062     }
1063 #ifdef LIBGOMP_USE_MEMKIND
1064   if (free_memkind)
1065     {
1066       struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1067       void *kind = *memkind_data->kinds[free_memkind];
1068       memkind_data->memkind_free (kind, data->ptr);
1069       return ret;
1070     }
1071 #endif
1072   free (data->ptr);
1073   return ret;
1074
1075 fail:
1076   if (allocator_data)
1077     {
1078       switch (allocator_data->fallback)
1079         {
1080         case omp_atv_default_mem_fb:
1081           if (new_alignment > sizeof (void *)
1082 #ifdef LIBGOMP_USE_MEMKIND
1083               || memkind
1084 #endif
1085               || (allocator_data
1086                   && allocator_data->pool_size < ~(uintptr_t) 0))
1087             {
1088               allocator = omp_default_mem_alloc;
1089               goto retry;
1090             }
1091           /* Otherwise, we've already performed default mem allocation
1092              and if that failed, it won't succeed again (unless it was
1093              intermittent.  Return NULL then, as that is the fallback.  */
1094           break;
1095         case omp_atv_null_fb:
1096           break;
1097         default:
1098         case omp_atv_abort_fb:
1099           gomp_fatal ("Out of memory allocating %lu bytes",
1100                       (unsigned long) size);
1101         case omp_atv_allocator_fb:
1102           allocator = allocator_data->fb_data;
1103           goto retry;
1104         }
1105     }
1106   return NULL;
1107 }