]>
Commit | Line | Data |
---|---|---|
1 | /* Copyright (C) 2020-2024 Free Software Foundation, Inc. | |
2 | Contributed by Jakub Jelinek <jakub@redhat.com>. | |
3 | ||
4 | This file is part of the GNU Offloading and Multi Processing Library | |
5 | (libgomp). | |
6 | ||
7 | Libgomp is free software; you can redistribute it and/or modify it | |
8 | under the terms of the GNU General Public License as published by | |
9 | the Free Software Foundation; either version 3, or (at your option) | |
10 | any later version. | |
11 | ||
12 | Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
14 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
15 | more details. | |
16 | ||
17 | Under Section 7 of GPL version 3, you are granted additional | |
18 | permissions described in the GCC Runtime Library Exception, version | |
19 | 3.1, as published by the Free Software Foundation. | |
20 | ||
21 | You should have received a copy of the GNU General Public License and | |
22 | a copy of the GCC Runtime Library Exception along with this program; | |
23 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
24 | <http://www.gnu.org/licenses/>. */ | |
25 | ||
26 | /* This file contains wrappers for the system allocation routines. Most | |
27 | places in the OpenMP API do not make any provision for failure, so in | |
28 | general we cannot allow memory allocation to fail. */ | |
29 | ||
30 | #define _GNU_SOURCE | |
31 | #include "libgomp.h" | |
32 | #include <stdlib.h> | |
33 | #include <string.h> | |
34 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
35 | #include <dlfcn.h> | |
36 | #endif | |
37 | ||
38 | /* Keeping track whether a Fortran scalar allocatable/pointer has been | |
39 | allocated via 'omp allocators'/'omp allocate'. */ | |
40 | ||
41 | struct fort_alloc_splay_tree_key_s { | |
42 | void *ptr; | |
43 | }; | |
44 | ||
45 | typedef struct fort_alloc_splay_tree_node_s *fort_alloc_splay_tree_node; | |
46 | typedef struct fort_alloc_splay_tree_s *fort_alloc_splay_tree; | |
47 | typedef struct fort_alloc_splay_tree_key_s *fort_alloc_splay_tree_key; | |
48 | ||
49 | static inline int | |
50 | fort_alloc_splay_compare (fort_alloc_splay_tree_key x, fort_alloc_splay_tree_key y) | |
51 | { | |
52 | if (x->ptr < y->ptr) | |
53 | return -1; | |
54 | if (x->ptr > y->ptr) | |
55 | return 1; | |
56 | return 0; | |
57 | } | |
58 | #define splay_tree_prefix fort_alloc | |
59 | #define splay_tree_static | |
60 | #include "splay-tree.h" | |
61 | ||
62 | #define splay_tree_prefix fort_alloc | |
63 | #define splay_tree_static | |
64 | #define splay_tree_c | |
65 | #include "splay-tree.h" | |
66 | ||
67 | static struct fort_alloc_splay_tree_s fort_alloc_scalars; | |
68 | ||
69 | /* Add pointer as being alloced by GOMP_alloc. */ | |
70 | void | |
71 | GOMP_add_alloc (void *ptr) | |
72 | { | |
73 | if (ptr == NULL) | |
74 | return; | |
75 | fort_alloc_splay_tree_node item; | |
76 | item = gomp_malloc (sizeof (struct splay_tree_node_s)); | |
77 | item->key.ptr = ptr; | |
78 | item->left = NULL; | |
79 | item->right = NULL; | |
80 | fort_alloc_splay_tree_insert (&fort_alloc_scalars, item); | |
81 | } | |
82 | ||
83 | /* Remove pointer, either called by FREE or by REALLOC, | |
84 | either of them can change the allocation status. */ | |
85 | bool | |
86 | GOMP_is_alloc (void *ptr) | |
87 | { | |
88 | struct fort_alloc_splay_tree_key_s needle; | |
89 | fort_alloc_splay_tree_node n; | |
90 | needle.ptr = ptr; | |
91 | n = fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars, &needle); | |
92 | if (n) | |
93 | { | |
94 | fort_alloc_splay_tree_remove (&fort_alloc_scalars, &n->key); | |
95 | free (n); | |
96 | } | |
97 | return n != NULL; | |
98 | } | |
99 | ||
100 | ||
101 | #define omp_max_predefined_alloc omp_thread_mem_alloc | |
102 | ||
103 | /* These macros may be overridden in config/<target>/allocator.c. | |
104 | The defaults (no override) are to return NULL for pinned memory requests | |
105 | and pass through to the regular OS calls otherwise. | |
106 | The following definitions (ab)use comma operators to avoid unused | |
107 | variable errors. */ | |
108 | #ifndef MEMSPACE_ALLOC | |
109 | #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \ | |
110 | (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE)))) | |
111 | #endif | |
112 | #ifndef MEMSPACE_CALLOC | |
113 | #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \ | |
114 | (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE))))) | |
115 | #endif | |
116 | #ifndef MEMSPACE_REALLOC | |
117 | #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \ | |
118 | ((PIN) || (OLDPIN) ? NULL \ | |
119 | : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE))))) | |
120 | #endif | |
121 | #ifndef MEMSPACE_FREE | |
122 | #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \ | |
123 | if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR))) | |
124 | #endif | |
125 | #ifndef MEMSPACE_VALIDATE | |
126 | #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \ | |
127 | (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1)) | |
128 | #endif | |
129 | ||
130 | /* Map the predefined allocators to the correct memory space. | |
131 | The index to this table is the omp_allocator_handle_t enum value. | |
132 | When the user calls omp_alloc with a predefined allocator this | |
133 | table determines what memory they get. */ | |
134 | static const omp_memspace_handle_t predefined_alloc_mapping[] = { | |
135 | omp_default_mem_space, /* omp_null_allocator doesn't actually use this. */ | |
136 | omp_default_mem_space, /* omp_default_mem_alloc. */ | |
137 | omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */ | |
138 | omp_const_mem_space, /* omp_const_mem_alloc. */ | |
139 | omp_high_bw_mem_space, /* omp_high_bw_mem_alloc. */ | |
140 | omp_low_lat_mem_space, /* omp_low_lat_mem_alloc. */ | |
141 | omp_low_lat_mem_space, /* omp_cgroup_mem_alloc (implementation defined). */ | |
142 | omp_low_lat_mem_space, /* omp_pteam_mem_alloc (implementation defined). */ | |
143 | omp_low_lat_mem_space, /* omp_thread_mem_alloc (implementation defined). */ | |
144 | }; | |
145 | ||
146 | #define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0])) | |
147 | _Static_assert (ARRAY_SIZE (predefined_alloc_mapping) | |
148 | == omp_max_predefined_alloc + 1, | |
149 | "predefined_alloc_mapping must match omp_memspace_handle_t"); | |
150 | ||
151 | enum gomp_numa_memkind_kind | |
152 | { | |
153 | GOMP_MEMKIND_NONE = 0, | |
154 | #define GOMP_MEMKIND_KINDS \ | |
155 | GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \ | |
156 | GOMP_MEMKIND_KIND (HBW_PREFERRED), \ | |
157 | GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \ | |
158 | GOMP_MEMKIND_KIND (DAX_KMEM), \ | |
159 | GOMP_MEMKIND_KIND (INTERLEAVE), \ | |
160 | GOMP_MEMKIND_KIND (DEFAULT) | |
161 | #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind | |
162 | GOMP_MEMKIND_KINDS, | |
163 | #undef GOMP_MEMKIND_KIND | |
164 | GOMP_MEMKIND_COUNT, | |
165 | GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT | |
166 | }; | |
167 | ||
168 | struct omp_allocator_data | |
169 | { | |
170 | omp_memspace_handle_t memspace; | |
171 | omp_uintptr_t alignment; | |
172 | omp_uintptr_t pool_size; | |
173 | omp_uintptr_t used_pool_size; | |
174 | omp_allocator_handle_t fb_data; | |
175 | unsigned int sync_hint : 8; | |
176 | unsigned int access : 8; | |
177 | unsigned int fallback : 8; | |
178 | unsigned int pinned : 1; | |
179 | unsigned int partition : 7; | |
180 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
181 | unsigned int memkind : 8; | |
182 | #endif | |
183 | #ifndef HAVE_SYNC_BUILTINS | |
184 | gomp_mutex_t lock; | |
185 | #endif | |
186 | }; | |
187 | ||
188 | struct omp_mem_header | |
189 | { | |
190 | void *ptr; | |
191 | size_t size; | |
192 | omp_allocator_handle_t allocator; | |
193 | void *pad; | |
194 | }; | |
195 | ||
196 | struct gomp_libnuma_data | |
197 | { | |
198 | void *numa_handle; | |
199 | void *(*numa_alloc_local) (size_t); | |
200 | void *(*numa_realloc) (void *, size_t, size_t); | |
201 | void (*numa_free) (void *, size_t); | |
202 | }; | |
203 | ||
204 | struct gomp_memkind_data | |
205 | { | |
206 | void *memkind_handle; | |
207 | void *(*memkind_malloc) (void *, size_t); | |
208 | void *(*memkind_calloc) (void *, size_t, size_t); | |
209 | void *(*memkind_realloc) (void *, void *, size_t); | |
210 | void (*memkind_free) (void *, void *); | |
211 | int (*memkind_check_available) (void *); | |
212 | void **kinds[GOMP_MEMKIND_COUNT]; | |
213 | }; | |
214 | ||
215 | #ifdef LIBGOMP_USE_LIBNUMA | |
216 | static struct gomp_libnuma_data *libnuma_data; | |
217 | static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT; | |
218 | ||
219 | static void | |
220 | gomp_init_libnuma (void) | |
221 | { | |
222 | void *handle = dlopen ("libnuma.so.1", RTLD_LAZY); | |
223 | struct gomp_libnuma_data *data; | |
224 | ||
225 | data = calloc (1, sizeof (struct gomp_libnuma_data)); | |
226 | if (data == NULL) | |
227 | { | |
228 | if (handle) | |
229 | dlclose (handle); | |
230 | return; | |
231 | } | |
232 | if (handle) | |
233 | { | |
234 | int (*numa_available) (void); | |
235 | numa_available | |
236 | = (__typeof (numa_available)) dlsym (handle, "numa_available"); | |
237 | if (!numa_available || numa_available () != 0) | |
238 | { | |
239 | dlclose (handle); | |
240 | handle = NULL; | |
241 | } | |
242 | } | |
243 | if (!handle) | |
244 | { | |
245 | __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE); | |
246 | return; | |
247 | } | |
248 | data->numa_handle = handle; | |
249 | data->numa_alloc_local | |
250 | = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local"); | |
251 | data->numa_realloc | |
252 | = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc"); | |
253 | data->numa_free | |
254 | = (__typeof (data->numa_free)) dlsym (handle, "numa_free"); | |
255 | __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE); | |
256 | } | |
257 | ||
258 | static struct gomp_libnuma_data * | |
259 | gomp_get_libnuma (void) | |
260 | { | |
261 | struct gomp_libnuma_data *data | |
262 | = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE); | |
263 | if (data) | |
264 | return data; | |
265 | pthread_once (&libnuma_data_once, gomp_init_libnuma); | |
266 | return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE); | |
267 | } | |
268 | #endif | |
269 | ||
270 | #ifdef LIBGOMP_USE_MEMKIND | |
271 | static struct gomp_memkind_data *memkind_data; | |
272 | static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT; | |
273 | ||
274 | static void | |
275 | gomp_init_memkind (void) | |
276 | { | |
277 | void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY); | |
278 | struct gomp_memkind_data *data; | |
279 | int i; | |
280 | static const char *kinds[] = { | |
281 | NULL, | |
282 | #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind | |
283 | GOMP_MEMKIND_KINDS | |
284 | #undef GOMP_MEMKIND_KIND | |
285 | }; | |
286 | ||
287 | data = calloc (1, sizeof (struct gomp_memkind_data)); | |
288 | if (data == NULL) | |
289 | { | |
290 | if (handle) | |
291 | dlclose (handle); | |
292 | return; | |
293 | } | |
294 | if (!handle) | |
295 | { | |
296 | __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE); | |
297 | return; | |
298 | } | |
299 | data->memkind_handle = handle; | |
300 | data->memkind_malloc | |
301 | = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc"); | |
302 | data->memkind_calloc | |
303 | = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc"); | |
304 | data->memkind_realloc | |
305 | = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc"); | |
306 | data->memkind_free | |
307 | = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free"); | |
308 | data->memkind_check_available | |
309 | = (__typeof (data->memkind_check_available)) | |
310 | dlsym (handle, "memkind_check_available"); | |
311 | if (data->memkind_malloc | |
312 | && data->memkind_calloc | |
313 | && data->memkind_realloc | |
314 | && data->memkind_free | |
315 | && data->memkind_check_available) | |
316 | for (i = 1; i < GOMP_MEMKIND_COUNT; ++i) | |
317 | { | |
318 | data->kinds[i] = (void **) dlsym (handle, kinds[i]); | |
319 | if (data->kinds[i] && data->memkind_check_available (*data->kinds[i])) | |
320 | data->kinds[i] = NULL; | |
321 | } | |
322 | __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE); | |
323 | } | |
324 | ||
325 | static struct gomp_memkind_data * | |
326 | gomp_get_memkind (void) | |
327 | { | |
328 | struct gomp_memkind_data *data | |
329 | = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE); | |
330 | if (data) | |
331 | return data; | |
332 | pthread_once (&memkind_data_once, gomp_init_memkind); | |
333 | return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE); | |
334 | } | |
335 | #endif | |
336 | ||
337 | omp_allocator_handle_t | |
338 | omp_init_allocator (omp_memspace_handle_t memspace, int ntraits, | |
339 | const omp_alloctrait_t traits[]) | |
340 | { | |
341 | struct omp_allocator_data data | |
342 | = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all, | |
343 | omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment, | |
344 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
345 | GOMP_MEMKIND_NONE | |
346 | #endif | |
347 | }; | |
348 | struct omp_allocator_data *ret; | |
349 | int i; | |
350 | ||
351 | if (memspace > omp_low_lat_mem_space) | |
352 | return omp_null_allocator; | |
353 | for (i = 0; i < ntraits; i++) | |
354 | switch (traits[i].key) | |
355 | { | |
356 | case omp_atk_sync_hint: | |
357 | switch (traits[i].value) | |
358 | { | |
359 | case omp_atv_default: | |
360 | data.sync_hint = omp_atv_contended; | |
361 | break; | |
362 | case omp_atv_contended: | |
363 | case omp_atv_uncontended: | |
364 | case omp_atv_serialized: | |
365 | case omp_atv_private: | |
366 | data.sync_hint = traits[i].value; | |
367 | break; | |
368 | default: | |
369 | return omp_null_allocator; | |
370 | } | |
371 | break; | |
372 | case omp_atk_alignment: | |
373 | if (traits[i].value == omp_atv_default) | |
374 | { | |
375 | data.alignment = 1; | |
376 | break; | |
377 | } | |
378 | if ((traits[i].value & (traits[i].value - 1)) != 0 | |
379 | || !traits[i].value) | |
380 | return omp_null_allocator; | |
381 | data.alignment = traits[i].value; | |
382 | break; | |
383 | case omp_atk_access: | |
384 | switch (traits[i].value) | |
385 | { | |
386 | case omp_atv_default: | |
387 | data.access = omp_atv_all; | |
388 | break; | |
389 | case omp_atv_all: | |
390 | case omp_atv_cgroup: | |
391 | case omp_atv_pteam: | |
392 | case omp_atv_thread: | |
393 | data.access = traits[i].value; | |
394 | break; | |
395 | default: | |
396 | return omp_null_allocator; | |
397 | } | |
398 | break; | |
399 | case omp_atk_pool_size: | |
400 | if (traits[i].value == omp_atv_default) | |
401 | data.pool_size = ~(uintptr_t) 0; | |
402 | else | |
403 | data.pool_size = traits[i].value; | |
404 | break; | |
405 | case omp_atk_fallback: | |
406 | switch (traits[i].value) | |
407 | { | |
408 | case omp_atv_default: | |
409 | data.fallback = omp_atv_default_mem_fb; | |
410 | break; | |
411 | case omp_atv_default_mem_fb: | |
412 | case omp_atv_null_fb: | |
413 | case omp_atv_abort_fb: | |
414 | case omp_atv_allocator_fb: | |
415 | data.fallback = traits[i].value; | |
416 | break; | |
417 | default: | |
418 | return omp_null_allocator; | |
419 | } | |
420 | break; | |
421 | case omp_atk_fb_data: | |
422 | data.fb_data = traits[i].value; | |
423 | break; | |
424 | case omp_atk_pinned: | |
425 | switch (traits[i].value) | |
426 | { | |
427 | case omp_atv_default: | |
428 | case omp_atv_false: | |
429 | data.pinned = omp_atv_false; | |
430 | break; | |
431 | case omp_atv_true: | |
432 | data.pinned = omp_atv_true; | |
433 | break; | |
434 | default: | |
435 | return omp_null_allocator; | |
436 | } | |
437 | break; | |
438 | case omp_atk_partition: | |
439 | switch (traits[i].value) | |
440 | { | |
441 | case omp_atv_default: | |
442 | data.partition = omp_atv_environment; | |
443 | break; | |
444 | case omp_atv_environment: | |
445 | case omp_atv_nearest: | |
446 | case omp_atv_blocked: | |
447 | case omp_atv_interleaved: | |
448 | data.partition = traits[i].value; | |
449 | break; | |
450 | default: | |
451 | return omp_null_allocator; | |
452 | } | |
453 | break; | |
454 | default: | |
455 | return omp_null_allocator; | |
456 | } | |
457 | ||
458 | if (data.alignment < sizeof (void *)) | |
459 | data.alignment = sizeof (void *); | |
460 | ||
461 | switch (memspace) | |
462 | { | |
463 | #ifdef LIBGOMP_USE_MEMKIND | |
464 | case omp_high_bw_mem_space: | |
465 | struct gomp_memkind_data *memkind_data; | |
466 | memkind_data = gomp_get_memkind (); | |
467 | if (data.partition == omp_atv_interleaved | |
468 | && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE]) | |
469 | { | |
470 | data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE; | |
471 | break; | |
472 | } | |
473 | else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED]) | |
474 | { | |
475 | data.memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
476 | break; | |
477 | } | |
478 | break; | |
479 | case omp_large_cap_mem_space: | |
480 | memkind_data = gomp_get_memkind (); | |
481 | if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL]) | |
482 | data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
483 | else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM]) | |
484 | data.memkind = GOMP_MEMKIND_DAX_KMEM; | |
485 | break; | |
486 | #endif | |
487 | default: | |
488 | #ifdef LIBGOMP_USE_MEMKIND | |
489 | if (data.partition == omp_atv_interleaved) | |
490 | { | |
491 | memkind_data = gomp_get_memkind (); | |
492 | if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE]) | |
493 | data.memkind = GOMP_MEMKIND_INTERLEAVE; | |
494 | } | |
495 | #endif | |
496 | break; | |
497 | } | |
498 | ||
499 | #ifdef LIBGOMP_USE_LIBNUMA | |
500 | if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest) | |
501 | { | |
502 | libnuma_data = gomp_get_libnuma (); | |
503 | if (libnuma_data->numa_alloc_local != NULL) | |
504 | data.memkind = GOMP_MEMKIND_LIBNUMA; | |
505 | } | |
506 | #endif | |
507 | ||
508 | /* Reject unsupported memory spaces. */ | |
509 | if (!MEMSPACE_VALIDATE (data.memspace, data.access, data.pinned)) | |
510 | return omp_null_allocator; | |
511 | ||
512 | ret = gomp_malloc (sizeof (struct omp_allocator_data)); | |
513 | *ret = data; | |
514 | #ifndef HAVE_SYNC_BUILTINS | |
515 | gomp_mutex_init (&ret->lock); | |
516 | #endif | |
517 | return (omp_allocator_handle_t) ret; | |
518 | } | |
519 | ||
520 | void | |
521 | omp_destroy_allocator (omp_allocator_handle_t allocator) | |
522 | { | |
523 | if (allocator != omp_null_allocator) | |
524 | { | |
525 | #ifndef HAVE_SYNC_BUILTINS | |
526 | gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock); | |
527 | #endif | |
528 | free ((void *) allocator); | |
529 | } | |
530 | } | |
531 | ||
532 | ialias (omp_init_allocator) | |
533 | ialias (omp_destroy_allocator) | |
534 | ||
535 | void * | |
536 | omp_aligned_alloc (size_t alignment, size_t size, | |
537 | omp_allocator_handle_t allocator) | |
538 | { | |
539 | struct omp_allocator_data *allocator_data; | |
540 | size_t new_size, new_alignment; | |
541 | void *ptr, *ret; | |
542 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
543 | enum gomp_numa_memkind_kind memkind; | |
544 | #endif | |
545 | ||
546 | if (__builtin_expect (size == 0, 0)) | |
547 | return NULL; | |
548 | ||
549 | retry: | |
550 | new_alignment = alignment; | |
551 | if (allocator == omp_null_allocator) | |
552 | { | |
553 | struct gomp_thread *thr = gomp_thread (); | |
554 | if (thr->ts.def_allocator == omp_null_allocator) | |
555 | thr->ts.def_allocator = gomp_def_allocator; | |
556 | allocator = (omp_allocator_handle_t) thr->ts.def_allocator; | |
557 | } | |
558 | ||
559 | if (allocator > omp_max_predefined_alloc) | |
560 | { | |
561 | allocator_data = (struct omp_allocator_data *) allocator; | |
562 | if (new_alignment < allocator_data->alignment) | |
563 | new_alignment = allocator_data->alignment; | |
564 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
565 | memkind = allocator_data->memkind; | |
566 | #endif | |
567 | } | |
568 | else | |
569 | { | |
570 | allocator_data = NULL; | |
571 | if (new_alignment < sizeof (void *)) | |
572 | new_alignment = sizeof (void *); | |
573 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
574 | memkind = GOMP_MEMKIND_NONE; | |
575 | #endif | |
576 | #ifdef LIBGOMP_USE_MEMKIND | |
577 | if (allocator == omp_high_bw_mem_alloc) | |
578 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
579 | else if (allocator == omp_large_cap_mem_alloc) | |
580 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
581 | if (memkind) | |
582 | { | |
583 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
584 | if (!memkind_data->kinds[memkind]) | |
585 | memkind = GOMP_MEMKIND_NONE; | |
586 | } | |
587 | #endif | |
588 | } | |
589 | ||
590 | new_size = sizeof (struct omp_mem_header); | |
591 | if (new_alignment > sizeof (void *)) | |
592 | new_size += new_alignment - sizeof (void *); | |
593 | if (__builtin_add_overflow (size, new_size, &new_size)) | |
594 | goto fail; | |
595 | #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID | |
596 | if (allocator == omp_low_lat_mem_alloc) | |
597 | goto fail; | |
598 | #endif | |
599 | ||
600 | if (__builtin_expect (allocator_data | |
601 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
602 | { | |
603 | uintptr_t used_pool_size; | |
604 | if (new_size > allocator_data->pool_size) | |
605 | goto fail; | |
606 | #ifdef HAVE_SYNC_BUILTINS | |
607 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
608 | MEMMODEL_RELAXED); | |
609 | do | |
610 | { | |
611 | uintptr_t new_pool_size; | |
612 | if (__builtin_add_overflow (used_pool_size, new_size, | |
613 | &new_pool_size) | |
614 | || new_pool_size > allocator_data->pool_size) | |
615 | goto fail; | |
616 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
617 | &used_pool_size, new_pool_size, | |
618 | true, MEMMODEL_RELAXED, | |
619 | MEMMODEL_RELAXED)) | |
620 | break; | |
621 | } | |
622 | while (1); | |
623 | #else | |
624 | gomp_mutex_lock (&allocator_data->lock); | |
625 | if (__builtin_add_overflow (allocator_data->used_pool_size, new_size, | |
626 | &used_pool_size) | |
627 | || used_pool_size > allocator_data->pool_size) | |
628 | { | |
629 | gomp_mutex_unlock (&allocator_data->lock); | |
630 | goto fail; | |
631 | } | |
632 | allocator_data->used_pool_size = used_pool_size; | |
633 | gomp_mutex_unlock (&allocator_data->lock); | |
634 | #endif | |
635 | #ifdef LIBGOMP_USE_LIBNUMA | |
636 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
637 | ptr = libnuma_data->numa_alloc_local (new_size); | |
638 | # ifdef LIBGOMP_USE_MEMKIND | |
639 | else | |
640 | # endif | |
641 | #endif | |
642 | #ifdef LIBGOMP_USE_MEMKIND | |
643 | if (memkind) | |
644 | { | |
645 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
646 | void *kind = *memkind_data->kinds[memkind]; | |
647 | ptr = memkind_data->memkind_malloc (kind, new_size); | |
648 | } | |
649 | else | |
650 | #endif | |
651 | ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size, | |
652 | allocator_data->pinned); | |
653 | if (ptr == NULL) | |
654 | { | |
655 | #ifdef HAVE_SYNC_BUILTINS | |
656 | __atomic_add_fetch (&allocator_data->used_pool_size, -new_size, | |
657 | MEMMODEL_RELAXED); | |
658 | #else | |
659 | gomp_mutex_lock (&allocator_data->lock); | |
660 | allocator_data->used_pool_size -= new_size; | |
661 | gomp_mutex_unlock (&allocator_data->lock); | |
662 | #endif | |
663 | goto fail; | |
664 | } | |
665 | } | |
666 | else | |
667 | { | |
668 | #ifdef LIBGOMP_USE_LIBNUMA | |
669 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
670 | ptr = libnuma_data->numa_alloc_local (new_size); | |
671 | # ifdef LIBGOMP_USE_MEMKIND | |
672 | else | |
673 | # endif | |
674 | #endif | |
675 | #ifdef LIBGOMP_USE_MEMKIND | |
676 | if (memkind) | |
677 | { | |
678 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
679 | void *kind = *memkind_data->kinds[memkind]; | |
680 | ptr = memkind_data->memkind_malloc (kind, new_size); | |
681 | } | |
682 | else | |
683 | #endif | |
684 | { | |
685 | omp_memspace_handle_t memspace; | |
686 | memspace = (allocator_data | |
687 | ? allocator_data->memspace | |
688 | : predefined_alloc_mapping[allocator]); | |
689 | ptr = MEMSPACE_ALLOC (memspace, new_size, | |
690 | allocator_data && allocator_data->pinned); | |
691 | } | |
692 | if (ptr == NULL) | |
693 | goto fail; | |
694 | } | |
695 | ||
696 | if (new_alignment > sizeof (void *)) | |
697 | ret = (void *) (((uintptr_t) ptr | |
698 | + sizeof (struct omp_mem_header) | |
699 | + new_alignment - sizeof (void *)) | |
700 | & ~(new_alignment - 1)); | |
701 | else | |
702 | ret = (char *) ptr + sizeof (struct omp_mem_header); | |
703 | ((struct omp_mem_header *) ret)[-1].ptr = ptr; | |
704 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
705 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
706 | return ret; | |
707 | ||
708 | fail:; | |
709 | int fallback = (allocator_data | |
710 | ? allocator_data->fallback | |
711 | : allocator == omp_default_mem_alloc | |
712 | ? omp_atv_null_fb | |
713 | : omp_atv_default_mem_fb); | |
714 | switch (fallback) | |
715 | { | |
716 | case omp_atv_default_mem_fb: | |
717 | allocator = omp_default_mem_alloc; | |
718 | goto retry; | |
719 | case omp_atv_null_fb: | |
720 | break; | |
721 | default: | |
722 | case omp_atv_abort_fb: | |
723 | gomp_fatal ("Out of memory allocating %lu bytes", | |
724 | (unsigned long) size); | |
725 | case omp_atv_allocator_fb: | |
726 | allocator = allocator_data->fb_data; | |
727 | goto retry; | |
728 | } | |
729 | return NULL; | |
730 | } | |
731 | ||
732 | ialias (omp_aligned_alloc) | |
733 | ||
734 | void * | |
735 | omp_alloc (size_t size, omp_allocator_handle_t allocator) | |
736 | { | |
737 | return ialias_call (omp_aligned_alloc) (1, size, allocator); | |
738 | } | |
739 | ||
740 | /* Like omp_aligned_alloc, but apply on top of that: | |
741 | "For allocations that arise from this ... the null_fb value of the | |
742 | fallback allocator trait behaves as if the abort_fb had been specified." */ | |
743 | ||
744 | void * | |
745 | GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator) | |
746 | { | |
747 | void *ret | |
748 | = ialias_call (omp_aligned_alloc) (alignment, size, | |
749 | (omp_allocator_handle_t) allocator); | |
750 | if (__builtin_expect (ret == NULL, 0) && size) | |
751 | gomp_fatal ("Out of memory allocating %lu bytes", | |
752 | (unsigned long) size); | |
753 | return ret; | |
754 | } | |
755 | ||
756 | void | |
757 | omp_free (void *ptr, omp_allocator_handle_t allocator) | |
758 | { | |
759 | struct omp_mem_header *data; | |
760 | omp_memspace_handle_t memspace = omp_default_mem_space; | |
761 | int pinned = false; | |
762 | ||
763 | if (ptr == NULL) | |
764 | return; | |
765 | (void) allocator; | |
766 | data = &((struct omp_mem_header *) ptr)[-1]; | |
767 | if (data->allocator > omp_max_predefined_alloc) | |
768 | { | |
769 | struct omp_allocator_data *allocator_data | |
770 | = (struct omp_allocator_data *) (data->allocator); | |
771 | if (allocator_data->pool_size < ~(uintptr_t) 0) | |
772 | { | |
773 | #ifdef HAVE_SYNC_BUILTINS | |
774 | __atomic_add_fetch (&allocator_data->used_pool_size, -data->size, | |
775 | MEMMODEL_RELAXED); | |
776 | #else | |
777 | gomp_mutex_lock (&allocator_data->lock); | |
778 | allocator_data->used_pool_size -= data->size; | |
779 | gomp_mutex_unlock (&allocator_data->lock); | |
780 | #endif | |
781 | } | |
782 | #ifdef LIBGOMP_USE_LIBNUMA | |
783 | if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA) | |
784 | { | |
785 | libnuma_data->numa_free (data->ptr, data->size); | |
786 | return; | |
787 | } | |
788 | # ifdef LIBGOMP_USE_MEMKIND | |
789 | else | |
790 | # endif | |
791 | #endif | |
792 | #ifdef LIBGOMP_USE_MEMKIND | |
793 | if (allocator_data->memkind) | |
794 | { | |
795 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
796 | void *kind = *memkind_data->kinds[allocator_data->memkind]; | |
797 | memkind_data->memkind_free (kind, data->ptr); | |
798 | return; | |
799 | } | |
800 | #endif | |
801 | ||
802 | memspace = allocator_data->memspace; | |
803 | pinned = allocator_data->pinned; | |
804 | } | |
805 | else | |
806 | { | |
807 | #ifdef LIBGOMP_USE_MEMKIND | |
808 | enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE; | |
809 | if (data->allocator == omp_high_bw_mem_alloc) | |
810 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
811 | else if (data->allocator == omp_large_cap_mem_alloc) | |
812 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
813 | if (memkind) | |
814 | { | |
815 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
816 | if (memkind_data->kinds[memkind]) | |
817 | { | |
818 | void *kind = *memkind_data->kinds[memkind]; | |
819 | memkind_data->memkind_free (kind, data->ptr); | |
820 | return; | |
821 | } | |
822 | } | |
823 | #endif | |
824 | ||
825 | memspace = predefined_alloc_mapping[data->allocator]; | |
826 | } | |
827 | ||
828 | MEMSPACE_FREE (memspace, data->ptr, data->size, pinned); | |
829 | } | |
830 | ||
831 | ialias (omp_free) | |
832 | ||
833 | void | |
834 | GOMP_free (void *ptr, uintptr_t allocator) | |
835 | { | |
836 | return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator); | |
837 | } | |
838 | ||
839 | void * | |
840 | omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size, | |
841 | omp_allocator_handle_t allocator) | |
842 | { | |
843 | struct omp_allocator_data *allocator_data; | |
844 | size_t new_size, size_temp, new_alignment; | |
845 | void *ptr, *ret; | |
846 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
847 | enum gomp_numa_memkind_kind memkind; | |
848 | #endif | |
849 | ||
850 | if (__builtin_expect (size == 0 || nmemb == 0, 0)) | |
851 | return NULL; | |
852 | ||
853 | retry: | |
854 | new_alignment = alignment; | |
855 | if (allocator == omp_null_allocator) | |
856 | { | |
857 | struct gomp_thread *thr = gomp_thread (); | |
858 | if (thr->ts.def_allocator == omp_null_allocator) | |
859 | thr->ts.def_allocator = gomp_def_allocator; | |
860 | allocator = (omp_allocator_handle_t) thr->ts.def_allocator; | |
861 | } | |
862 | ||
863 | if (allocator > omp_max_predefined_alloc) | |
864 | { | |
865 | allocator_data = (struct omp_allocator_data *) allocator; | |
866 | if (new_alignment < allocator_data->alignment) | |
867 | new_alignment = allocator_data->alignment; | |
868 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
869 | memkind = allocator_data->memkind; | |
870 | #endif | |
871 | } | |
872 | else | |
873 | { | |
874 | allocator_data = NULL; | |
875 | if (new_alignment < sizeof (void *)) | |
876 | new_alignment = sizeof (void *); | |
877 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
878 | memkind = GOMP_MEMKIND_NONE; | |
879 | #endif | |
880 | #ifdef LIBGOMP_USE_MEMKIND | |
881 | if (allocator == omp_high_bw_mem_alloc) | |
882 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
883 | else if (allocator == omp_large_cap_mem_alloc) | |
884 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
885 | if (memkind) | |
886 | { | |
887 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
888 | if (!memkind_data->kinds[memkind]) | |
889 | memkind = GOMP_MEMKIND_NONE; | |
890 | } | |
891 | #endif | |
892 | } | |
893 | ||
894 | new_size = sizeof (struct omp_mem_header); | |
895 | if (new_alignment > sizeof (void *)) | |
896 | new_size += new_alignment - sizeof (void *); | |
897 | if (__builtin_mul_overflow (size, nmemb, &size_temp)) | |
898 | goto fail; | |
899 | if (__builtin_add_overflow (size_temp, new_size, &new_size)) | |
900 | goto fail; | |
901 | #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID | |
902 | if (allocator == omp_low_lat_mem_alloc) | |
903 | goto fail; | |
904 | #endif | |
905 | ||
906 | if (__builtin_expect (allocator_data | |
907 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
908 | { | |
909 | uintptr_t used_pool_size; | |
910 | if (new_size > allocator_data->pool_size) | |
911 | goto fail; | |
912 | #ifdef HAVE_SYNC_BUILTINS | |
913 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
914 | MEMMODEL_RELAXED); | |
915 | do | |
916 | { | |
917 | uintptr_t new_pool_size; | |
918 | if (__builtin_add_overflow (used_pool_size, new_size, | |
919 | &new_pool_size) | |
920 | || new_pool_size > allocator_data->pool_size) | |
921 | goto fail; | |
922 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
923 | &used_pool_size, new_pool_size, | |
924 | true, MEMMODEL_RELAXED, | |
925 | MEMMODEL_RELAXED)) | |
926 | break; | |
927 | } | |
928 | while (1); | |
929 | #else | |
930 | gomp_mutex_lock (&allocator_data->lock); | |
931 | if (__builtin_add_overflow (allocator_data->used_pool_size, new_size, | |
932 | &used_pool_size) | |
933 | || used_pool_size > allocator_data->pool_size) | |
934 | { | |
935 | gomp_mutex_unlock (&allocator_data->lock); | |
936 | goto fail; | |
937 | } | |
938 | allocator_data->used_pool_size = used_pool_size; | |
939 | gomp_mutex_unlock (&allocator_data->lock); | |
940 | #endif | |
941 | #ifdef LIBGOMP_USE_LIBNUMA | |
942 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
943 | /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning | |
944 | memory that is initialized to zero. */ | |
945 | ptr = libnuma_data->numa_alloc_local (new_size); | |
946 | # ifdef LIBGOMP_USE_MEMKIND | |
947 | else | |
948 | # endif | |
949 | #endif | |
950 | #ifdef LIBGOMP_USE_MEMKIND | |
951 | if (memkind) | |
952 | { | |
953 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
954 | void *kind = *memkind_data->kinds[memkind]; | |
955 | ptr = memkind_data->memkind_calloc (kind, 1, new_size); | |
956 | } | |
957 | else | |
958 | #endif | |
959 | ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size, | |
960 | allocator_data->pinned); | |
961 | if (ptr == NULL) | |
962 | { | |
963 | #ifdef HAVE_SYNC_BUILTINS | |
964 | __atomic_add_fetch (&allocator_data->used_pool_size, -new_size, | |
965 | MEMMODEL_RELAXED); | |
966 | #else | |
967 | gomp_mutex_lock (&allocator_data->lock); | |
968 | allocator_data->used_pool_size -= new_size; | |
969 | gomp_mutex_unlock (&allocator_data->lock); | |
970 | #endif | |
971 | goto fail; | |
972 | } | |
973 | } | |
974 | else | |
975 | { | |
976 | #ifdef LIBGOMP_USE_LIBNUMA | |
977 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
978 | /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning | |
979 | memory that is initialized to zero. */ | |
980 | ptr = libnuma_data->numa_alloc_local (new_size); | |
981 | # ifdef LIBGOMP_USE_MEMKIND | |
982 | else | |
983 | # endif | |
984 | #endif | |
985 | #ifdef LIBGOMP_USE_MEMKIND | |
986 | if (memkind) | |
987 | { | |
988 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
989 | void *kind = *memkind_data->kinds[memkind]; | |
990 | ptr = memkind_data->memkind_calloc (kind, 1, new_size); | |
991 | } | |
992 | else | |
993 | #endif | |
994 | { | |
995 | omp_memspace_handle_t memspace; | |
996 | memspace = (allocator_data | |
997 | ? allocator_data->memspace | |
998 | : predefined_alloc_mapping[allocator]); | |
999 | ptr = MEMSPACE_CALLOC (memspace, new_size, | |
1000 | allocator_data && allocator_data->pinned); | |
1001 | } | |
1002 | if (ptr == NULL) | |
1003 | goto fail; | |
1004 | } | |
1005 | ||
1006 | if (new_alignment > sizeof (void *)) | |
1007 | ret = (void *) (((uintptr_t) ptr | |
1008 | + sizeof (struct omp_mem_header) | |
1009 | + new_alignment - sizeof (void *)) | |
1010 | & ~(new_alignment - 1)); | |
1011 | else | |
1012 | ret = (char *) ptr + sizeof (struct omp_mem_header); | |
1013 | ((struct omp_mem_header *) ret)[-1].ptr = ptr; | |
1014 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1015 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1016 | return ret; | |
1017 | ||
1018 | fail:; | |
1019 | int fallback = (allocator_data | |
1020 | ? allocator_data->fallback | |
1021 | : allocator == omp_default_mem_alloc | |
1022 | ? omp_atv_null_fb | |
1023 | : omp_atv_default_mem_fb); | |
1024 | switch (fallback) | |
1025 | { | |
1026 | case omp_atv_default_mem_fb: | |
1027 | allocator = omp_default_mem_alloc; | |
1028 | goto retry; | |
1029 | case omp_atv_null_fb: | |
1030 | break; | |
1031 | default: | |
1032 | case omp_atv_abort_fb: | |
1033 | gomp_fatal ("Out of memory allocating %lu bytes", | |
1034 | (unsigned long) (size * nmemb)); | |
1035 | case omp_atv_allocator_fb: | |
1036 | allocator = allocator_data->fb_data; | |
1037 | goto retry; | |
1038 | } | |
1039 | return NULL; | |
1040 | } | |
1041 | ||
1042 | ialias (omp_aligned_calloc) | |
1043 | ||
1044 | void * | |
1045 | omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator) | |
1046 | { | |
1047 | return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator); | |
1048 | } | |
1049 | ||
1050 | void * | |
1051 | omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator, | |
1052 | omp_allocator_handle_t free_allocator) | |
1053 | { | |
1054 | struct omp_allocator_data *allocator_data, *free_allocator_data; | |
1055 | size_t new_size, old_size, new_alignment, old_alignment; | |
1056 | void *new_ptr, *ret; | |
1057 | struct omp_mem_header *data; | |
1058 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1059 | enum gomp_numa_memkind_kind memkind, free_memkind; | |
1060 | #endif | |
1061 | ||
1062 | if (__builtin_expect (ptr == NULL, 0)) | |
1063 | return ialias_call (omp_aligned_alloc) (1, size, allocator); | |
1064 | ||
1065 | if (__builtin_expect (size == 0, 0)) | |
1066 | { | |
1067 | ialias_call (omp_free) (ptr, free_allocator); | |
1068 | return NULL; | |
1069 | } | |
1070 | ||
1071 | data = &((struct omp_mem_header *) ptr)[-1]; | |
1072 | free_allocator = data->allocator; | |
1073 | ||
1074 | retry: | |
1075 | new_alignment = sizeof (void *); | |
1076 | if (allocator == omp_null_allocator) | |
1077 | allocator = free_allocator; | |
1078 | ||
1079 | if (allocator > omp_max_predefined_alloc) | |
1080 | { | |
1081 | allocator_data = (struct omp_allocator_data *) allocator; | |
1082 | if (new_alignment < allocator_data->alignment) | |
1083 | new_alignment = allocator_data->alignment; | |
1084 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1085 | memkind = allocator_data->memkind; | |
1086 | #endif | |
1087 | } | |
1088 | else | |
1089 | { | |
1090 | allocator_data = NULL; | |
1091 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1092 | memkind = GOMP_MEMKIND_NONE; | |
1093 | #endif | |
1094 | #ifdef LIBGOMP_USE_MEMKIND | |
1095 | if (allocator == omp_high_bw_mem_alloc) | |
1096 | memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
1097 | else if (allocator == omp_large_cap_mem_alloc) | |
1098 | memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
1099 | if (memkind) | |
1100 | { | |
1101 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1102 | if (!memkind_data->kinds[memkind]) | |
1103 | memkind = GOMP_MEMKIND_NONE; | |
1104 | } | |
1105 | #endif | |
1106 | } | |
1107 | if (free_allocator > omp_max_predefined_alloc) | |
1108 | { | |
1109 | free_allocator_data = (struct omp_allocator_data *) free_allocator; | |
1110 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1111 | free_memkind = free_allocator_data->memkind; | |
1112 | #endif | |
1113 | } | |
1114 | else | |
1115 | { | |
1116 | free_allocator_data = NULL; | |
1117 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1118 | free_memkind = GOMP_MEMKIND_NONE; | |
1119 | #endif | |
1120 | #ifdef LIBGOMP_USE_MEMKIND | |
1121 | if (free_allocator == omp_high_bw_mem_alloc) | |
1122 | free_memkind = GOMP_MEMKIND_HBW_PREFERRED; | |
1123 | else if (free_allocator == omp_large_cap_mem_alloc) | |
1124 | free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL; | |
1125 | if (free_memkind) | |
1126 | { | |
1127 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1128 | if (!memkind_data->kinds[free_memkind]) | |
1129 | free_memkind = GOMP_MEMKIND_NONE; | |
1130 | } | |
1131 | #endif | |
1132 | } | |
1133 | old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr); | |
1134 | ||
1135 | new_size = sizeof (struct omp_mem_header); | |
1136 | if (new_alignment > sizeof (void *)) | |
1137 | new_size += new_alignment - sizeof (void *); | |
1138 | if (__builtin_add_overflow (size, new_size, &new_size)) | |
1139 | goto fail; | |
1140 | old_size = data->size; | |
1141 | #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID | |
1142 | if (allocator == omp_low_lat_mem_alloc) | |
1143 | goto fail; | |
1144 | #endif | |
1145 | ||
1146 | if (__builtin_expect (allocator_data | |
1147 | && allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
1148 | { | |
1149 | uintptr_t used_pool_size; | |
1150 | size_t prev_size = 0; | |
1151 | /* Check if we can use realloc. Don't use it if extra alignment | |
1152 | was used previously or newly, because realloc might return a pointer | |
1153 | with different alignment and then we'd need to memmove the data | |
1154 | again. */ | |
1155 | if (free_allocator_data | |
1156 | && free_allocator_data == allocator_data | |
1157 | && new_alignment == sizeof (void *) | |
1158 | && old_alignment == sizeof (struct omp_mem_header)) | |
1159 | prev_size = old_size; | |
1160 | if (new_size > prev_size | |
1161 | && new_size - prev_size > allocator_data->pool_size) | |
1162 | goto fail; | |
1163 | #ifdef HAVE_SYNC_BUILTINS | |
1164 | used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, | |
1165 | MEMMODEL_RELAXED); | |
1166 | do | |
1167 | { | |
1168 | uintptr_t new_pool_size; | |
1169 | if (new_size > prev_size) | |
1170 | { | |
1171 | if (__builtin_add_overflow (used_pool_size, new_size - prev_size, | |
1172 | &new_pool_size) | |
1173 | || new_pool_size > allocator_data->pool_size) | |
1174 | goto fail; | |
1175 | } | |
1176 | else | |
1177 | new_pool_size = used_pool_size + new_size - prev_size; | |
1178 | if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, | |
1179 | &used_pool_size, new_pool_size, | |
1180 | true, MEMMODEL_RELAXED, | |
1181 | MEMMODEL_RELAXED)) | |
1182 | break; | |
1183 | } | |
1184 | while (1); | |
1185 | #else | |
1186 | gomp_mutex_lock (&allocator_data->lock); | |
1187 | if (new_size > prev_size) | |
1188 | { | |
1189 | if (__builtin_add_overflow (allocator_data->used_pool_size, | |
1190 | new_size - prev_size, | |
1191 | &used_pool_size) | |
1192 | || used_pool_size > allocator_data->pool_size) | |
1193 | { | |
1194 | gomp_mutex_unlock (&allocator_data->lock); | |
1195 | goto fail; | |
1196 | } | |
1197 | } | |
1198 | else | |
1199 | used_pool_size = (allocator_data->used_pool_size | |
1200 | + new_size - prev_size); | |
1201 | allocator_data->used_pool_size = used_pool_size; | |
1202 | gomp_mutex_unlock (&allocator_data->lock); | |
1203 | #endif | |
1204 | #ifdef LIBGOMP_USE_LIBNUMA | |
1205 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
1206 | { | |
1207 | if (prev_size) | |
1208 | new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, | |
1209 | new_size); | |
1210 | else | |
1211 | new_ptr = libnuma_data->numa_alloc_local (new_size); | |
1212 | } | |
1213 | # ifdef LIBGOMP_USE_MEMKIND | |
1214 | else | |
1215 | # endif | |
1216 | #endif | |
1217 | #ifdef LIBGOMP_USE_MEMKIND | |
1218 | if (memkind) | |
1219 | { | |
1220 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1221 | void *kind = *memkind_data->kinds[memkind]; | |
1222 | if (prev_size) | |
1223 | new_ptr = memkind_data->memkind_realloc (kind, data->ptr, | |
1224 | new_size); | |
1225 | else | |
1226 | new_ptr = memkind_data->memkind_malloc (kind, new_size); | |
1227 | } | |
1228 | else | |
1229 | #endif | |
1230 | if (prev_size) | |
1231 | new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr, | |
1232 | data->size, new_size, | |
1233 | (free_allocator_data | |
1234 | && free_allocator_data->pinned), | |
1235 | allocator_data->pinned); | |
1236 | else | |
1237 | new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size, | |
1238 | allocator_data->pinned); | |
1239 | if (new_ptr == NULL) | |
1240 | { | |
1241 | #ifdef HAVE_SYNC_BUILTINS | |
1242 | __atomic_add_fetch (&allocator_data->used_pool_size, | |
1243 | prev_size - new_size, | |
1244 | MEMMODEL_RELAXED); | |
1245 | #else | |
1246 | gomp_mutex_lock (&allocator_data->lock); | |
1247 | allocator_data->used_pool_size -= new_size - prev_size; | |
1248 | gomp_mutex_unlock (&allocator_data->lock); | |
1249 | #endif | |
1250 | goto fail; | |
1251 | } | |
1252 | else if (prev_size) | |
1253 | { | |
1254 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
1255 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
1256 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1257 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1258 | return ret; | |
1259 | } | |
1260 | } | |
1261 | else if (new_alignment == sizeof (void *) | |
1262 | && old_alignment == sizeof (struct omp_mem_header) | |
1263 | #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA) | |
1264 | && memkind == free_memkind | |
1265 | #endif | |
1266 | && (free_allocator_data == NULL | |
1267 | || free_allocator_data->pool_size == ~(uintptr_t) 0)) | |
1268 | { | |
1269 | #ifdef LIBGOMP_USE_LIBNUMA | |
1270 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
1271 | new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size); | |
1272 | # ifdef LIBGOMP_USE_MEMKIND | |
1273 | else | |
1274 | # endif | |
1275 | #endif | |
1276 | #ifdef LIBGOMP_USE_MEMKIND | |
1277 | if (memkind) | |
1278 | { | |
1279 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1280 | void *kind = *memkind_data->kinds[memkind]; | |
1281 | new_ptr = memkind_data->memkind_realloc (kind, data->ptr, | |
1282 | new_size); | |
1283 | } | |
1284 | else | |
1285 | #endif | |
1286 | { | |
1287 | omp_memspace_handle_t memspace; | |
1288 | memspace = (allocator_data | |
1289 | ? allocator_data->memspace | |
1290 | : predefined_alloc_mapping[allocator]); | |
1291 | new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size, | |
1292 | (free_allocator_data | |
1293 | && free_allocator_data->pinned), | |
1294 | allocator_data && allocator_data->pinned); | |
1295 | } | |
1296 | if (new_ptr == NULL) | |
1297 | goto fail; | |
1298 | ||
1299 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
1300 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
1301 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1302 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1303 | return ret; | |
1304 | } | |
1305 | else | |
1306 | { | |
1307 | #ifdef LIBGOMP_USE_LIBNUMA | |
1308 | if (memkind == GOMP_MEMKIND_LIBNUMA) | |
1309 | new_ptr = libnuma_data->numa_alloc_local (new_size); | |
1310 | # ifdef LIBGOMP_USE_MEMKIND | |
1311 | else | |
1312 | # endif | |
1313 | #endif | |
1314 | #ifdef LIBGOMP_USE_MEMKIND | |
1315 | if (memkind) | |
1316 | { | |
1317 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1318 | void *kind = *memkind_data->kinds[memkind]; | |
1319 | new_ptr = memkind_data->memkind_malloc (kind, new_size); | |
1320 | } | |
1321 | else | |
1322 | #endif | |
1323 | { | |
1324 | omp_memspace_handle_t memspace; | |
1325 | memspace = (allocator_data | |
1326 | ? allocator_data->memspace | |
1327 | : predefined_alloc_mapping[allocator]); | |
1328 | new_ptr = MEMSPACE_ALLOC (memspace, new_size, | |
1329 | allocator_data && allocator_data->pinned); | |
1330 | } | |
1331 | if (new_ptr == NULL) | |
1332 | goto fail; | |
1333 | } | |
1334 | ||
1335 | if (new_alignment > sizeof (void *)) | |
1336 | ret = (void *) (((uintptr_t) new_ptr | |
1337 | + sizeof (struct omp_mem_header) | |
1338 | + new_alignment - sizeof (void *)) | |
1339 | & ~(new_alignment - 1)); | |
1340 | else | |
1341 | ret = (char *) new_ptr + sizeof (struct omp_mem_header); | |
1342 | ((struct omp_mem_header *) ret)[-1].ptr = new_ptr; | |
1343 | ((struct omp_mem_header *) ret)[-1].size = new_size; | |
1344 | ((struct omp_mem_header *) ret)[-1].allocator = allocator; | |
1345 | if (old_size - old_alignment < size) | |
1346 | size = old_size - old_alignment; | |
1347 | memcpy (ret, ptr, size); | |
1348 | if (__builtin_expect (free_allocator_data | |
1349 | && free_allocator_data->pool_size < ~(uintptr_t) 0, 0)) | |
1350 | { | |
1351 | #ifdef HAVE_SYNC_BUILTINS | |
1352 | __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size, | |
1353 | MEMMODEL_RELAXED); | |
1354 | #else | |
1355 | gomp_mutex_lock (&free_allocator_data->lock); | |
1356 | free_allocator_data->used_pool_size -= data->size; | |
1357 | gomp_mutex_unlock (&free_allocator_data->lock); | |
1358 | #endif | |
1359 | } | |
1360 | #ifdef LIBGOMP_USE_LIBNUMA | |
1361 | if (free_memkind == GOMP_MEMKIND_LIBNUMA) | |
1362 | { | |
1363 | libnuma_data->numa_free (data->ptr, data->size); | |
1364 | return ret; | |
1365 | } | |
1366 | # ifdef LIBGOMP_USE_MEMKIND | |
1367 | else | |
1368 | # endif | |
1369 | #endif | |
1370 | #ifdef LIBGOMP_USE_MEMKIND | |
1371 | if (free_memkind) | |
1372 | { | |
1373 | struct gomp_memkind_data *memkind_data = gomp_get_memkind (); | |
1374 | void *kind = *memkind_data->kinds[free_memkind]; | |
1375 | memkind_data->memkind_free (kind, data->ptr); | |
1376 | return ret; | |
1377 | } | |
1378 | #endif | |
1379 | { | |
1380 | omp_memspace_handle_t was_memspace; | |
1381 | was_memspace = (free_allocator_data | |
1382 | ? free_allocator_data->memspace | |
1383 | : predefined_alloc_mapping[free_allocator]); | |
1384 | int was_pinned = (free_allocator_data && free_allocator_data->pinned); | |
1385 | MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned); | |
1386 | } | |
1387 | return ret; | |
1388 | ||
1389 | fail:; | |
1390 | int fallback = (allocator_data | |
1391 | ? allocator_data->fallback | |
1392 | : allocator == omp_default_mem_alloc | |
1393 | ? omp_atv_null_fb | |
1394 | : omp_atv_default_mem_fb); | |
1395 | switch (fallback) | |
1396 | { | |
1397 | case omp_atv_default_mem_fb: | |
1398 | allocator = omp_default_mem_alloc; | |
1399 | goto retry; | |
1400 | case omp_atv_null_fb: | |
1401 | break; | |
1402 | default: | |
1403 | case omp_atv_abort_fb: | |
1404 | gomp_fatal ("Out of memory allocating %lu bytes", | |
1405 | (unsigned long) size); | |
1406 | case omp_atv_allocator_fb: | |
1407 | allocator = allocator_data->fb_data; | |
1408 | goto retry; | |
1409 | } | |
1410 | return NULL; | |
1411 | } |