]>
Commit | Line | Data |
---|---|---|
41dbbb37 TS |
1 | /* OpenACC Runtime initialization routines |
2 | ||
83ffe9cd | 3 | Copyright (C) 2013-2023 Free Software Foundation, Inc. |
41dbbb37 TS |
4 | |
5 | Contributed by Mentor Embedded. | |
6 | ||
7 | This file is part of the GNU Offloading and Multi Processing Library | |
8 | (libgomp). | |
9 | ||
10 | Libgomp is free software; you can redistribute it and/or modify it | |
11 | under the terms of the GNU General Public License as published by | |
12 | the Free Software Foundation; either version 3, or (at your option) | |
13 | any later version. | |
14 | ||
15 | Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY | |
16 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
17 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
18 | more details. | |
19 | ||
20 | Under Section 7 of GPL version 3, you are granted additional | |
21 | permissions described in the GCC Runtime Library Exception, version | |
22 | 3.1, as published by the Free Software Foundation. | |
23 | ||
24 | You should have received a copy of the GNU General Public License and | |
25 | a copy of the GCC Runtime Library Exception along with this program; | |
26 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
27 | <http://www.gnu.org/licenses/>. */ | |
28 | ||
29 | #include "openacc.h" | |
41dbbb37 TS |
30 | #include "libgomp.h" |
31 | #include "gomp-constants.h" | |
32 | #include "oacc-int.h" | |
e46c7770 | 33 | #include <string.h> |
41dbbb37 TS |
34 | #include <assert.h> |
35 | ||
e38fdba4 JB |
36 | /* Return block containing [H->S), or NULL if not contained. The device lock |
37 | for DEV must be locked on entry, and remains locked on exit. */ | |
41dbbb37 TS |
38 | |
39 | static splay_tree_key | |
a51df54e | 40 | lookup_host (struct gomp_device_descr *dev, void *h, size_t s) |
41dbbb37 TS |
41 | { |
42 | struct splay_tree_key_s node; | |
43 | splay_tree_key key; | |
44 | ||
45 | node.host_start = (uintptr_t) h; | |
46 | node.host_end = (uintptr_t) h + s; | |
47 | ||
a51df54e | 48 | key = splay_tree_lookup (&dev->mem_map, &node); |
41dbbb37 TS |
49 | |
50 | return key; | |
51 | } | |
52 | ||
47afc7b4 | 53 | /* Helper for lookup_dev. Iterate over splay tree. */ |
41dbbb37 TS |
54 | |
55 | static splay_tree_key | |
47afc7b4 | 56 | lookup_dev_1 (splay_tree_node node, uintptr_t d, size_t s) |
41dbbb37 | 57 | { |
47afc7b4 TS |
58 | splay_tree_key key = &node->key; |
59 | if (d >= key->tgt->tgt_start && d + s <= key->tgt->tgt_end) | |
60 | return key; | |
41dbbb37 | 61 | |
47afc7b4 TS |
62 | key = NULL; |
63 | if (node->left) | |
64 | key = lookup_dev_1 (node->left, d, s); | |
65 | if (!key && node->right) | |
66 | key = lookup_dev_1 (node->right, d, s); | |
41dbbb37 | 67 | |
47afc7b4 TS |
68 | return key; |
69 | } | |
41dbbb37 | 70 | |
47afc7b4 | 71 | /* Return block containing [D->S), or NULL if not contained. |
41dbbb37 | 72 | |
47afc7b4 TS |
73 | This iterates over the splay tree. This is not expected to be a common |
74 | operation. | |
41dbbb37 | 75 | |
47afc7b4 TS |
76 | The device lock associated with MEM_MAP must be locked on entry, and remains |
77 | locked on exit. */ | |
41dbbb37 | 78 | |
47afc7b4 TS |
79 | static splay_tree_key |
80 | lookup_dev (splay_tree mem_map, void *d, size_t s) | |
81 | { | |
82 | if (!mem_map || !mem_map->root) | |
83 | return NULL; | |
41dbbb37 | 84 | |
47afc7b4 | 85 | return lookup_dev_1 (mem_map->root, (uintptr_t) d, s); |
41dbbb37 TS |
86 | } |
87 | ||
47afc7b4 | 88 | |
41dbbb37 TS |
89 | /* OpenACC is silent on how memory exhaustion is indicated. We return |
90 | NULL. */ | |
91 | ||
92 | void * | |
93 | acc_malloc (size_t s) | |
94 | { | |
95 | if (!s) | |
96 | return NULL; | |
97 | ||
98 | goacc_lazy_initialize (); | |
99 | ||
100 | struct goacc_thread *thr = goacc_thread (); | |
101 | ||
d93bdab5 JB |
102 | assert (thr->dev); |
103 | ||
e46c7770 CP |
104 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
105 | return malloc (s); | |
106 | ||
5fae049d TS |
107 | acc_prof_info prof_info; |
108 | acc_api_info api_info; | |
109 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
110 | ||
111 | void *res = thr->dev->alloc_func (thr->dev->target_id, s); | |
112 | ||
113 | if (profiling_p) | |
114 | { | |
115 | thr->prof_info = NULL; | |
116 | thr->api_info = NULL; | |
117 | } | |
118 | ||
119 | return res; | |
41dbbb37 TS |
120 | } |
121 | ||
41dbbb37 TS |
122 | void |
123 | acc_free (void *d) | |
124 | { | |
125 | splay_tree_key k; | |
41dbbb37 TS |
126 | |
127 | if (!d) | |
128 | return; | |
129 | ||
e38fdba4 JB |
130 | struct goacc_thread *thr = goacc_thread (); |
131 | ||
d93bdab5 JB |
132 | assert (thr && thr->dev); |
133 | ||
e38fdba4 JB |
134 | struct gomp_device_descr *acc_dev = thr->dev; |
135 | ||
e46c7770 CP |
136 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
137 | return free (d); | |
138 | ||
5fae049d TS |
139 | acc_prof_info prof_info; |
140 | acc_api_info api_info; | |
141 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
142 | ||
e38fdba4 JB |
143 | gomp_mutex_lock (&acc_dev->lock); |
144 | ||
41dbbb37 TS |
145 | /* We don't have to call lazy open here, as the ptr value must have |
146 | been returned by acc_malloc. It's not permitted to pass NULL in | |
147 | (unless you got that null from acc_malloc). */ | |
47afc7b4 | 148 | if ((k = lookup_dev (&acc_dev->mem_map, d, 1))) |
e38fdba4 | 149 | { |
cec41816 TS |
150 | void *offset = d - k->tgt->tgt_start + k->tgt_offset; |
151 | void *h = k->host_start + offset; | |
152 | size_t h_size = k->host_end - k->host_start; | |
e38fdba4 | 153 | gomp_mutex_unlock (&acc_dev->lock); |
cec41816 TS |
154 | /* PR92503 "[OpenACC] Behavior of 'acc_free' if the memory space is still |
155 | used in a mapping". */ | |
156 | gomp_fatal ("refusing to free device memory space at %p that is still" | |
157 | " mapped at [%p,+%d]", | |
158 | d, h, (int) h_size); | |
e38fdba4 JB |
159 | } |
160 | else | |
161 | gomp_mutex_unlock (&acc_dev->lock); | |
162 | ||
6ce13072 CLT |
163 | if (!acc_dev->free_func (acc_dev->target_id, d)) |
164 | gomp_fatal ("error in freeing device memory in %s", __FUNCTION__); | |
5fae049d TS |
165 | |
166 | if (profiling_p) | |
167 | { | |
168 | thr->prof_info = NULL; | |
169 | thr->api_info = NULL; | |
170 | } | |
41dbbb37 TS |
171 | } |
172 | ||
58168bbf CLT |
173 | static void |
174 | memcpy_tofrom_device (bool from, void *d, void *h, size_t s, int async, | |
175 | const char *libfnname) | |
41dbbb37 TS |
176 | { |
177 | /* No need to call lazy open here, as the device pointer must have | |
178 | been obtained from a routine that did that. */ | |
179 | struct goacc_thread *thr = goacc_thread (); | |
180 | ||
d93bdab5 JB |
181 | assert (thr && thr->dev); |
182 | ||
e46c7770 CP |
183 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
184 | { | |
58168bbf CLT |
185 | if (from) |
186 | memmove (h, d, s); | |
187 | else | |
188 | memmove (d, h, s); | |
e46c7770 CP |
189 | return; |
190 | } | |
191 | ||
5fae049d TS |
192 | acc_prof_info prof_info; |
193 | acc_api_info api_info; | |
194 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
195 | if (profiling_p) | |
196 | { | |
197 | prof_info.async = async; | |
198 | prof_info.async_queue = prof_info.async; | |
199 | } | |
200 | ||
1f4c5b9b CLT |
201 | goacc_aq aq = get_goacc_asyncqueue (async); |
202 | if (from) | |
203 | gomp_copy_dev2host (thr->dev, aq, h, d, s); | |
204 | else | |
9c41f5b9 | 205 | gomp_copy_host2dev (thr->dev, aq, d, h, s, false, /* TODO: cbuf? */ NULL); |
5fae049d TS |
206 | |
207 | if (profiling_p) | |
208 | { | |
209 | thr->prof_info = NULL; | |
210 | thr->api_info = NULL; | |
211 | } | |
41dbbb37 TS |
212 | } |
213 | ||
214 | void | |
58168bbf | 215 | acc_memcpy_to_device (void *d, void *h, size_t s) |
41dbbb37 | 216 | { |
58168bbf CLT |
217 | memcpy_tofrom_device (false, d, h, s, acc_async_sync, __FUNCTION__); |
218 | } | |
41dbbb37 | 219 | |
58168bbf CLT |
220 | void |
221 | acc_memcpy_to_device_async (void *d, void *h, size_t s, int async) | |
222 | { | |
223 | memcpy_tofrom_device (false, d, h, s, async, __FUNCTION__); | |
224 | } | |
d93bdab5 | 225 | |
58168bbf CLT |
226 | void |
227 | acc_memcpy_from_device (void *h, void *d, size_t s) | |
228 | { | |
229 | memcpy_tofrom_device (true, d, h, s, acc_async_sync, __FUNCTION__); | |
230 | } | |
e46c7770 | 231 | |
58168bbf CLT |
232 | void |
233 | acc_memcpy_from_device_async (void *h, void *d, size_t s, int async) | |
234 | { | |
235 | memcpy_tofrom_device (true, d, h, s, async, __FUNCTION__); | |
41dbbb37 TS |
236 | } |
237 | ||
238 | /* Return the device pointer that corresponds to host data H. Or NULL | |
239 | if no mapping. */ | |
240 | ||
241 | void * | |
242 | acc_deviceptr (void *h) | |
243 | { | |
244 | splay_tree_key n; | |
245 | void *d; | |
246 | void *offset; | |
247 | ||
248 | goacc_lazy_initialize (); | |
249 | ||
250 | struct goacc_thread *thr = goacc_thread (); | |
e38fdba4 JB |
251 | struct gomp_device_descr *dev = thr->dev; |
252 | ||
e46c7770 CP |
253 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
254 | return h; | |
255 | ||
5fae049d TS |
256 | /* In the following, no OpenACC Profiling Interface events can possibly be |
257 | generated. */ | |
258 | ||
e38fdba4 | 259 | gomp_mutex_lock (&dev->lock); |
41dbbb37 | 260 | |
e38fdba4 | 261 | n = lookup_host (dev, h, 1); |
41dbbb37 TS |
262 | |
263 | if (!n) | |
e38fdba4 JB |
264 | { |
265 | gomp_mutex_unlock (&dev->lock); | |
266 | return NULL; | |
267 | } | |
41dbbb37 TS |
268 | |
269 | offset = h - n->host_start; | |
270 | ||
271 | d = n->tgt->tgt_start + n->tgt_offset + offset; | |
272 | ||
e38fdba4 JB |
273 | gomp_mutex_unlock (&dev->lock); |
274 | ||
41dbbb37 TS |
275 | return d; |
276 | } | |
277 | ||
278 | /* Return the host pointer that corresponds to device data D. Or NULL | |
279 | if no mapping. */ | |
280 | ||
281 | void * | |
282 | acc_hostptr (void *d) | |
283 | { | |
284 | splay_tree_key n; | |
285 | void *h; | |
286 | void *offset; | |
287 | ||
288 | goacc_lazy_initialize (); | |
289 | ||
290 | struct goacc_thread *thr = goacc_thread (); | |
e38fdba4 | 291 | struct gomp_device_descr *acc_dev = thr->dev; |
41dbbb37 | 292 | |
e46c7770 CP |
293 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
294 | return d; | |
295 | ||
5fae049d TS |
296 | /* In the following, no OpenACC Profiling Interface events can possibly be |
297 | generated. */ | |
298 | ||
e38fdba4 JB |
299 | gomp_mutex_lock (&acc_dev->lock); |
300 | ||
47afc7b4 | 301 | n = lookup_dev (&acc_dev->mem_map, d, 1); |
41dbbb37 TS |
302 | |
303 | if (!n) | |
e38fdba4 JB |
304 | { |
305 | gomp_mutex_unlock (&acc_dev->lock); | |
306 | return NULL; | |
307 | } | |
41dbbb37 TS |
308 | |
309 | offset = d - n->tgt->tgt_start + n->tgt_offset; | |
310 | ||
311 | h = n->host_start + offset; | |
312 | ||
e38fdba4 JB |
313 | gomp_mutex_unlock (&acc_dev->lock); |
314 | ||
41dbbb37 TS |
315 | return h; |
316 | } | |
317 | ||
318 | /* Return 1 if host data [H,+S] is present on the device. */ | |
319 | ||
320 | int | |
321 | acc_is_present (void *h, size_t s) | |
322 | { | |
323 | splay_tree_key n; | |
324 | ||
325 | if (!s || !h) | |
326 | return 0; | |
327 | ||
328 | goacc_lazy_initialize (); | |
329 | ||
330 | struct goacc_thread *thr = goacc_thread (); | |
331 | struct gomp_device_descr *acc_dev = thr->dev; | |
332 | ||
e46c7770 CP |
333 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
334 | return h != NULL; | |
335 | ||
5fae049d TS |
336 | /* In the following, no OpenACC Profiling Interface events can possibly be |
337 | generated. */ | |
338 | ||
e38fdba4 JB |
339 | gomp_mutex_lock (&acc_dev->lock); |
340 | ||
a51df54e | 341 | n = lookup_host (acc_dev, h, s); |
41dbbb37 TS |
342 | |
343 | if (n && ((uintptr_t)h < n->host_start | |
344 | || (uintptr_t)h + s > n->host_end | |
345 | || s > n->host_end - n->host_start)) | |
346 | n = NULL; | |
347 | ||
e38fdba4 JB |
348 | gomp_mutex_unlock (&acc_dev->lock); |
349 | ||
41dbbb37 TS |
350 | return n != NULL; |
351 | } | |
352 | ||
353 | /* Create a mapping for host [H,+S] -> device [D,+S] */ | |
354 | ||
355 | void | |
356 | acc_map_data (void *h, void *d, size_t s) | |
357 | { | |
41dbbb37 TS |
358 | size_t mapnum = 1; |
359 | void *hostaddrs = h; | |
360 | void *devaddrs = d; | |
361 | size_t sizes = s; | |
362 | unsigned short kinds = GOMP_MAP_ALLOC; | |
363 | ||
364 | goacc_lazy_initialize (); | |
365 | ||
366 | struct goacc_thread *thr = goacc_thread (); | |
367 | struct gomp_device_descr *acc_dev = thr->dev; | |
368 | ||
369 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
370 | { | |
371 | if (d != h) | |
372 | gomp_fatal ("cannot map data on shared-memory system"); | |
41dbbb37 TS |
373 | } |
374 | else | |
375 | { | |
376 | struct goacc_thread *thr = goacc_thread (); | |
377 | ||
378 | if (!d || !h || !s) | |
379 | gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map", | |
380 | (void *)h, (int)s, (void *)d, (int)s); | |
381 | ||
5fae049d TS |
382 | acc_prof_info prof_info; |
383 | acc_api_info api_info; | |
384 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
385 | ||
e38fdba4 JB |
386 | gomp_mutex_lock (&acc_dev->lock); |
387 | ||
a51df54e | 388 | if (lookup_host (acc_dev, h, s)) |
e38fdba4 JB |
389 | { |
390 | gomp_mutex_unlock (&acc_dev->lock); | |
391 | gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h, | |
392 | (int)s); | |
393 | } | |
41dbbb37 | 394 | |
47afc7b4 | 395 | if (lookup_dev (&thr->dev->mem_map, d, s)) |
e38fdba4 JB |
396 | { |
397 | gomp_mutex_unlock (&acc_dev->lock); | |
398 | gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d, | |
399 | (int)s); | |
400 | } | |
401 | ||
402 | gomp_mutex_unlock (&acc_dev->lock); | |
41dbbb37 | 403 | |
f233418c | 404 | struct target_mem_desc *tgt |
275c736e CLT |
405 | = goacc_map_vars (acc_dev, NULL, mapnum, &hostaddrs, &devaddrs, &sizes, |
406 | &kinds, true, GOMP_MAP_VARS_ENTER_DATA); | |
ba40277f | 407 | assert (tgt); |
f233418c | 408 | assert (tgt->list_count == 1); |
e103542b | 409 | splay_tree_key n = tgt->list[0].key; |
f233418c | 410 | assert (n); |
e103542b | 411 | assert (n->refcount == 1); |
6f5b4b64 | 412 | assert (n->dynamic_refcount == 0); |
e103542b TS |
413 | /* Special reference counting behavior. */ |
414 | n->refcount = REFCOUNT_INFINITY; | |
5fae049d TS |
415 | |
416 | if (profiling_p) | |
417 | { | |
418 | thr->prof_info = NULL; | |
419 | thr->api_info = NULL; | |
420 | } | |
41dbbb37 | 421 | } |
41dbbb37 TS |
422 | } |
423 | ||
424 | void | |
425 | acc_unmap_data (void *h) | |
426 | { | |
427 | struct goacc_thread *thr = goacc_thread (); | |
428 | struct gomp_device_descr *acc_dev = thr->dev; | |
429 | ||
430 | /* No need to call lazy open, as the address must have been mapped. */ | |
431 | ||
e46c7770 CP |
432 | /* This is a no-op on shared-memory targets. */ |
433 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
434 | return; | |
435 | ||
5fae049d TS |
436 | acc_prof_info prof_info; |
437 | acc_api_info api_info; | |
438 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
439 | ||
e38fdba4 JB |
440 | gomp_mutex_lock (&acc_dev->lock); |
441 | ||
a51df54e | 442 | splay_tree_key n = lookup_host (acc_dev, h, 1); |
41dbbb37 TS |
443 | |
444 | if (!n) | |
e38fdba4 JB |
445 | { |
446 | gomp_mutex_unlock (&acc_dev->lock); | |
447 | gomp_fatal ("%p is not a mapped block", (void *)h); | |
448 | } | |
41dbbb37 | 449 | |
378da98f | 450 | size_t host_size = n->host_end - n->host_start; |
41dbbb37 TS |
451 | |
452 | if (n->host_start != (uintptr_t) h) | |
e38fdba4 JB |
453 | { |
454 | gomp_mutex_unlock (&acc_dev->lock); | |
455 | gomp_fatal ("[%p,%d] surrounds %p", | |
456 | (void *) n->host_start, (int) host_size, (void *) h); | |
457 | } | |
e103542b | 458 | /* TODO This currently doesn't catch 'REFCOUNT_INFINITY' usage different from |
6f5b4b64 | 459 | 'acc_map_data'. Maybe 'dynamic_refcount' can be used for disambiguating |
e103542b TS |
460 | the different 'REFCOUNT_INFINITY' cases, or simply separate |
461 | 'REFCOUNT_INFINITY' values per different usage ('REFCOUNT_ACC_MAP_DATA' | |
462 | etc.)? */ | |
463 | else if (n->refcount != REFCOUNT_INFINITY) | |
464 | { | |
465 | gomp_mutex_unlock (&acc_dev->lock); | |
466 | gomp_fatal ("refusing to unmap block [%p,+%d] that has not been mapped" | |
467 | " by 'acc_map_data'", | |
468 | (void *) h, (int) host_size); | |
469 | } | |
41dbbb37 | 470 | |
378da98f | 471 | struct target_mem_desc *tgt = n->tgt; |
41dbbb37 | 472 | |
378da98f | 473 | if (tgt->refcount == REFCOUNT_INFINITY) |
41dbbb37 | 474 | { |
378da98f JB |
475 | gomp_mutex_unlock (&acc_dev->lock); |
476 | gomp_fatal ("cannot unmap target block"); | |
477 | } | |
4662f7fe TS |
478 | |
479 | /* Above, we've verified that the mapping must have been set up by | |
480 | 'acc_map_data'. */ | |
481 | assert (tgt->refcount == 1); | |
482 | ||
2112d324 TS |
483 | /* Nullifying these fields prevents 'gomp_unmap_tgt' via 'gomp_remove_var' |
484 | from freeing the target memory. */ | |
485 | tgt->tgt_end = 0; | |
486 | tgt->to_free = NULL; | |
487 | ||
06ec6172 TS |
488 | bool is_tgt_unmapped = gomp_remove_var (acc_dev, n); |
489 | assert (is_tgt_unmapped); | |
e38fdba4 | 490 | |
ba40277f | 491 | gomp_mutex_unlock (&acc_dev->lock); |
5fae049d TS |
492 | |
493 | if (profiling_p) | |
494 | { | |
495 | thr->prof_info = NULL; | |
496 | thr->api_info = NULL; | |
497 | } | |
41dbbb37 TS |
498 | } |
499 | ||
aaf0e9d7 | 500 | |
cb7effde JB |
501 | /* Helper function to map a single dynamic data item, represented by a single |
502 | mapping. The acc_dev->lock should be held on entry, and remains locked on | |
503 | exit. */ | |
504 | ||
505 | static void * | |
506 | goacc_map_var_existing (struct gomp_device_descr *acc_dev, void *hostaddr, | |
507 | size_t size, splay_tree_key n) | |
508 | { | |
509 | assert (n); | |
510 | ||
511 | /* Present. */ | |
512 | void *d = (void *) (n->tgt->tgt_start + n->tgt_offset + hostaddr | |
513 | - n->host_start); | |
514 | ||
515 | if (hostaddr + size > (void *) n->host_end) | |
516 | { | |
517 | gomp_mutex_unlock (&acc_dev->lock); | |
518 | gomp_fatal ("[%p,+%d] not mapped", hostaddr, (int) size); | |
519 | } | |
520 | ||
521 | assert (n->refcount != REFCOUNT_LINK); | |
522 | if (n->refcount != REFCOUNT_INFINITY) | |
6f5b4b64 JB |
523 | n->refcount++; |
524 | n->dynamic_refcount++; | |
cb7effde JB |
525 | |
526 | return d; | |
527 | } | |
528 | ||
378da98f | 529 | /* Enter dynamic mapping for a single datum. Return the device pointer. */ |
41dbbb37 TS |
530 | |
531 | static void * | |
378da98f | 532 | goacc_enter_datum (void **hostaddrs, size_t *sizes, void *kinds, int async) |
41dbbb37 TS |
533 | { |
534 | void *d; | |
535 | splay_tree_key n; | |
536 | ||
378da98f | 537 | if (!hostaddrs[0] || !sizes[0]) |
9444a299 | 538 | gomp_fatal ("[%p,+%d] is a bad range", hostaddrs[0], (int) sizes[0]); |
41dbbb37 TS |
539 | |
540 | goacc_lazy_initialize (); | |
541 | ||
542 | struct goacc_thread *thr = goacc_thread (); | |
543 | struct gomp_device_descr *acc_dev = thr->dev; | |
544 | ||
e46c7770 | 545 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
378da98f | 546 | return hostaddrs[0]; |
e46c7770 | 547 | |
5fae049d TS |
548 | acc_prof_info prof_info; |
549 | acc_api_info api_info; | |
550 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
551 | if (profiling_p) | |
552 | { | |
553 | prof_info.async = async; | |
554 | prof_info.async_queue = prof_info.async; | |
555 | } | |
556 | ||
e38fdba4 JB |
557 | gomp_mutex_lock (&acc_dev->lock); |
558 | ||
9444a299 | 559 | n = lookup_host (acc_dev, hostaddrs[0], sizes[0]); |
378da98f | 560 | if (n) |
41dbbb37 | 561 | { |
cb7effde | 562 | d = goacc_map_var_existing (acc_dev, hostaddrs[0], sizes[0], n); |
77ce5555 | 563 | gomp_mutex_unlock (&acc_dev->lock); |
77ce5555 | 564 | } |
41dbbb37 TS |
565 | else |
566 | { | |
378da98f JB |
567 | const size_t mapnum = 1; |
568 | ||
e38fdba4 JB |
569 | gomp_mutex_unlock (&acc_dev->lock); |
570 | ||
1f4c5b9b | 571 | goacc_aq aq = get_goacc_asyncqueue (async); |
58168bbf | 572 | |
f233418c | 573 | struct target_mem_desc *tgt |
275c736e CLT |
574 | = goacc_map_vars (acc_dev, aq, mapnum, hostaddrs, NULL, sizes, |
575 | kinds, true, GOMP_MAP_VARS_ENTER_DATA); | |
f233418c TS |
576 | assert (tgt); |
577 | assert (tgt->list_count == 1); | |
578 | n = tgt->list[0].key; | |
579 | assert (n); | |
580 | assert (n->refcount == 1); | |
6f5b4b64 JB |
581 | assert (n->dynamic_refcount == 0); |
582 | n->dynamic_refcount++; | |
41dbbb37 | 583 | |
f233418c | 584 | d = (void *) tgt->tgt_start; |
41dbbb37 TS |
585 | } |
586 | ||
5fae049d TS |
587 | if (profiling_p) |
588 | { | |
589 | thr->prof_info = NULL; | |
590 | thr->api_info = NULL; | |
591 | } | |
592 | ||
41dbbb37 TS |
593 | return d; |
594 | } | |
595 | ||
596 | void * | |
597 | acc_create (void *h, size_t s) | |
598 | { | |
9444a299 | 599 | unsigned short kinds[1] = { GOMP_MAP_ALLOC }; |
378da98f | 600 | return goacc_enter_datum (&h, &s, &kinds, acc_async_sync); |
58168bbf CLT |
601 | } |
602 | ||
603 | void | |
604 | acc_create_async (void *h, size_t s, int async) | |
605 | { | |
9444a299 | 606 | unsigned short kinds[1] = { GOMP_MAP_ALLOC }; |
378da98f | 607 | goacc_enter_datum (&h, &s, &kinds, async); |
41dbbb37 TS |
608 | } |
609 | ||
c759830b TS |
610 | /* acc_present_or_create used to be what acc_create is now. */ |
611 | /* acc_pcreate is acc_present_or_create by a different name. */ | |
612 | #ifdef HAVE_ATTRIBUTE_ALIAS | |
613 | strong_alias (acc_create, acc_present_or_create) | |
614 | strong_alias (acc_create, acc_pcreate) | |
615 | #else | |
41dbbb37 TS |
616 | void * |
617 | acc_present_or_create (void *h, size_t s) | |
618 | { | |
c759830b | 619 | return acc_create (h, s); |
41dbbb37 TS |
620 | } |
621 | ||
9b94fbc7 TS |
622 | void * |
623 | acc_pcreate (void *h, size_t s) | |
624 | { | |
c759830b | 625 | return acc_create (h, s); |
9b94fbc7 TS |
626 | } |
627 | #endif | |
628 | ||
41dbbb37 | 629 | void * |
c759830b | 630 | acc_copyin (void *h, size_t s) |
41dbbb37 | 631 | { |
9444a299 | 632 | unsigned short kinds[1] = { GOMP_MAP_TO }; |
378da98f | 633 | return goacc_enter_datum (&h, &s, &kinds, acc_async_sync); |
41dbbb37 TS |
634 | } |
635 | ||
c759830b TS |
636 | void |
637 | acc_copyin_async (void *h, size_t s, int async) | |
638 | { | |
9444a299 | 639 | unsigned short kinds[1] = { GOMP_MAP_TO }; |
378da98f | 640 | goacc_enter_datum (&h, &s, &kinds, async); |
c759830b TS |
641 | } |
642 | ||
643 | /* acc_present_or_copyin used to be what acc_copyin is now. */ | |
9b94fbc7 TS |
644 | /* acc_pcopyin is acc_present_or_copyin by a different name. */ |
645 | #ifdef HAVE_ATTRIBUTE_ALIAS | |
c759830b TS |
646 | strong_alias (acc_copyin, acc_present_or_copyin) |
647 | strong_alias (acc_copyin, acc_pcopyin) | |
9b94fbc7 | 648 | #else |
c759830b TS |
649 | void * |
650 | acc_present_or_copyin (void *h, size_t s) | |
651 | { | |
652 | return acc_copyin (h, s); | |
653 | } | |
654 | ||
9b94fbc7 TS |
655 | void * |
656 | acc_pcopyin (void *h, size_t s) | |
657 | { | |
c759830b | 658 | return acc_copyin (h, s); |
9b94fbc7 TS |
659 | } |
660 | #endif | |
661 | ||
34cfe31e | 662 | |
cb7effde JB |
663 | /* Helper function to unmap a single data item. Device lock should be held on |
664 | entry, and remains locked on exit. */ | |
41dbbb37 TS |
665 | |
666 | static void | |
cb7effde JB |
667 | goacc_exit_datum_1 (struct gomp_device_descr *acc_dev, void *h, size_t s, |
668 | unsigned short kind, splay_tree_key n, goacc_aq aq) | |
41dbbb37 | 669 | { |
bc4ed079 JB |
670 | assert (kind != GOMP_MAP_DETACH |
671 | && kind != GOMP_MAP_FORCE_DETACH); | |
672 | ||
e307b05f | 673 | if ((uintptr_t) h < n->host_start || (uintptr_t) h + s > n->host_end) |
e38fdba4 | 674 | { |
e307b05f | 675 | size_t host_size = n->host_end - n->host_start; |
e38fdba4 | 676 | gomp_mutex_unlock (&acc_dev->lock); |
e307b05f JB |
677 | gomp_fatal ("[%p,+%d] outside mapped block [%p,+%d]", |
678 | (void *) h, (int) s, (void *) n->host_start, (int) host_size); | |
e38fdba4 JB |
679 | } |
680 | ||
6f5b4b64 | 681 | bool finalize = (kind == GOMP_MAP_FORCE_FROM |
bc4ed079 | 682 | || kind == GOMP_MAP_DELETE); |
6f5b4b64 JB |
683 | |
684 | assert (n->refcount != REFCOUNT_LINK); | |
685 | if (n->refcount != REFCOUNT_INFINITY | |
686 | && n->refcount < n->dynamic_refcount) | |
687 | { | |
688 | gomp_mutex_unlock (&acc_dev->lock); | |
689 | gomp_fatal ("Dynamic reference counting assert fail\n"); | |
690 | } | |
cb7effde | 691 | |
34cfe31e | 692 | if (finalize) |
829c6349 | 693 | { |
d6e8c01c | 694 | if (n->refcount != REFCOUNT_INFINITY) |
6f5b4b64 JB |
695 | n->refcount -= n->dynamic_refcount; |
696 | n->dynamic_refcount = 0; | |
829c6349 | 697 | } |
6f5b4b64 | 698 | else if (n->dynamic_refcount) |
829c6349 | 699 | { |
d6e8c01c TS |
700 | if (n->refcount != REFCOUNT_INFINITY) |
701 | n->refcount--; | |
6f5b4b64 | 702 | n->dynamic_refcount--; |
829c6349 | 703 | } |
41dbbb37 | 704 | |
829c6349 CLT |
705 | if (n->refcount == 0) |
706 | { | |
34cfe31e TS |
707 | bool copyout = (kind == GOMP_MAP_FROM |
708 | || kind == GOMP_MAP_FORCE_FROM); | |
709 | if (copyout) | |
58168bbf | 710 | { |
1cbd94e8 JB |
711 | void *d = (void *) (n->tgt->tgt_start + n->tgt_offset |
712 | + (uintptr_t) h - n->host_start); | |
1f4c5b9b | 713 | gomp_copy_dev2host (acc_dev, aq, h, d, s); |
58168bbf | 714 | } |
ba40277f TS |
715 | |
716 | if (aq) | |
717 | /* TODO We can't do the 'is_tgt_unmapped' checking -- see the | |
718 | 'gomp_unref_tgt' comment in | |
719 | <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>; | |
720 | PR92881. */ | |
721 | gomp_remove_var_async (acc_dev, n, aq); | |
722 | else | |
723 | { | |
06ec6172 TS |
724 | size_t num_mappings = 0; |
725 | /* If the target_mem_desc represents a single data mapping, we can | |
726 | check that it is freed when this splay tree key's refcount reaches | |
727 | zero. Otherwise (e.g. for a 'GOMP_MAP_STRUCT' mapping with | |
728 | multiple members), fall back to skipping the test. */ | |
729 | for (size_t l_i = 0; l_i < n->tgt->list_count; ++l_i) | |
bc4ed079 JB |
730 | if (n->tgt->list[l_i].key |
731 | && !n->tgt->list[l_i].is_attach) | |
06ec6172 | 732 | ++num_mappings; |
ba40277f | 733 | bool is_tgt_unmapped = gomp_remove_var (acc_dev, n); |
06ec6172 | 734 | assert (is_tgt_unmapped || num_mappings > 1); |
ba40277f | 735 | } |
829c6349 | 736 | } |
cb7effde JB |
737 | } |
738 | ||
739 | ||
740 | /* Exit a dynamic mapping for a single variable. */ | |
741 | ||
742 | static void | |
743 | goacc_exit_datum (void *h, size_t s, unsigned short kind, int async) | |
744 | { | |
745 | /* No need to call lazy open, as the data must already have been | |
746 | mapped. */ | |
747 | ||
748 | kind &= 0xff; | |
749 | ||
750 | struct goacc_thread *thr = goacc_thread (); | |
751 | struct gomp_device_descr *acc_dev = thr->dev; | |
752 | ||
753 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
754 | return; | |
755 | ||
756 | acc_prof_info prof_info; | |
757 | acc_api_info api_info; | |
758 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
759 | if (profiling_p) | |
760 | { | |
761 | prof_info.async = async; | |
762 | prof_info.async_queue = prof_info.async; | |
763 | } | |
764 | ||
765 | gomp_mutex_lock (&acc_dev->lock); | |
766 | ||
767 | splay_tree_key n = lookup_host (acc_dev, h, s); | |
768 | /* Non-present data is a no-op: PR92726, RP92970, PR92984. */ | |
769 | if (n) | |
770 | { | |
771 | goacc_aq aq = get_goacc_asyncqueue (async); | |
772 | goacc_exit_datum_1 (acc_dev, h, s, kind, n, aq); | |
773 | } | |
829c6349 CLT |
774 | |
775 | gomp_mutex_unlock (&acc_dev->lock); | |
5fae049d TS |
776 | |
777 | if (profiling_p) | |
778 | { | |
779 | thr->prof_info = NULL; | |
780 | thr->api_info = NULL; | |
781 | } | |
41dbbb37 TS |
782 | } |
783 | ||
784 | void | |
785 | acc_delete (void *h , size_t s) | |
786 | { | |
378da98f | 787 | goacc_exit_datum (h, s, GOMP_MAP_RELEASE, acc_async_sync); |
58168bbf CLT |
788 | } |
789 | ||
790 | void | |
791 | acc_delete_async (void *h , size_t s, int async) | |
792 | { | |
378da98f | 793 | goacc_exit_datum (h, s, GOMP_MAP_RELEASE, async); |
41dbbb37 TS |
794 | } |
795 | ||
829c6349 CLT |
796 | void |
797 | acc_delete_finalize (void *h , size_t s) | |
798 | { | |
378da98f | 799 | goacc_exit_datum (h, s, GOMP_MAP_DELETE, acc_async_sync); |
829c6349 CLT |
800 | } |
801 | ||
802 | void | |
803 | acc_delete_finalize_async (void *h , size_t s, int async) | |
804 | { | |
378da98f | 805 | goacc_exit_datum (h, s, GOMP_MAP_DELETE, async); |
829c6349 CLT |
806 | } |
807 | ||
6ce13072 CLT |
808 | void |
809 | acc_copyout (void *h, size_t s) | |
41dbbb37 | 810 | { |
378da98f | 811 | goacc_exit_datum (h, s, GOMP_MAP_FROM, acc_async_sync); |
58168bbf CLT |
812 | } |
813 | ||
814 | void | |
815 | acc_copyout_async (void *h, size_t s, int async) | |
816 | { | |
378da98f | 817 | goacc_exit_datum (h, s, GOMP_MAP_FROM, async); |
41dbbb37 TS |
818 | } |
819 | ||
829c6349 CLT |
820 | void |
821 | acc_copyout_finalize (void *h, size_t s) | |
822 | { | |
378da98f | 823 | goacc_exit_datum (h, s, GOMP_MAP_FORCE_FROM, acc_async_sync); |
829c6349 CLT |
824 | } |
825 | ||
826 | void | |
827 | acc_copyout_finalize_async (void *h, size_t s, int async) | |
828 | { | |
378da98f | 829 | goacc_exit_datum (h, s, GOMP_MAP_FORCE_FROM, async); |
829c6349 CLT |
830 | } |
831 | ||
41dbbb37 | 832 | static void |
58168bbf | 833 | update_dev_host (int is_dev, void *h, size_t s, int async) |
41dbbb37 TS |
834 | { |
835 | splay_tree_key n; | |
836 | void *d; | |
8baa7864 TS |
837 | |
838 | goacc_lazy_initialize (); | |
839 | ||
41dbbb37 TS |
840 | struct goacc_thread *thr = goacc_thread (); |
841 | struct gomp_device_descr *acc_dev = thr->dev; | |
842 | ||
e46c7770 CP |
843 | if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) |
844 | return; | |
845 | ||
6e4d01d6 TB |
846 | /* Fortran optional arguments that are non-present result in a |
847 | NULL host address here. This can safely be ignored as it is | |
848 | not possible to 'update' a non-present optional argument. */ | |
849 | if (h == NULL) | |
850 | return; | |
851 | ||
5fae049d TS |
852 | acc_prof_info prof_info; |
853 | acc_api_info api_info; | |
854 | bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info); | |
855 | if (profiling_p) | |
856 | { | |
857 | prof_info.async = async; | |
858 | prof_info.async_queue = prof_info.async; | |
859 | } | |
860 | ||
e38fdba4 JB |
861 | gomp_mutex_lock (&acc_dev->lock); |
862 | ||
a51df54e | 863 | n = lookup_host (acc_dev, h, s); |
41dbbb37 | 864 | |
41dbbb37 | 865 | if (!n) |
e38fdba4 JB |
866 | { |
867 | gomp_mutex_unlock (&acc_dev->lock); | |
868 | gomp_fatal ("[%p,%d] is not mapped", h, (int)s); | |
869 | } | |
41dbbb37 | 870 | |
b6d1f2b5 JN |
871 | d = (void *) (n->tgt->tgt_start + n->tgt_offset |
872 | + (uintptr_t) h - n->host_start); | |
41dbbb37 | 873 | |
1f4c5b9b | 874 | goacc_aq aq = get_goacc_asyncqueue (async); |
58168bbf | 875 | |
41dbbb37 | 876 | if (is_dev) |
9c41f5b9 | 877 | gomp_copy_host2dev (acc_dev, aq, d, h, s, false, /* TODO: cbuf? */ NULL); |
41dbbb37 | 878 | else |
1f4c5b9b | 879 | gomp_copy_dev2host (acc_dev, aq, h, d, s); |
58168bbf | 880 | |
6ce13072 | 881 | gomp_mutex_unlock (&acc_dev->lock); |
5fae049d TS |
882 | |
883 | if (profiling_p) | |
884 | { | |
885 | thr->prof_info = NULL; | |
886 | thr->api_info = NULL; | |
887 | } | |
41dbbb37 TS |
888 | } |
889 | ||
890 | void | |
891 | acc_update_device (void *h, size_t s) | |
892 | { | |
58168bbf CLT |
893 | update_dev_host (1, h, s, acc_async_sync); |
894 | } | |
895 | ||
896 | void | |
897 | acc_update_device_async (void *h, size_t s, int async) | |
898 | { | |
899 | update_dev_host (1, h, s, async); | |
41dbbb37 TS |
900 | } |
901 | ||
902 | void | |
903 | acc_update_self (void *h, size_t s) | |
904 | { | |
58168bbf CLT |
905 | update_dev_host (0, h, s, acc_async_sync); |
906 | } | |
907 | ||
908 | void | |
909 | acc_update_self_async (void *h, size_t s, int async) | |
910 | { | |
911 | update_dev_host (0, h, s, async); | |
41dbbb37 TS |
912 | } |
913 | ||
5d5be7bf JB |
914 | void |
915 | acc_attach_async (void **hostaddr, int async) | |
916 | { | |
917 | struct goacc_thread *thr = goacc_thread (); | |
918 | struct gomp_device_descr *acc_dev = thr->dev; | |
919 | goacc_aq aq = get_goacc_asyncqueue (async); | |
920 | ||
921 | struct splay_tree_key_s cur_node; | |
922 | splay_tree_key n; | |
923 | ||
924 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
925 | return; | |
926 | ||
927 | gomp_mutex_lock (&acc_dev->lock); | |
928 | ||
929 | cur_node.host_start = (uintptr_t) hostaddr; | |
930 | cur_node.host_end = cur_node.host_start + sizeof (void *); | |
931 | n = splay_tree_lookup (&acc_dev->mem_map, &cur_node); | |
932 | ||
933 | if (n == NULL) | |
dc954154 JB |
934 | { |
935 | gomp_mutex_unlock (&acc_dev->lock); | |
936 | gomp_fatal ("struct not mapped for acc_attach"); | |
937 | } | |
5d5be7bf JB |
938 | |
939 | gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, n, (uintptr_t) hostaddr, | |
0ab29cf0 | 940 | 0, NULL, false); |
5d5be7bf JB |
941 | |
942 | gomp_mutex_unlock (&acc_dev->lock); | |
943 | } | |
944 | ||
945 | void | |
946 | acc_attach (void **hostaddr) | |
947 | { | |
948 | acc_attach_async (hostaddr, acc_async_sync); | |
949 | } | |
950 | ||
951 | static void | |
952 | goacc_detach_internal (void **hostaddr, int async, bool finalize) | |
953 | { | |
954 | struct goacc_thread *thr = goacc_thread (); | |
955 | struct gomp_device_descr *acc_dev = thr->dev; | |
956 | struct splay_tree_key_s cur_node; | |
957 | splay_tree_key n; | |
958 | struct goacc_asyncqueue *aq = get_goacc_asyncqueue (async); | |
959 | ||
960 | if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
961 | return; | |
962 | ||
963 | gomp_mutex_lock (&acc_dev->lock); | |
964 | ||
965 | cur_node.host_start = (uintptr_t) hostaddr; | |
966 | cur_node.host_end = cur_node.host_start + sizeof (void *); | |
967 | n = splay_tree_lookup (&acc_dev->mem_map, &cur_node); | |
968 | ||
969 | if (n == NULL) | |
dc954154 JB |
970 | { |
971 | gomp_mutex_unlock (&acc_dev->lock); | |
972 | gomp_fatal ("struct not mapped for acc_detach"); | |
973 | } | |
5d5be7bf JB |
974 | |
975 | gomp_detach_pointer (acc_dev, aq, n, (uintptr_t) hostaddr, finalize, NULL); | |
976 | ||
977 | gomp_mutex_unlock (&acc_dev->lock); | |
978 | } | |
979 | ||
980 | void | |
981 | acc_detach (void **hostaddr) | |
982 | { | |
983 | goacc_detach_internal (hostaddr, acc_async_sync, false); | |
984 | } | |
985 | ||
986 | void | |
987 | acc_detach_async (void **hostaddr, int async) | |
988 | { | |
989 | goacc_detach_internal (hostaddr, async, false); | |
990 | } | |
991 | ||
992 | void | |
993 | acc_detach_finalize (void **hostaddr) | |
994 | { | |
995 | goacc_detach_internal (hostaddr, acc_async_sync, true); | |
996 | } | |
997 | ||
998 | void | |
999 | acc_detach_finalize_async (void **hostaddr, int async) | |
1000 | { | |
1001 | goacc_detach_internal (hostaddr, async, true); | |
1002 | } | |
1003 | ||
378da98f JB |
1004 | /* Some types of (pointer) variables use several consecutive mappings, which |
1005 | must be treated as a group for enter/exit data directives. This function | |
1006 | returns the last mapping in such a group (inclusive), or POS for singleton | |
1007 | mappings. */ | |
57963e39 | 1008 | |
378da98f | 1009 | static int |
8e7e71ff | 1010 | find_group_last (int pos, size_t mapnum, size_t *sizes, unsigned short *kinds) |
378da98f JB |
1011 | { |
1012 | unsigned char kind0 = kinds[pos] & 0xff; | |
8e7e71ff | 1013 | int first_pos = pos; |
57963e39 | 1014 | |
8e7e71ff | 1015 | switch (kind0) |
378da98f | 1016 | { |
8e7e71ff | 1017 | case GOMP_MAP_TO_PSET: |
8d2e5026 JB |
1018 | if (pos + 1 < mapnum |
1019 | && (kinds[pos + 1] & 0xff) == GOMP_MAP_ATTACH) | |
1020 | return pos + 1; | |
1021 | ||
1022 | while (pos + 1 < mapnum | |
1023 | && (kinds[pos + 1] & 0xff) == GOMP_MAP_POINTER) | |
8e7e71ff | 1024 | pos++; |
8d2e5026 JB |
1025 | /* We expect at least one GOMP_MAP_POINTER (if not a single |
1026 | GOMP_MAP_ATTACH) after a GOMP_MAP_TO_PSET. */ | |
8e7e71ff JB |
1027 | assert (pos > first_pos); |
1028 | break; | |
1029 | ||
1030 | case GOMP_MAP_STRUCT: | |
1031 | pos += sizes[pos]; | |
1032 | break; | |
1033 | ||
1034 | case GOMP_MAP_POINTER: | |
1035 | case GOMP_MAP_ALWAYS_POINTER: | |
1036 | /* These mappings are only expected after some other mapping. If we | |
1037 | see one by itself, something has gone wrong. */ | |
1038 | gomp_fatal ("unexpected mapping"); | |
1039 | break; | |
1040 | ||
8d2e5026 JB |
1041 | case GOMP_MAP_ATTACH: |
1042 | break; | |
1043 | ||
8e7e71ff | 1044 | default: |
378da98f JB |
1045 | /* GOMP_MAP_ALWAYS_POINTER can only appear directly after some other |
1046 | mapping. */ | |
8e7e71ff JB |
1047 | if (pos + 1 < mapnum) |
1048 | { | |
1049 | unsigned char kind1 = kinds[pos + 1] & 0xff; | |
1050 | if (kind1 == GOMP_MAP_ALWAYS_POINTER) | |
1051 | return pos + 1; | |
1052 | } | |
378da98f | 1053 | |
8d2e5026 JB |
1054 | /* We can have a single GOMP_MAP_ATTACH mapping after a to/from |
1055 | mapping. */ | |
1056 | if (pos + 1 < mapnum | |
1057 | && (kinds[pos + 1] & 0xff) == GOMP_MAP_ATTACH) | |
1058 | return pos + 1; | |
1059 | ||
8e7e71ff | 1060 | /* We can have zero or more GOMP_MAP_POINTER mappings after a to/from |
378da98f | 1061 | (etc.) mapping. */ |
8d2e5026 JB |
1062 | while (pos + 1 < mapnum |
1063 | && (kinds[pos + 1] & 0xff) == GOMP_MAP_POINTER) | |
8e7e71ff | 1064 | pos++; |
378da98f | 1065 | } |
57963e39 | 1066 | |
8e7e71ff | 1067 | return pos; |
378da98f JB |
1068 | } |
1069 | ||
1070 | /* Map variables for OpenACC "enter data". We can't just call | |
275c736e | 1071 | goacc_map_vars once, because individual mapped variables might have |
378da98f | 1072 | "exit data" called for them at different times. */ |
57963e39 | 1073 | |
57963e39 | 1074 | static void |
378da98f JB |
1075 | goacc_enter_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum, |
1076 | void **hostaddrs, size_t *sizes, | |
1077 | unsigned short *kinds, goacc_aq aq) | |
41dbbb37 | 1078 | { |
6f5b4b64 JB |
1079 | gomp_mutex_lock (&acc_dev->lock); |
1080 | ||
378da98f JB |
1081 | for (size_t i = 0; i < mapnum; i++) |
1082 | { | |
6f5b4b64 JB |
1083 | splay_tree_key n; |
1084 | size_t group_last = find_group_last (i, mapnum, sizes, kinds); | |
1085 | bool struct_p = false; | |
1086 | size_t size, groupnum = (group_last - i) + 1; | |
1087 | ||
1088 | switch (kinds[i] & 0xff) | |
1089 | { | |
1090 | case GOMP_MAP_STRUCT: | |
1091 | { | |
1092 | size = (uintptr_t) hostaddrs[group_last] + sizes[group_last] | |
1093 | - (uintptr_t) hostaddrs[i]; | |
1094 | struct_p = true; | |
1095 | } | |
1096 | break; | |
1097 | ||
1098 | case GOMP_MAP_ATTACH: | |
1099 | size = sizeof (void *); | |
1100 | break; | |
1101 | ||
1102 | default: | |
1103 | size = sizes[i]; | |
1104 | } | |
1105 | ||
1106 | n = lookup_host (acc_dev, hostaddrs[i], size); | |
1107 | ||
1108 | if (n && struct_p) | |
1109 | { | |
1110 | for (size_t j = i + 1; j <= group_last; j++) | |
1111 | { | |
1112 | struct splay_tree_key_s cur_node; | |
1113 | cur_node.host_start = (uintptr_t) hostaddrs[j]; | |
1114 | cur_node.host_end = cur_node.host_start + sizes[j]; | |
1115 | splay_tree_key n2 | |
1116 | = splay_tree_lookup (&acc_dev->mem_map, &cur_node); | |
1117 | if (!n2 | |
1118 | || n2->tgt != n->tgt | |
1119 | || n2->host_start - n->host_start | |
1120 | != n2->tgt_offset - n->tgt_offset) | |
1121 | { | |
1122 | gomp_mutex_unlock (&acc_dev->lock); | |
1123 | gomp_fatal ("Trying to map into device [%p..%p) structure " | |
1124 | "element when other mapped elements from the " | |
1125 | "same structure weren't mapped together with " | |
1126 | "it", (void *) cur_node.host_start, | |
1127 | (void *) cur_node.host_end); | |
1128 | } | |
1129 | } | |
1130 | /* This is a special case because we must increment the refcount by | |
1131 | the number of mapped struct elements, rather than by one. */ | |
1132 | if (n->refcount != REFCOUNT_INFINITY) | |
1133 | n->refcount += groupnum - 1; | |
1134 | n->dynamic_refcount += groupnum - 1; | |
1135 | } | |
1136 | else if (n && groupnum == 1) | |
1137 | { | |
1138 | void *h = hostaddrs[i]; | |
1139 | size_t s = sizes[i]; | |
1140 | ||
6f5b4b64 | 1141 | if ((kinds[i] & 0xff) == GOMP_MAP_ATTACH) |
bc4ed079 JB |
1142 | { |
1143 | gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, n, | |
0ab29cf0 | 1144 | (uintptr_t) h, s, NULL, false); |
bc4ed079 JB |
1145 | /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic |
1146 | reference counts ('n->refcount', 'n->dynamic_refcount'). */ | |
1147 | } | |
1148 | else | |
1149 | goacc_map_var_existing (acc_dev, h, s, n); | |
6f5b4b64 JB |
1150 | } |
1151 | else if (n && groupnum > 1) | |
1152 | { | |
da8e0e11 | 1153 | assert (n->refcount != REFCOUNT_LINK); |
6f5b4b64 JB |
1154 | |
1155 | for (size_t j = i + 1; j <= group_last; j++) | |
1156 | if ((kinds[j] & 0xff) == GOMP_MAP_ATTACH) | |
1157 | { | |
1158 | splay_tree_key m | |
1159 | = lookup_host (acc_dev, hostaddrs[j], sizeof (void *)); | |
1160 | gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, m, | |
0ab29cf0 CLT |
1161 | (uintptr_t) hostaddrs[j], sizes[j], NULL, |
1162 | false); | |
6f5b4b64 JB |
1163 | } |
1164 | ||
1165 | bool processed = false; | |
1166 | ||
1167 | struct target_mem_desc *tgt = n->tgt; | |
da8e0e11 | 1168 | |
f6ce1e77 TS |
1169 | /* Minimal OpenACC variant corresponding to PR96668 |
1170 | "[OpenMP] Re-mapping allocated but previously unallocated | |
1171 | allocatable does not work" 'libgomp/target.c' changes, so that | |
1172 | OpenACC 'declare' code à la PR106643 | |
da8e0e11 TS |
1173 | "[gfortran + OpenACC] Allocate in module causes refcount error" |
1174 | has a chance to work. */ | |
1175 | if ((kinds[i] & 0xff) == GOMP_MAP_TO_PSET | |
1176 | && tgt->list_count == 0) | |
1177 | { | |
1178 | /* 'declare target'. */ | |
1179 | assert (n->refcount == REFCOUNT_INFINITY); | |
1180 | ||
1181 | for (size_t k = 1; k < groupnum; k++) | |
1182 | { | |
1183 | /* The only thing we expect to see here. */ | |
1184 | assert ((kinds[i + k] & 0xff) == GOMP_MAP_POINTER); | |
1185 | } | |
1186 | ||
f6ce1e77 TS |
1187 | /* Let 'goacc_map_vars' -> 'gomp_map_vars_internal' handle |
1188 | this. */ | |
1189 | gomp_mutex_unlock (&acc_dev->lock); | |
1190 | struct target_mem_desc *tgt_ | |
1191 | = goacc_map_vars (acc_dev, aq, groupnum, &hostaddrs[i], NULL, | |
1192 | &sizes[i], &kinds[i], true, | |
1193 | GOMP_MAP_VARS_ENTER_DATA); | |
1194 | assert (tgt_ == NULL); | |
1195 | gomp_mutex_lock (&acc_dev->lock); | |
1196 | ||
da8e0e11 TS |
1197 | /* Given that 'goacc_exit_data_internal'/'goacc_exit_datum_1' |
1198 | will always see 'n->refcount == REFCOUNT_INFINITY', | |
1199 | there's no need to adjust 'n->dynamic_refcount' here. */ | |
1200 | ||
1201 | processed = true; | |
1202 | } | |
1203 | else | |
1204 | assert (n->refcount != REFCOUNT_INFINITY); | |
1205 | ||
6f5b4b64 JB |
1206 | for (size_t j = 0; j < tgt->list_count; j++) |
1207 | if (tgt->list[j].key == n) | |
1208 | { | |
1209 | /* We are processing a group of mappings (e.g. | |
1210 | [GOMP_MAP_TO, GOMP_MAP_TO_PSET, GOMP_MAP_POINTER]). | |
1211 | Find the right group in the target_mem_desc's variable | |
1212 | list, and increment the refcounts for each item in that | |
1213 | group. */ | |
1214 | for (size_t k = 0; k < groupnum; k++) | |
bc4ed079 JB |
1215 | if (j + k < tgt->list_count |
1216 | && tgt->list[j + k].key | |
1217 | && !tgt->list[j + k].is_attach) | |
6f5b4b64 JB |
1218 | { |
1219 | tgt->list[j + k].key->refcount++; | |
1220 | tgt->list[j + k].key->dynamic_refcount++; | |
1221 | } | |
1222 | processed = true; | |
1223 | break; | |
1224 | } | |
e38fdba4 | 1225 | |
6f5b4b64 JB |
1226 | if (!processed) |
1227 | { | |
1228 | gomp_mutex_unlock (&acc_dev->lock); | |
1229 | gomp_fatal ("dynamic refcount incrementing failed for " | |
1230 | "pointer/pset"); | |
1231 | } | |
1232 | } | |
1233 | else if (hostaddrs[i]) | |
1234 | { | |
1235 | /* The data is not mapped already. Map it now, unless the first | |
1236 | member in the group has a NULL pointer (e.g. a non-present | |
1237 | optional parameter). */ | |
1238 | gomp_mutex_unlock (&acc_dev->lock); | |
1239 | ||
1240 | struct target_mem_desc *tgt | |
275c736e CLT |
1241 | = goacc_map_vars (acc_dev, aq, groupnum, &hostaddrs[i], NULL, |
1242 | &sizes[i], &kinds[i], true, | |
1243 | GOMP_MAP_VARS_ENTER_DATA); | |
6f5b4b64 JB |
1244 | assert (tgt); |
1245 | ||
1246 | gomp_mutex_lock (&acc_dev->lock); | |
1247 | ||
1248 | for (size_t j = 0; j < tgt->list_count; j++) | |
1249 | { | |
1250 | n = tgt->list[j].key; | |
bc4ed079 | 1251 | if (n && !tgt->list[j].is_attach) |
6f5b4b64 JB |
1252 | n->dynamic_refcount++; |
1253 | } | |
1254 | } | |
41dbbb37 | 1255 | |
378da98f | 1256 | i = group_last; |
e38fdba4 | 1257 | } |
6f5b4b64 JB |
1258 | |
1259 | gomp_mutex_unlock (&acc_dev->lock); | |
378da98f | 1260 | } |
41dbbb37 | 1261 | |
378da98f | 1262 | /* Unmap variables for OpenACC "exit data". */ |
41dbbb37 | 1263 | |
378da98f JB |
1264 | static void |
1265 | goacc_exit_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum, | |
1266 | void **hostaddrs, size_t *sizes, | |
1267 | unsigned short *kinds, goacc_aq aq) | |
1268 | { | |
1269 | gomp_mutex_lock (&acc_dev->lock); | |
41dbbb37 | 1270 | |
8e7e71ff JB |
1271 | /* Handle "detach" before copyback/deletion of mapped data. */ |
1272 | for (size_t i = 0; i < mapnum; ++i) | |
1273 | { | |
1274 | unsigned char kind = kinds[i] & 0xff; | |
1275 | bool finalize = false; | |
1276 | switch (kind) | |
1277 | { | |
1278 | case GOMP_MAP_FORCE_DETACH: | |
1279 | finalize = true; | |
1280 | /* Fallthrough. */ | |
1281 | ||
1282 | case GOMP_MAP_DETACH: | |
1283 | { | |
1284 | struct splay_tree_key_s cur_node; | |
1285 | uintptr_t hostaddr = (uintptr_t) hostaddrs[i]; | |
1286 | cur_node.host_start = hostaddr; | |
1287 | cur_node.host_end = cur_node.host_start + sizeof (void *); | |
1288 | splay_tree_key n | |
1289 | = splay_tree_lookup (&acc_dev->mem_map, &cur_node); | |
1290 | ||
1291 | if (n == NULL) | |
2e24d457 TS |
1292 | { |
1293 | gomp_mutex_unlock (&acc_dev->lock); | |
1294 | gomp_fatal ("struct not mapped for detach operation"); | |
1295 | } | |
8e7e71ff JB |
1296 | |
1297 | gomp_detach_pointer (acc_dev, aq, n, hostaddr, finalize, NULL); | |
1298 | } | |
1299 | break; | |
1300 | default: | |
1301 | ; | |
1302 | } | |
1303 | } | |
1304 | ||
378da98f | 1305 | for (size_t i = 0; i < mapnum; ++i) |
829c6349 | 1306 | { |
378da98f | 1307 | unsigned char kind = kinds[i] & 0xff; |
41dbbb37 | 1308 | |
378da98f | 1309 | switch (kind) |
ba40277f | 1310 | { |
378da98f JB |
1311 | case GOMP_MAP_FROM: |
1312 | case GOMP_MAP_FORCE_FROM: | |
378da98f JB |
1313 | case GOMP_MAP_TO_PSET: |
1314 | case GOMP_MAP_POINTER: | |
1315 | case GOMP_MAP_DELETE: | |
1316 | case GOMP_MAP_RELEASE: | |
1317 | { | |
1318 | struct splay_tree_key_s cur_node; | |
1319 | size_t size; | |
bc4ed079 | 1320 | if (kind == GOMP_MAP_POINTER) |
378da98f JB |
1321 | size = sizeof (void *); |
1322 | else | |
1323 | size = sizes[i]; | |
1324 | cur_node.host_start = (uintptr_t) hostaddrs[i]; | |
1325 | cur_node.host_end = cur_node.host_start + size; | |
1326 | splay_tree_key n | |
1327 | = splay_tree_lookup (&acc_dev->mem_map, &cur_node); | |
1328 | ||
1329 | if (n == NULL) | |
1330 | continue; | |
1331 | ||
6f5b4b64 | 1332 | goacc_exit_datum_1 (acc_dev, hostaddrs[i], size, kind, n, aq); |
378da98f JB |
1333 | } |
1334 | break; | |
8e7e71ff JB |
1335 | |
1336 | case GOMP_MAP_STRUCT: | |
1809628f | 1337 | /* Skip the 'GOMP_MAP_STRUCT' itself, and use the regular processing |
1afc4672 TS |
1338 | for all its entries. This special handling exists for GCC 10.1 |
1339 | compatibility; afterwards, we're not generating these no-op | |
1340 | 'GOMP_MAP_STRUCT's anymore. */ | |
8e7e71ff JB |
1341 | break; |
1342 | ||
bc4ed079 JB |
1343 | case GOMP_MAP_DETACH: |
1344 | case GOMP_MAP_FORCE_DETACH: | |
1345 | /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic | |
1346 | reference counts ('n->refcount', 'n->dynamic_refcount'). */ | |
1347 | break; | |
1348 | ||
378da98f JB |
1349 | default: |
1350 | gomp_fatal (">>>> goacc_exit_data_internal UNHANDLED kind 0x%.2x", | |
1351 | kind); | |
1f4c5b9b | 1352 | } |
829c6349 | 1353 | } |
41dbbb37 | 1354 | |
829c6349 | 1355 | gomp_mutex_unlock (&acc_dev->lock); |
57963e39 TS |
1356 | } |
1357 | ||
79993639 TS |
1358 | static void |
1359 | goacc_enter_exit_data_internal (int flags_m, size_t mapnum, void **hostaddrs, | |
1360 | size_t *sizes, unsigned short *kinds, | |
1361 | bool data_enter, int async, int num_waits, | |
1362 | va_list *ap) | |
57963e39 TS |
1363 | { |
1364 | int flags = GOACC_FLAGS_UNMARSHAL (flags_m); | |
1365 | ||
1366 | struct goacc_thread *thr; | |
1367 | struct gomp_device_descr *acc_dev; | |
57963e39 TS |
1368 | |
1369 | goacc_lazy_initialize (); | |
1370 | ||
1371 | thr = goacc_thread (); | |
1372 | acc_dev = thr->dev; | |
1373 | ||
57963e39 TS |
1374 | bool profiling_p = GOACC_PROFILING_DISPATCH_P (true); |
1375 | ||
1376 | acc_prof_info prof_info; | |
1377 | if (profiling_p) | |
1378 | { | |
1379 | thr->prof_info = &prof_info; | |
1380 | ||
1381 | prof_info.event_type | |
1382 | = data_enter ? acc_ev_enter_data_start : acc_ev_exit_data_start; | |
1383 | prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES; | |
1384 | prof_info.version = _ACC_PROF_INFO_VERSION; | |
1385 | prof_info.device_type = acc_device_type (acc_dev->type); | |
1386 | prof_info.device_number = acc_dev->target_id; | |
1387 | prof_info.thread_id = -1; | |
1388 | prof_info.async = async; | |
1389 | prof_info.async_queue = prof_info.async; | |
1390 | prof_info.src_file = NULL; | |
1391 | prof_info.func_name = NULL; | |
1392 | prof_info.line_no = -1; | |
1393 | prof_info.end_line_no = -1; | |
1394 | prof_info.func_line_no = -1; | |
1395 | prof_info.func_end_line_no = -1; | |
1396 | } | |
1397 | acc_event_info enter_exit_data_event_info; | |
1398 | if (profiling_p) | |
1399 | { | |
1400 | enter_exit_data_event_info.other_event.event_type | |
1401 | = prof_info.event_type; | |
1402 | enter_exit_data_event_info.other_event.valid_bytes | |
1403 | = _ACC_OTHER_EVENT_INFO_VALID_BYTES; | |
1404 | enter_exit_data_event_info.other_event.parent_construct | |
1405 | = data_enter ? acc_construct_enter_data : acc_construct_exit_data; | |
1406 | enter_exit_data_event_info.other_event.implicit = 0; | |
1407 | enter_exit_data_event_info.other_event.tool_info = NULL; | |
1408 | } | |
1409 | acc_api_info api_info; | |
1410 | if (profiling_p) | |
1411 | { | |
1412 | thr->api_info = &api_info; | |
1413 | ||
1414 | api_info.device_api = acc_device_api_none; | |
1415 | api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES; | |
1416 | api_info.device_type = prof_info.device_type; | |
1417 | api_info.vendor = -1; | |
1418 | api_info.device_handle = NULL; | |
1419 | api_info.context_handle = NULL; | |
1420 | api_info.async_handle = NULL; | |
1421 | } | |
1422 | ||
1423 | if (profiling_p) | |
1424 | goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info, | |
1425 | &api_info); | |
1426 | ||
1427 | if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM) | |
1428 | || (flags & GOACC_FLAG_HOST_FALLBACK)) | |
1429 | { | |
1430 | prof_info.device_type = acc_device_host; | |
1431 | api_info.device_type = prof_info.device_type; | |
1432 | ||
1433 | goto out_prof; | |
1434 | } | |
1435 | ||
1436 | if (num_waits) | |
79993639 | 1437 | goacc_wait (async, num_waits, ap); |
57963e39 | 1438 | |
378da98f | 1439 | goacc_aq aq = get_goacc_asyncqueue (async); |
57963e39 TS |
1440 | |
1441 | if (data_enter) | |
378da98f | 1442 | goacc_enter_data_internal (acc_dev, mapnum, hostaddrs, sizes, kinds, aq); |
57963e39 | 1443 | else |
378da98f | 1444 | goacc_exit_data_internal (acc_dev, mapnum, hostaddrs, sizes, kinds, aq); |
57963e39 TS |
1445 | |
1446 | out_prof: | |
1447 | if (profiling_p) | |
1448 | { | |
1449 | prof_info.event_type | |
1450 | = data_enter ? acc_ev_enter_data_end : acc_ev_exit_data_end; | |
1451 | enter_exit_data_event_info.other_event.event_type = prof_info.event_type; | |
1452 | goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info, | |
1453 | &api_info); | |
1454 | ||
1455 | thr->prof_info = NULL; | |
1456 | thr->api_info = NULL; | |
1457 | } | |
1458 | } | |
0a77c703 | 1459 | |
7aefef31 AS |
1460 | /* Legacy entry point (GCC 11 and earlier). */ |
1461 | ||
79993639 TS |
1462 | void |
1463 | GOACC_enter_exit_data (int flags_m, size_t mapnum, void **hostaddrs, | |
1464 | size_t *sizes, unsigned short *kinds, int async, | |
1465 | int num_waits, ...) | |
1466 | { | |
1467 | /* Determine if this is an OpenACC "enter data". */ | |
1468 | bool data_enter = false; | |
1469 | for (size_t i = 0; i < mapnum; ++i) | |
1470 | { | |
1471 | unsigned char kind = kinds[i] & 0xff; | |
1472 | ||
1473 | if (kind == GOMP_MAP_POINTER | |
1474 | || kind == GOMP_MAP_TO_PSET | |
1475 | || kind == GOMP_MAP_STRUCT) | |
1476 | continue; | |
1477 | ||
1478 | if (kind == GOMP_MAP_FORCE_ALLOC | |
1479 | || kind == GOMP_MAP_FORCE_PRESENT | |
1480 | || kind == GOMP_MAP_ATTACH | |
1481 | || kind == GOMP_MAP_FORCE_TO | |
1482 | || kind == GOMP_MAP_TO | |
1483 | || kind == GOMP_MAP_ALLOC) | |
1484 | { | |
1485 | data_enter = true; | |
1486 | break; | |
1487 | } | |
1488 | ||
1489 | if (kind == GOMP_MAP_RELEASE | |
1490 | || kind == GOMP_MAP_DELETE | |
1491 | || kind == GOMP_MAP_DETACH | |
1492 | || kind == GOMP_MAP_FORCE_DETACH | |
1493 | || kind == GOMP_MAP_FROM | |
1494 | || kind == GOMP_MAP_FORCE_FROM) | |
1495 | break; | |
1496 | ||
1497 | gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x", | |
1498 | kind); | |
1499 | } | |
1500 | ||
1501 | va_list ap; | |
1502 | va_start (ap, num_waits); | |
1503 | goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds, | |
1504 | data_enter, async, num_waits, &ap); | |
1505 | va_end (ap); | |
1506 | } | |
1507 | ||
7aefef31 AS |
1508 | void |
1509 | GOACC_enter_data (int flags_m, size_t mapnum, void **hostaddrs, | |
1510 | size_t *sizes, unsigned short *kinds, int async, | |
1511 | int num_waits, ...) | |
1512 | { | |
1513 | va_list ap; | |
1514 | va_start (ap, num_waits); | |
1515 | goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds, | |
1516 | true, async, num_waits, &ap); | |
1517 | va_end (ap); | |
1518 | } | |
1519 | ||
1520 | void | |
1521 | GOACC_exit_data (int flags_m, size_t mapnum, void **hostaddrs, | |
1522 | size_t *sizes, unsigned short *kinds, int async, | |
1523 | int num_waits, ...) | |
1524 | { | |
1525 | va_list ap; | |
1526 | va_start (ap, num_waits); | |
1527 | goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds, | |
1528 | false, async, num_waits, &ap); | |
1529 | va_end (ap); | |
1530 | } | |
1531 | ||
0a77c703 TS |
1532 | void |
1533 | GOACC_declare (int flags_m, size_t mapnum, | |
1534 | void **hostaddrs, size_t *sizes, unsigned short *kinds) | |
1535 | { | |
1536 | for (size_t i = 0; i < mapnum; i++) | |
1537 | { | |
1538 | unsigned char kind = kinds[i] & 0xff; | |
1539 | ||
1540 | if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET) | |
1541 | continue; | |
1542 | ||
1543 | switch (kind) | |
1544 | { | |
79993639 TS |
1545 | case GOMP_MAP_ALLOC: |
1546 | if (acc_is_present (hostaddrs[i], sizes[i])) | |
1547 | continue; | |
1548 | /* FALLTHRU */ | |
0a77c703 | 1549 | case GOMP_MAP_FORCE_ALLOC: |
79993639 | 1550 | case GOMP_MAP_TO: |
0a77c703 | 1551 | case GOMP_MAP_FORCE_TO: |
79993639 TS |
1552 | goacc_enter_exit_data_internal (flags_m, 1, &hostaddrs[i], &sizes[i], |
1553 | &kinds[i], true, GOMP_ASYNC_SYNC, 0, NULL); | |
1554 | break; | |
1555 | ||
1556 | case GOMP_MAP_FROM: | |
1557 | case GOMP_MAP_FORCE_FROM: | |
0a77c703 TS |
1558 | case GOMP_MAP_RELEASE: |
1559 | case GOMP_MAP_DELETE: | |
79993639 TS |
1560 | goacc_enter_exit_data_internal (flags_m, 1, &hostaddrs[i], &sizes[i], |
1561 | &kinds[i], false, GOMP_ASYNC_SYNC, 0, NULL); | |
0a77c703 TS |
1562 | break; |
1563 | ||
1564 | case GOMP_MAP_FORCE_DEVICEPTR: | |
1565 | break; | |
1566 | ||
0a77c703 TS |
1567 | case GOMP_MAP_FORCE_PRESENT: |
1568 | if (!acc_is_present (hostaddrs[i], sizes[i])) | |
1569 | gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i], | |
1570 | (unsigned long) sizes[i]); | |
1571 | break; | |
1572 | ||
1573 | default: | |
1574 | assert (0); | |
1575 | break; | |
1576 | } | |
1577 | } | |
1578 | } |