]> git.ipfire.org Git - thirdparty/gcc.git/blame - libgomp/oacc-parallel.c
projects / thirdparty / gcc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PR90743] Fortran 'allocatable' with OpenACC data/OpenMP 'target' 'map' clauses
[thirdparty/gcc.git] / libgomp / oacc-parallel.c
CommitLineData
fbd26352 1/* Copyright (C) 2013-2019 Free Software Foundation, Inc.
ca4c3545 2
3 Contributed by Mentor Embedded.
4
5 This file is part of the GNU Offloading and Multi Processing Library
6 (libgomp).
7
8 Libgomp is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
11 any later version.
12
13 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 more details.
17
18 Under Section 7 of GPL version 3, you are granted additional
19 permissions described in the GCC Runtime Library Exception, version
20 3.1, as published by the Free Software Foundation.
21
22 You should have received a copy of the GNU General Public License and
23 a copy of the GCC Runtime Library Exception along with this program;
24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
25 <http://www.gnu.org/licenses/>. */
26
27/* This file handles OpenACC constructs. */
28
29#include "openacc.h"
30#include "libgomp.h"
31#include "libgomp_g.h"
32#include "gomp-constants.h"
33#include "oacc-int.h"
2634aed9 34#ifdef HAVE_INTTYPES_H
35# include <inttypes.h> /* For PRIu64. */
36#endif
ca4c3545 37#include <string.h>
38#include <stdarg.h>
39#include <assert.h>
ca4c3545 40
813421cd 41
42/* In the ABI, the GOACC_FLAGs are encoded as an inverted bitmask, so that we
43 continue to support the following two legacy values. */
44_Static_assert (GOACC_FLAGS_UNMARSHAL (GOMP_DEVICE_ICV) == 0,
45 "legacy GOMP_DEVICE_ICV broken");
46_Static_assert (GOACC_FLAGS_UNMARSHAL (GOMP_DEVICE_HOST_FALLBACK)
47 == GOACC_FLAG_HOST_FALLBACK,
48 "legacy GOMP_DEVICE_HOST_FALLBACK broken");
49
50
737cc978 51/* Returns the number of mappings associated with the pointer or pset. PSET
52 have three mappings, whereas pointer have two. */
53
ca4c3545 54static int
737cc978 55find_pointer (int pos, size_t mapnum, unsigned short *kinds)
ca4c3545 56{
57 if (pos + 1 >= mapnum)
58 return 0;
59
60 unsigned char kind = kinds[pos+1] & 0xff;
61
737cc978 62 if (kind == GOMP_MAP_TO_PSET)
63 return 3;
64 else if (kind == GOMP_MAP_POINTER)
65 return 2;
66
67 return 0;
68}
69
70/* Handle the mapping pair that are presented when a
71 deviceptr clause is used with Fortran. */
72
73static void
74handle_ftn_pointers (size_t mapnum, void **hostaddrs, size_t *sizes,
75 unsigned short *kinds)
76{
77 int i;
78
79 for (i = 0; i < mapnum; i++)
80 {
81 unsigned short kind1 = kinds[i] & 0xff;
82
83 /* Handle Fortran deviceptr clause. */
84 if (kind1 == GOMP_MAP_FORCE_DEVICEPTR)
85 {
86 unsigned short kind2;
87
88 if (i < (signed)mapnum - 1)
89 kind2 = kinds[i + 1] & 0xff;
90 else
91 kind2 = 0xffff;
92
93 if (sizes[i] == sizeof (void *))
94 continue;
95
96 /* At this point, we're dealing with a Fortran deviceptr.
97 If the next element is not what we're expecting, then
98 this is an instance of where the deviceptr variable was
99 not used within the region and the pointer was removed
100 by the gimplifier. */
101 if (kind2 == GOMP_MAP_POINTER
102 && sizes[i + 1] == 0
103 && hostaddrs[i] == *(void **)hostaddrs[i + 1])
104 {
105 kinds[i+1] = kinds[i];
106 sizes[i+1] = sizeof (void *);
107 }
108
109 /* Invalidate the entry. */
110 hostaddrs[i] = NULL;
111 }
112 }
ca4c3545 113}
114
e561d5e1 115static void goacc_wait (int async, int num_waits, va_list *ap);
116
117
813421cd 118/* Launch a possibly offloaded function with FLAGS. FN is the host fn
e561d5e1 119 address. MAPNUM, HOSTADDRS, SIZES & KINDS describe the memory
120 blocks to be copied to/from the device. Varadic arguments are
121 keyed optional parameters terminated with a zero. */
ca4c3545 122
123void
813421cd 124GOACC_parallel_keyed (int flags_m, void (*fn) (void *),
e561d5e1 125 size_t mapnum, void **hostaddrs, size_t *sizes,
126 unsigned short *kinds, ...)
ca4c3545 127{
813421cd 128 int flags = GOACC_FLAGS_UNMARSHAL (flags_m);
129
ca4c3545 130 va_list ap;
131 struct goacc_thread *thr;
132 struct gomp_device_descr *acc_dev;
133 struct target_mem_desc *tgt;
134 void **devaddrs;
135 unsigned int i;
136 struct splay_tree_key_s k;
137 splay_tree_key tgt_fn_key;
138 void (*tgt_fn);
e561d5e1 139 int async = GOMP_ASYNC_SYNC;
140 unsigned dims[GOMP_DIM_MAX];
141 unsigned tag;
ca4c3545 142
2634aed9 143#ifdef HAVE_INTTYPES_H
e561d5e1 144 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
145 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
2634aed9 146#else
e561d5e1 147 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
148 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
2634aed9 149#endif
0a1fe572 150 goacc_lazy_initialize ();
ca4c3545 151
152 thr = goacc_thread ();
153 acc_dev = thr->dev;
154
5feffd51 155 bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);
156
157 acc_prof_info prof_info;
158 if (profiling_p)
159 {
160 thr->prof_info = &prof_info;
161
162 prof_info.event_type = acc_ev_compute_construct_start;
163 prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
164 prof_info.version = _ACC_PROF_INFO_VERSION;
165 prof_info.device_type = acc_device_type (acc_dev->type);
166 prof_info.device_number = acc_dev->target_id;
167 prof_info.thread_id = -1;
168 prof_info.async = async;
169 prof_info.async_queue = prof_info.async;
170 prof_info.src_file = NULL;
171 prof_info.func_name = NULL;
172 prof_info.line_no = -1;
173 prof_info.end_line_no = -1;
174 prof_info.func_line_no = -1;
175 prof_info.func_end_line_no = -1;
176 }
177 acc_event_info compute_construct_event_info;
178 if (profiling_p)
179 {
180 compute_construct_event_info.other_event.event_type
181 = prof_info.event_type;
182 compute_construct_event_info.other_event.valid_bytes
183 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
184 compute_construct_event_info.other_event.parent_construct
185 = acc_construct_parallel;
186 compute_construct_event_info.other_event.implicit = 0;
187 compute_construct_event_info.other_event.tool_info = NULL;
188 }
189 acc_api_info api_info;
190 if (profiling_p)
191 {
192 thr->api_info = &api_info;
193
194 api_info.device_api = acc_device_api_none;
195 api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
196 api_info.device_type = prof_info.device_type;
197 api_info.vendor = -1;
198 api_info.device_handle = NULL;
199 api_info.context_handle = NULL;
200 api_info.async_handle = NULL;
201 }
202
203 if (profiling_p)
204 goacc_profiling_dispatch (&prof_info, &compute_construct_event_info,
205 &api_info);
206
737cc978 207 handle_ftn_pointers (mapnum, hostaddrs, sizes, kinds);
208
ca4c3545 209 /* Host fallback if "if" clause is false or if the current device is set to
210 the host. */
813421cd 211 if (flags & GOACC_FLAG_HOST_FALLBACK)
ca4c3545 212 {
5feffd51 213 prof_info.device_type = acc_device_host;
214 api_info.device_type = prof_info.device_type;
ca4c3545 215 goacc_save_and_set_bind (acc_device_host);
216 fn (hostaddrs);
217 goacc_restore_bind ();
5feffd51 218 goto out_prof;
ca4c3545 219 }
220 else if (acc_device_type (acc_dev->type) == acc_device_host)
221 {
222 fn (hostaddrs);
5feffd51 223 goto out_prof;
ca4c3545 224 }
225
3a37a410 226 /* Default: let the runtime choose. */
227 for (i = 0; i != GOMP_DIM_MAX; i++)
228 dims[i] = 0;
229
e561d5e1 230 va_start (ap, kinds);
231 /* TODO: This will need amending when device_type is implemented. */
232 while ((tag = va_arg (ap, unsigned)) != 0)
933b0582 233 {
e561d5e1 234 if (GOMP_LAUNCH_DEVICE (tag))
235 gomp_fatal ("device_type '%d' offload parameters, libgomp is too old",
236 GOMP_LAUNCH_DEVICE (tag));
237
238 switch (GOMP_LAUNCH_CODE (tag))
239 {
240 case GOMP_LAUNCH_DIM:
241 {
242 unsigned mask = GOMP_LAUNCH_OP (tag);
243
244 for (i = 0; i != GOMP_DIM_MAX; i++)
245 if (mask & GOMP_DIM_MASK (i))
246 dims[i] = va_arg (ap, unsigned);
247 }
248 break;
249
250 case GOMP_LAUNCH_ASYNC:
251 {
252 /* Small constant values are encoded in the operand. */
253 async = GOMP_LAUNCH_OP (tag);
254
255 if (async == GOMP_LAUNCH_OP_MAX)
256 async = va_arg (ap, unsigned);
5feffd51 257
258 if (profiling_p)
259 {
260 prof_info.async = async;
261 prof_info.async_queue = prof_info.async;
262 }
263
e561d5e1 264 break;
265 }
266
267 case GOMP_LAUNCH_WAIT:
268 {
269 unsigned num_waits = GOMP_LAUNCH_OP (tag);
d572a4c3 270 goacc_wait (async, num_waits, &ap);
e561d5e1 271 break;
272 }
273
274 default:
275 gomp_fatal ("unrecognized offload code '%d',"
276 " libgomp is too old", GOMP_LAUNCH_CODE (tag));
277 }
933b0582 278 }
e561d5e1 279 va_end (ap);
ca4c3545 280
ca4c3545 281 if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC))
282 {
283 k.host_start = (uintptr_t) fn;
284 k.host_end = k.host_start + 1;
0d8c703d 285 gomp_mutex_lock (&acc_dev->lock);
286 tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k);
287 gomp_mutex_unlock (&acc_dev->lock);
ca4c3545 288
289 if (tgt_fn_key == NULL)
290 gomp_fatal ("target function wasn't mapped");
291
0a1fe572 292 tgt_fn = (void (*)) tgt_fn_key->tgt_offset;
ca4c3545 293 }
294 else
295 tgt_fn = (void (*)) fn;
296
5feffd51 297 acc_event_info enter_exit_data_event_info;
298 if (profiling_p)
299 {
300 prof_info.event_type = acc_ev_enter_data_start;
301 enter_exit_data_event_info.other_event.event_type
302 = prof_info.event_type;
303 enter_exit_data_event_info.other_event.valid_bytes
304 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
305 enter_exit_data_event_info.other_event.parent_construct
306 = compute_construct_event_info.other_event.parent_construct;
307 enter_exit_data_event_info.other_event.implicit = 1;
308 enter_exit_data_event_info.other_event.tool_info = NULL;
309 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
310 &api_info);
311 }
312
534b5e00 313 goacc_aq aq = get_goacc_asyncqueue (async);
ca4c3545 314
534b5e00 315 tgt = gomp_map_vars_async (acc_dev, aq, mapnum, hostaddrs, NULL, sizes, kinds,
316 true, GOMP_MAP_VARS_OPENACC);
5feffd51 317 if (profiling_p)
318 {
319 prof_info.event_type = acc_ev_enter_data_end;
320 enter_exit_data_event_info.other_event.event_type
321 = prof_info.event_type;
322 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
323 &api_info);
324 }
534b5e00 325
a6f19a7c 326 devaddrs = gomp_alloca (sizeof (void *) * mapnum);
ca4c3545 327 for (i = 0; i < mapnum; i++)
561ffc69 328 if (tgt->list[i].key != NULL)
329 devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start
330 + tgt->list[i].key->tgt_offset
331 + tgt->list[i].offset);
332 else
333 devaddrs[i] = NULL;
534b5e00 334 if (aq == NULL)
5feffd51 335 acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs, dims,
336 tgt);
337 else
338 acc_dev->openacc.async.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
339 dims, tgt, aq);
340
341 if (profiling_p)
534b5e00 342 {
5feffd51 343 prof_info.event_type = acc_ev_exit_data_start;
344 enter_exit_data_event_info.other_event.event_type = prof_info.event_type;
345 enter_exit_data_event_info.other_event.tool_info = NULL;
346 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
347 &api_info);
534b5e00 348 }
5feffd51 349
350 /* If running synchronously, unmap immediately. */
351 if (aq == NULL)
352 gomp_unmap_vars (tgt, true);
ca4c3545 353 else
5feffd51 354 gomp_unmap_vars_async (tgt, true, aq);
355
356 if (profiling_p)
737cc978 357 {
5feffd51 358 prof_info.event_type = acc_ev_exit_data_end;
359 enter_exit_data_event_info.other_event.event_type = prof_info.event_type;
360 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
361 &api_info);
362 }
363
364 out_prof:
365 if (profiling_p)
366 {
367 prof_info.event_type = acc_ev_compute_construct_end;
368 compute_construct_event_info.other_event.event_type
369 = prof_info.event_type;
370 goacc_profiling_dispatch (&prof_info, &compute_construct_event_info,
371 &api_info);
372
373 thr->prof_info = NULL;
374 thr->api_info = NULL;
737cc978 375 }
ca4c3545 376}
377
75a3472e 378/* Legacy entry point (GCC 5). Only provide host fallback execution. */
e561d5e1 379
380void
813421cd 381GOACC_parallel (int flags_m, void (*fn) (void *),
e561d5e1 382 size_t mapnum, void **hostaddrs, size_t *sizes,
383 unsigned short *kinds,
384 int num_gangs, int num_workers, int vector_length,
385 int async, int num_waits, ...)
386{
387 goacc_save_and_set_bind (acc_device_host);
388 fn (hostaddrs);
389 goacc_restore_bind ();
390}
391
ca4c3545 392void
813421cd 393GOACC_data_start (int flags_m, size_t mapnum,
ca4c3545 394 void **hostaddrs, size_t *sizes, unsigned short *kinds)
395{
813421cd 396 int flags = GOACC_FLAGS_UNMARSHAL (flags_m);
397
ca4c3545 398 struct target_mem_desc *tgt;
399
2634aed9 400#ifdef HAVE_INTTYPES_H
401 gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
402 __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
403#else
404 gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
405 __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
406#endif
ca4c3545 407
0a1fe572 408 goacc_lazy_initialize ();
ca4c3545 409
410 struct goacc_thread *thr = goacc_thread ();
411 struct gomp_device_descr *acc_dev = thr->dev;
412
5feffd51 413 bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);
414
415 acc_prof_info prof_info;
416 if (profiling_p)
417 {
418 thr->prof_info = &prof_info;
419
420 prof_info.event_type = acc_ev_enter_data_start;
421 prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
422 prof_info.version = _ACC_PROF_INFO_VERSION;
423 prof_info.device_type = acc_device_type (acc_dev->type);
424 prof_info.device_number = acc_dev->target_id;
425 prof_info.thread_id = -1;
426 prof_info.async = acc_async_sync; /* Always synchronous. */
427 prof_info.async_queue = prof_info.async;
428 prof_info.src_file = NULL;
429 prof_info.func_name = NULL;
430 prof_info.line_no = -1;
431 prof_info.end_line_no = -1;
432 prof_info.func_line_no = -1;
433 prof_info.func_end_line_no = -1;
434 }
435 acc_event_info enter_data_event_info;
436 if (profiling_p)
437 {
438 enter_data_event_info.other_event.event_type
439 = prof_info.event_type;
440 enter_data_event_info.other_event.valid_bytes
441 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
442 enter_data_event_info.other_event.parent_construct = acc_construct_data;
443 for (int i = 0; i < mapnum; ++i)
444 if ((kinds[i] & 0xff) == GOMP_MAP_USE_DEVICE_PTR)
445 {
446 /* If there is one such data mapping kind, then this is actually an
447 OpenACC 'host_data' construct. (GCC maps the OpenACC
448 'host_data' construct to the OpenACC 'data' construct.) Apart
449 from artificial test cases (such as an OpenACC 'host_data'
450 construct's (implicit) device initialization when there hasn't
451 been any device data be set up before...), there can't really
452 any meaningful events be generated from OpenACC 'host_data'
453 constructs, though. */
454 enter_data_event_info.other_event.parent_construct
455 = acc_construct_host_data;
456 break;
457 }
458 enter_data_event_info.other_event.implicit = 0;
459 enter_data_event_info.other_event.tool_info = NULL;
460 }
461 acc_api_info api_info;
462 if (profiling_p)
463 {
464 thr->api_info = &api_info;
465
466 api_info.device_api = acc_device_api_none;
467 api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
468 api_info.device_type = prof_info.device_type;
469 api_info.vendor = -1;
470 api_info.device_handle = NULL;
471 api_info.context_handle = NULL;
472 api_info.async_handle = NULL;
473 }
474
475 if (profiling_p)
476 goacc_profiling_dispatch (&prof_info, &enter_data_event_info, &api_info);
477
ca4c3545 478 /* Host fallback or 'do nothing'. */
479 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
813421cd 480 || (flags & GOACC_FLAG_HOST_FALLBACK))
ca4c3545 481 {
5feffd51 482 prof_info.device_type = acc_device_host;
483 api_info.device_type = prof_info.device_type;
43895be5 484 tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
485 GOMP_MAP_VARS_OPENACC);
ca4c3545 486 tgt->prev = thr->mapped_data;
487 thr->mapped_data = tgt;
488
5feffd51 489 goto out_prof;
ca4c3545 490 }
491
492 gomp_debug (0, " %s: prepare mappings\n", __FUNCTION__);
493 tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
43895be5 494 GOMP_MAP_VARS_OPENACC);
ca4c3545 495 gomp_debug (0, " %s: mappings prepared\n", __FUNCTION__);
496 tgt->prev = thr->mapped_data;
497 thr->mapped_data = tgt;
5feffd51 498
499 out_prof:
500 if (profiling_p)
501 {
502 prof_info.event_type = acc_ev_enter_data_end;
503 enter_data_event_info.other_event.event_type = prof_info.event_type;
504 goacc_profiling_dispatch (&prof_info, &enter_data_event_info, &api_info);
505
506 thr->prof_info = NULL;
507 thr->api_info = NULL;
508 }
ca4c3545 509}
510
511void
512GOACC_data_end (void)
513{
514 struct goacc_thread *thr = goacc_thread ();
5feffd51 515 struct gomp_device_descr *acc_dev = thr->dev;
ca4c3545 516 struct target_mem_desc *tgt = thr->mapped_data;
517
5feffd51 518 bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);
519
520 acc_prof_info prof_info;
521 if (profiling_p)
522 {
523 thr->prof_info = &prof_info;
524
525 prof_info.event_type = acc_ev_exit_data_start;
526 prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
527 prof_info.version = _ACC_PROF_INFO_VERSION;
528 prof_info.device_type = acc_device_type (acc_dev->type);
529 prof_info.device_number = acc_dev->target_id;
530 prof_info.thread_id = -1;
531 prof_info.async = acc_async_sync; /* Always synchronous. */
532 prof_info.async_queue = prof_info.async;
533 prof_info.src_file = NULL;
534 prof_info.func_name = NULL;
535 prof_info.line_no = -1;
536 prof_info.end_line_no = -1;
537 prof_info.func_line_no = -1;
538 prof_info.func_end_line_no = -1;
539 }
540 acc_event_info exit_data_event_info;
541 if (profiling_p)
542 {
543 exit_data_event_info.other_event.event_type
544 = prof_info.event_type;
545 exit_data_event_info.other_event.valid_bytes
546 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
547 exit_data_event_info.other_event.parent_construct = acc_construct_data;
548 exit_data_event_info.other_event.implicit = 0;
549 exit_data_event_info.other_event.tool_info = NULL;
550 }
551 acc_api_info api_info;
552 if (profiling_p)
553 {
554 thr->api_info = &api_info;
555
556 api_info.device_api = acc_device_api_none;
557 api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
558 api_info.device_type = prof_info.device_type;
559 api_info.vendor = -1;
560 api_info.device_handle = NULL;
561 api_info.context_handle = NULL;
562 api_info.async_handle = NULL;
563 }
564
565 if (profiling_p)
566 goacc_profiling_dispatch (&prof_info, &exit_data_event_info, &api_info);
567
ca4c3545 568 gomp_debug (0, " %s: restore mappings\n", __FUNCTION__);
569 thr->mapped_data = tgt->prev;
570 gomp_unmap_vars (tgt, true);
571 gomp_debug (0, " %s: mappings restored\n", __FUNCTION__);
5feffd51 572
573 if (profiling_p)
574 {
575 prof_info.event_type = acc_ev_exit_data_end;
576 exit_data_event_info.other_event.event_type = prof_info.event_type;
577 goacc_profiling_dispatch (&prof_info, &exit_data_event_info, &api_info);
578
579 thr->prof_info = NULL;
580 thr->api_info = NULL;
581 }
ca4c3545 582}
583
584void
813421cd 585GOACC_enter_exit_data (int flags_m, size_t mapnum,
ca4c3545 586 void **hostaddrs, size_t *sizes, unsigned short *kinds,
587 int async, int num_waits, ...)
588{
813421cd 589 int flags = GOACC_FLAGS_UNMARSHAL (flags_m);
590
ca4c3545 591 struct goacc_thread *thr;
592 struct gomp_device_descr *acc_dev;
ca4c3545 593 bool data_enter = false;
594 size_t i;
595
0a1fe572 596 goacc_lazy_initialize ();
ca4c3545 597
598 thr = goacc_thread ();
599 acc_dev = thr->dev;
600
737cc978 601 /* Determine whether "finalize" semantics apply to all mappings of this
602 OpenACC directive. */
603 bool finalize = false;
604 if (mapnum > 0)
605 {
606 unsigned char kind = kinds[0] & 0xff;
607 if (kind == GOMP_MAP_DELETE
608 || kind == GOMP_MAP_FORCE_FROM)
609 finalize = true;
610 }
611
ca4c3545 612 /* Determine if this is an "acc enter data". */
613 for (i = 0; i < mapnum; ++i)
614 {
615 unsigned char kind = kinds[i] & 0xff;
616
617 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
618 continue;
619
620 if (kind == GOMP_MAP_FORCE_ALLOC
621 || kind == GOMP_MAP_FORCE_PRESENT
737cc978 622 || kind == GOMP_MAP_FORCE_TO
623 || kind == GOMP_MAP_TO
624 || kind == GOMP_MAP_ALLOC)
ca4c3545 625 {
626 data_enter = true;
627 break;
628 }
629
737cc978 630 if (kind == GOMP_MAP_RELEASE
631 || kind == GOMP_MAP_DELETE
632 || kind == GOMP_MAP_FROM
ca4c3545 633 || kind == GOMP_MAP_FORCE_FROM)
634 break;
635
636 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
637 kind);
638 }
639
5feffd51 640 bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);
641
642 acc_prof_info prof_info;
643 if (profiling_p)
644 {
645 thr->prof_info = &prof_info;
646
647 prof_info.event_type
648 = data_enter ? acc_ev_enter_data_start : acc_ev_exit_data_start;
649 prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
650 prof_info.version = _ACC_PROF_INFO_VERSION;
651 prof_info.device_type = acc_device_type (acc_dev->type);
652 prof_info.device_number = acc_dev->target_id;
653 prof_info.thread_id = -1;
654 prof_info.async = async;
655 prof_info.async_queue = prof_info.async;
656 prof_info.src_file = NULL;
657 prof_info.func_name = NULL;
658 prof_info.line_no = -1;
659 prof_info.end_line_no = -1;
660 prof_info.func_line_no = -1;
661 prof_info.func_end_line_no = -1;
662 }
663 acc_event_info enter_exit_data_event_info;
664 if (profiling_p)
665 {
666 enter_exit_data_event_info.other_event.event_type
667 = prof_info.event_type;
668 enter_exit_data_event_info.other_event.valid_bytes
669 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
670 enter_exit_data_event_info.other_event.parent_construct
671 = data_enter ? acc_construct_enter_data : acc_construct_exit_data;
672 enter_exit_data_event_info.other_event.implicit = 0;
673 enter_exit_data_event_info.other_event.tool_info = NULL;
674 }
675 acc_api_info api_info;
676 if (profiling_p)
677 {
678 thr->api_info = &api_info;
679
680 api_info.device_api = acc_device_api_none;
681 api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
682 api_info.device_type = prof_info.device_type;
683 api_info.vendor = -1;
684 api_info.device_handle = NULL;
685 api_info.context_handle = NULL;
686 api_info.async_handle = NULL;
687 }
688
689 if (profiling_p)
690 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
691 &api_info);
692
693 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
694 || (flags & GOACC_FLAG_HOST_FALLBACK))
695 {
696 prof_info.device_type = acc_device_host;
697 api_info.device_type = prof_info.device_type;
698
699 goto out_prof;
700 }
701
702 if (num_waits)
703 {
704 va_list ap;
705
706 va_start (ap, num_waits);
707 goacc_wait (async, num_waits, &ap);
708 va_end (ap);
709 }
710
737cc978 711 /* In c, non-pointers and arrays are represented by a single data clause.
712 Dynamically allocated arrays and subarrays are represented by a data
713 clause followed by an internal GOMP_MAP_POINTER.
714
715 In fortran, scalars and not allocated arrays are represented by a
716 single data clause. Allocated arrays and subarrays have three mappings:
717 1) the original data clause, 2) a PSET 3) a pointer to the array data.
718 */
719
ca4c3545 720 if (data_enter)
721 {
722 for (i = 0; i < mapnum; i++)
723 {
724 unsigned char kind = kinds[i] & 0xff;
725
737cc978 726 /* Scan for pointers and PSETs. */
727 int pointer = find_pointer (i, mapnum, kinds);
ca4c3545 728
737cc978 729 if (!pointer)
ca4c3545 730 {
731 switch (kind)
732 {
737cc978 733 case GOMP_MAP_ALLOC:
ca4c3545 734 case GOMP_MAP_FORCE_ALLOC:
534b5e00 735 acc_create_async (hostaddrs[i], sizes[i], async);
ca4c3545 736 break;
737cc978 737 case GOMP_MAP_TO:
ca4c3545 738 case GOMP_MAP_FORCE_TO:
534b5e00 739 acc_copyin_async (hostaddrs[i], sizes[i], async);
ca4c3545 740 break;
741 default:
742 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
743 kind);
744 break;
745 }
746 }
747 else
748 {
737cc978 749 gomp_acc_insert_pointer (pointer, &hostaddrs[i],
534b5e00 750 &sizes[i], &kinds[i], async);
ca4c3545 751 /* Increment 'i' by two because OpenACC requires fortran
752 arrays to be contiguous, so each PSET is associated with
753 one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and
754 one MAP_POINTER. */
737cc978 755 i += pointer - 1;
ca4c3545 756 }
757 }
758 }
759 else
760 for (i = 0; i < mapnum; ++i)
761 {
762 unsigned char kind = kinds[i] & 0xff;
763
737cc978 764 int pointer = find_pointer (i, mapnum, kinds);
ca4c3545 765
737cc978 766 if (!pointer)
ca4c3545 767 {
768 switch (kind)
769 {
737cc978 770 case GOMP_MAP_RELEASE:
5cb6b0b9 771 case GOMP_MAP_DELETE:
737cc978 772 if (acc_is_present (hostaddrs[i], sizes[i]))
773 {
774 if (finalize)
534b5e00 775 acc_delete_finalize_async (hostaddrs[i], sizes[i], async);
737cc978 776 else
534b5e00 777 acc_delete_async (hostaddrs[i], sizes[i], async);
737cc978 778 }
ca4c3545 779 break;
737cc978 780 case GOMP_MAP_FROM:
ca4c3545 781 case GOMP_MAP_FORCE_FROM:
737cc978 782 if (finalize)
534b5e00 783 acc_copyout_finalize_async (hostaddrs[i], sizes[i], async);
737cc978 784 else
534b5e00 785 acc_copyout_async (hostaddrs[i], sizes[i], async);
ca4c3545 786 break;
787 default:
788 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
789 kind);
790 break;
791 }
792 }
793 else
794 {
737cc978 795 bool copyfrom = (kind == GOMP_MAP_FORCE_FROM
796 || kind == GOMP_MAP_FROM);
797 gomp_acc_remove_pointer (hostaddrs[i], sizes[i], copyfrom, async,
798 finalize, pointer);
ca4c3545 799 /* See the above comment. */
737cc978 800 i += pointer - 1;
ca4c3545 801 }
802 }
5feffd51 803
804 out_prof:
805 if (profiling_p)
806 {
807 prof_info.event_type
808 = data_enter ? acc_ev_enter_data_end : acc_ev_exit_data_end;
809 enter_exit_data_event_info.other_event.event_type = prof_info.event_type;
810 goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
811 &api_info);
812
813 thr->prof_info = NULL;
814 thr->api_info = NULL;
815 }
ca4c3545 816}
817
818static void
e561d5e1 819goacc_wait (int async, int num_waits, va_list *ap)
ca4c3545 820{
933b0582 821 while (num_waits--)
ca4c3545 822 {
e561d5e1 823 int qid = va_arg (*ap, int);
d572a4c3 824
825 /* Waiting on ACC_ASYNC_NOVAL maps to 'wait all'. */
826 if (qid == acc_async_noval)
827 {
828 if (async == acc_async_sync)
829 acc_wait_all ();
830 else
831 acc_wait_all_async (async);
832 break;
833 }
834
ca4c3545 835 if (acc_async_test (qid))
836 continue;
837
933b0582 838 if (async == acc_async_sync)
839 acc_wait (qid);
840 else if (qid == async)
534b5e00 841 /* If we're waiting on the same asynchronous queue as we're
842 launching on, the queue itself will order work as
843 required, so there's no need to wait explicitly. */
844 ;
933b0582 845 else
d572a4c3 846 acc_wait_async (qid, async);
ca4c3545 847 }
848}
849
850void
813421cd 851GOACC_update (int flags_m, size_t mapnum,
ca4c3545 852 void **hostaddrs, size_t *sizes, unsigned short *kinds,
853 int async, int num_waits, ...)
854{
813421cd 855 int flags = GOACC_FLAGS_UNMARSHAL (flags_m);
856
ca4c3545 857 size_t i;
858
0a1fe572 859 goacc_lazy_initialize ();
ca4c3545 860
861 struct goacc_thread *thr = goacc_thread ();
862 struct gomp_device_descr *acc_dev = thr->dev;
863
5feffd51 864 bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);
865
866 acc_prof_info prof_info;
867 if (profiling_p)
868 {
869 thr->prof_info = &prof_info;
870
871 prof_info.event_type = acc_ev_update_start;
872 prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
873 prof_info.version = _ACC_PROF_INFO_VERSION;
874 prof_info.device_type = acc_device_type (acc_dev->type);
875 prof_info.device_number = acc_dev->target_id;
876 prof_info.thread_id = -1;
877 prof_info.async = async;
878 prof_info.async_queue = prof_info.async;
879 prof_info.src_file = NULL;
880 prof_info.func_name = NULL;
881 prof_info.line_no = -1;
882 prof_info.end_line_no = -1;
883 prof_info.func_line_no = -1;
884 prof_info.func_end_line_no = -1;
885 }
886 acc_event_info update_event_info;
887 if (profiling_p)
888 {
889 update_event_info.other_event.event_type
890 = prof_info.event_type;
891 update_event_info.other_event.valid_bytes
892 = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
893 update_event_info.other_event.parent_construct = acc_construct_update;
894 update_event_info.other_event.implicit = 0;
895 update_event_info.other_event.tool_info = NULL;
896 }
897 acc_api_info api_info;
898 if (profiling_p)
899 {
900 thr->api_info = &api_info;
901
902 api_info.device_api = acc_device_api_none;
903 api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
904 api_info.device_type = prof_info.device_type;
905 api_info.vendor = -1;
906 api_info.device_handle = NULL;
907 api_info.context_handle = NULL;
908 api_info.async_handle = NULL;
909 }
910
911 if (profiling_p)
912 goacc_profiling_dispatch (&prof_info, &update_event_info, &api_info);
913
ca4c3545 914 if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
813421cd 915 || (flags & GOACC_FLAG_HOST_FALLBACK))
5feffd51 916 {
917 prof_info.device_type = acc_device_host;
918 api_info.device_type = prof_info.device_type;
919
920 goto out_prof;
921 }
ca4c3545 922
933b0582 923 if (num_waits)
ca4c3545 924 {
925 va_list ap;
926
927 va_start (ap, num_waits);
e561d5e1 928 goacc_wait (async, num_waits, &ap);
ca4c3545 929 va_end (ap);
930 }
931
737cc978 932 bool update_device = false;
ca4c3545 933 for (i = 0; i < mapnum; ++i)
934 {
935 unsigned char kind = kinds[i] & 0xff;
936
937 switch (kind)
938 {
939 case GOMP_MAP_POINTER:
940 case GOMP_MAP_TO_PSET:
941 break;
942
737cc978 943 case GOMP_MAP_ALWAYS_POINTER:
944 if (update_device)
945 {
946 /* Save the contents of the host pointer. */
947 void *dptr = acc_deviceptr (hostaddrs[i-1]);
948 uintptr_t t = *(uintptr_t *) hostaddrs[i];
949
950 /* Update the contents of the host pointer to reflect
951 the value of the allocated device memory in the
952 previous pointer. */
953 *(uintptr_t *) hostaddrs[i] = (uintptr_t)dptr;
534b5e00 954 /* TODO: verify that we really cannot use acc_update_device_async
955 here. */
737cc978 956 acc_update_device (hostaddrs[i], sizeof (uintptr_t));
957
958 /* Restore the host pointer. */
959 *(uintptr_t *) hostaddrs[i] = t;
960 update_device = false;
961 }
962 break;
963
964 case GOMP_MAP_TO:
965 if (!acc_is_present (hostaddrs[i], sizes[i]))
966 {
967 update_device = false;
968 break;
969 }
970 /* Fallthru */
ca4c3545 971 case GOMP_MAP_FORCE_TO:
737cc978 972 update_device = true;
534b5e00 973 acc_update_device_async (hostaddrs[i], sizes[i], async);
ca4c3545 974 break;
975
737cc978 976 case GOMP_MAP_FROM:
977 if (!acc_is_present (hostaddrs[i], sizes[i]))
978 {
979 update_device = false;
980 break;
981 }
982 /* Fallthru */
ca4c3545 983 case GOMP_MAP_FORCE_FROM:
737cc978 984 update_device = false;
534b5e00 985 acc_update_self_async (hostaddrs[i], sizes[i], async);
ca4c3545 986 break;
987
988 default:
989 gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind);
990 break;
991 }
992 }
5feffd51 993
994 out_prof:
995 if (profiling_p)
996 {
997 prof_info.event_type = acc_ev_update_end;
998 update_event_info.other_event.event_type = prof_info.event_type;
999 goacc_profiling_dispatch (&prof_info, &update_event_info, &api_info);
1000
1001 thr->prof_info = NULL;
1002 thr->api_info = NULL;
1003 }
ca4c3545 1004}
1005
1006void
1007GOACC_wait (int async, int num_waits, ...)
1008{
5feffd51 1009 goacc_lazy_initialize ();
1010
1011 struct goacc_thread *thr = goacc_thread ();
1012
1013 /* No nesting. */
1014 assert (thr->prof_info == NULL);
1015 assert (thr->api_info == NULL);
1016 acc_prof_info prof_info;
1017 acc_api_info api_info;
1018 bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
1019 if (profiling_p)
1020 {
1021 prof_info.async = async;
1022 prof_info.async_queue = prof_info.async;
1023 }
1024
933b0582 1025 if (num_waits)
1026 {
1027 va_list ap;
ca4c3545 1028
933b0582 1029 va_start (ap, num_waits);
e561d5e1 1030 goacc_wait (async, num_waits, &ap);
933b0582 1031 va_end (ap);
1032 }
1033 else if (async == acc_async_sync)
1034 acc_wait_all ();
44b7d2b9 1035 else
1036 acc_wait_all_async (async);
5feffd51 1037
1038 if (profiling_p)
1039 {
1040 thr->prof_info = NULL;
1041 thr->api_info = NULL;
1042 }
ca4c3545 1043}
1044
75a3472e 1045/* Legacy entry point (GCC 5). */
1046
ca4c3545 1047int
1048GOACC_get_num_threads (void)
1049{
1050 return 1;
1051}
1052
75a3472e 1053/* Legacy entry point (GCC 5). */
1054
ca4c3545 1055int
1056GOACC_get_thread_num (void)
1057{
1058 return 0;
1059}
2fc5e987 1060
1061void
813421cd 1062GOACC_declare (int flags_m, size_t mapnum,
2fc5e987 1063 void **hostaddrs, size_t *sizes, unsigned short *kinds)
1064{
1065 int i;
1066
1067 for (i = 0; i < mapnum; i++)
1068 {
1069 unsigned char kind = kinds[i] & 0xff;
1070
1071 if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
1072 continue;
1073
1074 switch (kind)
1075 {
1076 case GOMP_MAP_FORCE_ALLOC:
2fc5e987 1077 case GOMP_MAP_FORCE_FROM:
1078 case GOMP_MAP_FORCE_TO:
1079 case GOMP_MAP_POINTER:
737cc978 1080 case GOMP_MAP_RELEASE:
5cb6b0b9 1081 case GOMP_MAP_DELETE:
813421cd 1082 GOACC_enter_exit_data (flags_m, 1, &hostaddrs[i], &sizes[i],
dffc468e 1083 &kinds[i], GOMP_ASYNC_SYNC, 0);
2fc5e987 1084 break;
1085
1086 case GOMP_MAP_FORCE_DEVICEPTR:
1087 break;
1088
1089 case GOMP_MAP_ALLOC:
1090 if (!acc_is_present (hostaddrs[i], sizes[i]))
813421cd 1091 GOACC_enter_exit_data (flags_m, 1, &hostaddrs[i], &sizes[i],
dffc468e 1092 &kinds[i], GOMP_ASYNC_SYNC, 0);
2fc5e987 1093 break;
1094
1095 case GOMP_MAP_TO:
813421cd 1096 GOACC_enter_exit_data (flags_m, 1, &hostaddrs[i], &sizes[i],
dffc468e 1097 &kinds[i], GOMP_ASYNC_SYNC, 0);
2fc5e987 1098
1099 break;
1100
1101 case GOMP_MAP_FROM:
813421cd 1102 GOACC_enter_exit_data (flags_m, 1, &hostaddrs[i], &sizes[i],
dffc468e 1103 &kinds[i], GOMP_ASYNC_SYNC, 0);
2fc5e987 1104 break;
1105
1106 case GOMP_MAP_FORCE_PRESENT:
1107 if (!acc_is_present (hostaddrs[i], sizes[i]))
1108 gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i],
1109 (unsigned long) sizes[i]);
1110 break;
1111
1112 default:
1113 assert (0);
1114 break;
1115 }
1116 }
1117}
Mirror of https://gcc.gnu.org/git/gcc.git