1 /* OpenACC Runtime initialization routines
3 Copyright (C) 2013-2019 Free Software Foundation, Inc.
5 Contributed by Mentor Embedded.
7 This file is part of the GNU Offloading and Multi Processing Library
10 Libgomp is free software; you can redistribute it and/or modify it
11 under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3, or (at your option)
15 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
20 Under Section 7 of GPL version 3, you are granted additional
21 permissions described in the GCC Runtime Library Exception, version
22 3.1, as published by the Free Software Foundation.
24 You should have received a copy of the GNU General Public License and
25 a copy of the GCC Runtime Library Exception along with this program;
26 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
27 <http://www.gnu.org/licenses/>. */
38 /* This lock is used to protect access to cached_base_dev, dispatchers and
39 the (abstract) initialisation state of attached offloading devices. */
41 static gomp_mutex_t acc_device_lock
;
43 /* A cached version of the dispatcher for the global "current" accelerator type,
44 e.g. used as the default when creating new host threads. This is the
45 device-type equivalent of goacc_device_num (which specifies which device to
46 use out of potentially several of the same type). If there are several
47 devices of a given type, this points at the first one. */
49 static struct gomp_device_descr
*cached_base_dev
= NULL
;
51 #if defined HAVE_TLS || defined USE_EMUTLS
52 __thread
struct goacc_thread
*goacc_tls_data
;
54 pthread_key_t goacc_tls_key
;
56 static pthread_key_t goacc_cleanup_key
;
58 static struct goacc_thread
*goacc_threads
;
59 static gomp_mutex_t goacc_thread_lock
;
61 /* An array of dispatchers for device types, indexed by the type. This array
62 only references "base" devices, and other instances of the same type are
63 found by simply indexing from each such device (which are stored linearly,
64 grouped by device in target.c:devices). */
65 static struct gomp_device_descr
*dispatchers
[_ACC_device_hwm
] = { 0 };
68 goacc_register (struct gomp_device_descr
*disp
)
70 /* Only register the 0th device here. */
71 if (disp
->target_id
!= 0)
74 gomp_mutex_lock (&acc_device_lock
);
76 assert (acc_device_type (disp
->type
) != acc_device_none
77 && acc_device_type (disp
->type
) != acc_device_default
78 && acc_device_type (disp
->type
) != acc_device_not_host
);
79 assert (!dispatchers
[disp
->type
]);
80 dispatchers
[disp
->type
] = disp
;
82 gomp_mutex_unlock (&acc_device_lock
);
85 /* OpenACC names some things a little differently. */
88 get_openacc_name (const char *name
)
90 if (strcmp (name
, "nvptx") == 0)
97 name_of_acc_device_t (enum acc_device_t type
)
101 case acc_device_none
: return "none";
102 case acc_device_default
: return "default";
103 case acc_device_host
: return "host";
104 case acc_device_not_host
: return "not_host";
105 case acc_device_nvidia
: return "nvidia";
106 default: gomp_fatal ("unknown device type %u", (unsigned) type
);
110 /* ACC_DEVICE_LOCK must be held before calling this function. If FAIL_IS_ERROR
111 is true, this function raises an error if there are no devices of type D,
112 otherwise it returns NULL in that case. */
114 static struct gomp_device_descr
*
115 resolve_device (acc_device_t d
, bool fail_is_error
)
117 acc_device_t d_arg
= d
;
121 case acc_device_default
:
123 if (goacc_device_type
)
125 /* Lookup the named device. */
126 while (++d
!= _ACC_device_hwm
)
128 && !strcasecmp (goacc_device_type
,
129 get_openacc_name (dispatchers
[d
]->name
))
130 && dispatchers
[d
]->get_num_devices_func () > 0)
135 gomp_mutex_unlock (&acc_device_lock
);
136 gomp_fatal ("device type %s not supported", goacc_device_type
);
142 /* No default device specified, so start scanning for any non-host
143 device that is available. */
144 d
= acc_device_not_host
;
148 case acc_device_not_host
:
149 /* Find the first available device after acc_device_not_host. */
150 while (++d
!= _ACC_device_hwm
)
151 if (dispatchers
[d
] && dispatchers
[d
]->get_num_devices_func () > 0)
153 if (d_arg
== acc_device_default
)
160 gomp_mutex_unlock (&acc_device_lock
);
161 gomp_fatal ("no device found");
167 case acc_device_host
:
171 if (d
> _ACC_device_hwm
)
174 goto unsupported_device
;
182 assert (d
!= acc_device_none
183 && d
!= acc_device_default
184 && d
!= acc_device_not_host
);
186 if (dispatchers
[d
] == NULL
&& fail_is_error
)
189 gomp_mutex_unlock (&acc_device_lock
);
190 gomp_fatal ("device type %s not supported", name_of_acc_device_t (d
));
193 return dispatchers
[d
];
196 /* Emit a suitable error if no device of a particular type is available, or
197 the given device number is out-of-range. */
199 acc_dev_num_out_of_range (acc_device_t d
, int ord
, int ndevs
)
202 gomp_fatal ("no devices of type %s available", name_of_acc_device_t (d
));
204 gomp_fatal ("device %u out of range", ord
);
207 /* This is called when plugins have been initialized, and serves to call
208 (indirectly) the target's device_init hook. Calling multiple times without
209 an intervening acc_shutdown_1 call is an error. ACC_DEVICE_LOCK must be
210 held before calling this function. */
212 static struct gomp_device_descr
*
213 acc_init_1 (acc_device_t d
)
215 struct gomp_device_descr
*base_dev
, *acc_dev
;
218 base_dev
= resolve_device (d
, true);
220 ndevs
= base_dev
->get_num_devices_func ();
222 if (ndevs
<= 0 || goacc_device_num
>= ndevs
)
223 acc_dev_num_out_of_range (d
, goacc_device_num
, ndevs
);
225 acc_dev
= &base_dev
[goacc_device_num
];
227 gomp_mutex_lock (&acc_dev
->lock
);
228 if (acc_dev
->state
== GOMP_DEVICE_INITIALIZED
)
230 gomp_mutex_unlock (&acc_dev
->lock
);
231 gomp_fatal ("device already active");
234 gomp_init_device (acc_dev
);
235 gomp_mutex_unlock (&acc_dev
->lock
);
240 /* ACC_DEVICE_LOCK must be held before calling this function. */
243 acc_shutdown_1 (acc_device_t d
)
245 struct gomp_device_descr
*base_dev
;
246 struct goacc_thread
*walk
;
248 bool devices_active
= false;
250 /* Get the base device for this device type. */
251 base_dev
= resolve_device (d
, true);
253 ndevs
= base_dev
->get_num_devices_func ();
255 /* Unload all the devices of this type that have been opened. */
256 for (i
= 0; i
< ndevs
; i
++)
258 struct gomp_device_descr
*acc_dev
= &base_dev
[i
];
260 gomp_mutex_lock (&acc_dev
->lock
);
261 gomp_unload_device (acc_dev
);
262 gomp_mutex_unlock (&acc_dev
->lock
);
265 gomp_mutex_lock (&goacc_thread_lock
);
267 /* Free target-specific TLS data and close all devices. */
268 for (walk
= goacc_threads
; walk
!= NULL
; walk
= walk
->next
)
270 if (walk
->target_tls
)
271 base_dev
->openacc
.destroy_thread_data_func (walk
->target_tls
);
273 walk
->target_tls
= NULL
;
275 /* This would mean the user is shutting down OpenACC in the middle of an
276 "acc data" pragma. Likely not intentional. */
277 if (walk
->mapped_data
)
279 gomp_mutex_unlock (&goacc_thread_lock
);
280 gomp_fatal ("shutdown in 'acc data' region");
283 /* Similarly, if this happens then user code has done something weird. */
284 if (walk
->saved_bound_dev
)
286 gomp_mutex_unlock (&goacc_thread_lock
);
287 gomp_fatal ("shutdown during host fallback");
292 gomp_mutex_lock (&walk
->dev
->lock
);
293 gomp_free_memmap (&walk
->dev
->mem_map
);
294 gomp_mutex_unlock (&walk
->dev
->lock
);
297 walk
->base_dev
= NULL
;
301 gomp_mutex_unlock (&goacc_thread_lock
);
303 /* Close all the devices of this type that have been opened. */
305 for (i
= 0; i
< ndevs
; i
++)
307 struct gomp_device_descr
*acc_dev
= &base_dev
[i
];
308 gomp_mutex_lock (&acc_dev
->lock
);
309 if (acc_dev
->state
== GOMP_DEVICE_INITIALIZED
)
311 devices_active
= true;
312 ret
&= gomp_fini_device (acc_dev
);
313 acc_dev
->state
= GOMP_DEVICE_UNINITIALIZED
;
315 gomp_mutex_unlock (&acc_dev
->lock
);
319 gomp_fatal ("device finalization failed");
322 gomp_fatal ("no device initialized");
325 static struct goacc_thread
*
326 goacc_new_thread (void)
328 struct goacc_thread
*thr
= gomp_malloc (sizeof (struct goacc_thread
));
330 #if defined HAVE_TLS || defined USE_EMUTLS
331 goacc_tls_data
= thr
;
333 pthread_setspecific (goacc_tls_key
, thr
);
336 pthread_setspecific (goacc_cleanup_key
, thr
);
338 gomp_mutex_lock (&goacc_thread_lock
);
339 thr
->next
= goacc_threads
;
341 gomp_mutex_unlock (&goacc_thread_lock
);
347 goacc_destroy_thread (void *data
)
349 struct goacc_thread
*thr
= data
, *walk
, *prev
;
351 gomp_mutex_lock (&goacc_thread_lock
);
355 struct gomp_device_descr
*acc_dev
= thr
->dev
;
357 if (acc_dev
&& thr
->target_tls
)
359 acc_dev
->openacc
.destroy_thread_data_func (thr
->target_tls
);
360 thr
->target_tls
= NULL
;
363 assert (!thr
->mapped_data
);
365 /* Remove from thread list. */
366 for (prev
= NULL
, walk
= goacc_threads
; walk
;
367 prev
= walk
, walk
= walk
->next
)
371 goacc_threads
= walk
->next
;
373 prev
->next
= walk
->next
;
383 gomp_mutex_unlock (&goacc_thread_lock
);
386 /* Use the ORD'th device instance for the current host thread (or -1 for the
387 current global default). The device (and the runtime) must be initialised
388 before calling this function. */
391 goacc_attach_host_thread_to_device (int ord
)
393 struct goacc_thread
*thr
= goacc_thread ();
394 struct gomp_device_descr
*acc_dev
= NULL
, *base_dev
= NULL
;
397 if (thr
&& thr
->dev
&& (thr
->dev
->target_id
== ord
|| ord
< 0))
401 ord
= goacc_device_num
;
403 /* Decide which type of device to use. If the current thread has a device
404 type already (e.g. set by acc_set_device_type), use that, else use the
406 if (thr
&& thr
->base_dev
)
407 base_dev
= thr
->base_dev
;
410 assert (cached_base_dev
);
411 base_dev
= cached_base_dev
;
414 num_devices
= base_dev
->get_num_devices_func ();
415 if (num_devices
<= 0 || ord
>= num_devices
)
416 acc_dev_num_out_of_range (acc_device_type (base_dev
->type
), ord
,
420 thr
= goacc_new_thread ();
422 thr
->base_dev
= base_dev
;
423 thr
->dev
= acc_dev
= &base_dev
[ord
];
424 thr
->saved_bound_dev
= NULL
;
425 thr
->mapped_data
= NULL
;
428 = acc_dev
->openacc
.create_thread_data_func (ord
);
431 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
432 init/shutdown is per-process or per-thread. We choose per-process. */
435 acc_init (acc_device_t d
)
437 gomp_init_targets_once ();
439 gomp_mutex_lock (&acc_device_lock
);
441 cached_base_dev
= acc_init_1 (d
);
443 gomp_mutex_unlock (&acc_device_lock
);
445 goacc_attach_host_thread_to_device (-1);
451 acc_shutdown (acc_device_t d
)
453 gomp_init_targets_once ();
455 gomp_mutex_lock (&acc_device_lock
);
459 gomp_mutex_unlock (&acc_device_lock
);
462 ialias (acc_shutdown
)
465 acc_get_num_devices (acc_device_t d
)
468 struct gomp_device_descr
*acc_dev
;
470 if (d
== acc_device_none
)
473 gomp_init_targets_once ();
475 gomp_mutex_lock (&acc_device_lock
);
476 acc_dev
= resolve_device (d
, false);
477 gomp_mutex_unlock (&acc_device_lock
);
482 n
= acc_dev
->get_num_devices_func ();
489 ialias (acc_get_num_devices
)
491 /* Set the device type for the current thread only (using the current global
492 default device number), initialising that device if necessary. Also set the
493 default device type for new threads to D. */
496 acc_set_device_type (acc_device_t d
)
498 struct gomp_device_descr
*base_dev
, *acc_dev
;
499 struct goacc_thread
*thr
= goacc_thread ();
501 gomp_init_targets_once ();
503 gomp_mutex_lock (&acc_device_lock
);
505 cached_base_dev
= base_dev
= resolve_device (d
, true);
506 acc_dev
= &base_dev
[goacc_device_num
];
508 gomp_mutex_lock (&acc_dev
->lock
);
509 if (acc_dev
->state
== GOMP_DEVICE_UNINITIALIZED
)
510 gomp_init_device (acc_dev
);
511 gomp_mutex_unlock (&acc_dev
->lock
);
513 gomp_mutex_unlock (&acc_device_lock
);
515 /* We're changing device type: invalidate the current thread's dev and
516 base_dev pointers. */
517 if (thr
&& thr
->base_dev
!= base_dev
)
519 thr
->base_dev
= thr
->dev
= NULL
;
520 if (thr
->mapped_data
)
521 gomp_fatal ("acc_set_device_type in 'acc data' region");
524 goacc_attach_host_thread_to_device (-1);
527 ialias (acc_set_device_type
)
530 acc_get_device_type (void)
532 acc_device_t res
= acc_device_none
;
533 struct gomp_device_descr
*dev
;
534 struct goacc_thread
*thr
= goacc_thread ();
536 if (thr
&& thr
->base_dev
)
537 res
= acc_device_type (thr
->base_dev
->type
);
540 gomp_init_targets_once ();
542 gomp_mutex_lock (&acc_device_lock
);
543 dev
= resolve_device (acc_device_default
, true);
544 gomp_mutex_unlock (&acc_device_lock
);
545 res
= acc_device_type (dev
->type
);
548 assert (res
!= acc_device_default
549 && res
!= acc_device_not_host
);
554 ialias (acc_get_device_type
)
557 acc_get_device_num (acc_device_t d
)
559 const struct gomp_device_descr
*dev
;
560 struct goacc_thread
*thr
= goacc_thread ();
562 if (d
>= _ACC_device_hwm
)
563 gomp_fatal ("unknown device type %u", (unsigned) d
);
565 gomp_init_targets_once ();
567 gomp_mutex_lock (&acc_device_lock
);
568 dev
= resolve_device (d
, true);
569 gomp_mutex_unlock (&acc_device_lock
);
571 if (thr
&& thr
->base_dev
== dev
&& thr
->dev
)
572 return thr
->dev
->target_id
;
574 return goacc_device_num
;
577 ialias (acc_get_device_num
)
580 acc_set_device_num (int ord
, acc_device_t d
)
582 struct gomp_device_descr
*base_dev
, *acc_dev
;
585 gomp_init_targets_once ();
588 ord
= goacc_device_num
;
591 /* Set whatever device is being used by the current host thread to use
592 device instance ORD. It's unclear if this is supposed to affect other
593 host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num). */
594 goacc_attach_host_thread_to_device (ord
);
597 gomp_mutex_lock (&acc_device_lock
);
599 cached_base_dev
= base_dev
= resolve_device (d
, true);
601 num_devices
= base_dev
->get_num_devices_func ();
603 if (num_devices
<= 0 || ord
>= num_devices
)
604 acc_dev_num_out_of_range (d
, ord
, num_devices
);
606 acc_dev
= &base_dev
[ord
];
608 gomp_mutex_lock (&acc_dev
->lock
);
609 if (acc_dev
->state
== GOMP_DEVICE_UNINITIALIZED
)
610 gomp_init_device (acc_dev
);
611 gomp_mutex_unlock (&acc_dev
->lock
);
613 gomp_mutex_unlock (&acc_device_lock
);
615 goacc_attach_host_thread_to_device (ord
);
618 goacc_device_num
= ord
;
621 ialias (acc_set_device_num
)
623 /* For -O and higher, the compiler always attempts to expand acc_on_device, but
624 if the user disables the builtin, or calls it via a pointer, we'll need this
627 Compile this with optimization, so that the compiler expands
628 this, rather than generating infinitely recursive code. */
630 int __attribute__ ((__optimize__ ("O2")))
631 acc_on_device (acc_device_t dev
)
633 return __builtin_acc_on_device (dev
);
636 ialias (acc_on_device
)
638 attribute_hidden
void
639 goacc_runtime_initialize (void)
641 gomp_mutex_init (&acc_device_lock
);
643 #if !(defined HAVE_TLS || defined USE_EMUTLS)
644 pthread_key_create (&goacc_tls_key
, NULL
);
647 pthread_key_create (&goacc_cleanup_key
, goacc_destroy_thread
);
649 cached_base_dev
= NULL
;
651 goacc_threads
= NULL
;
652 gomp_mutex_init (&goacc_thread_lock
);
654 /* Initialize and register the 'host' device type. */
658 /* Compiler helper functions */
660 attribute_hidden
void
661 goacc_save_and_set_bind (acc_device_t d
)
663 struct goacc_thread
*thr
= goacc_thread ();
665 assert (!thr
->saved_bound_dev
);
667 thr
->saved_bound_dev
= thr
->dev
;
668 thr
->dev
= dispatchers
[d
];
671 attribute_hidden
void
672 goacc_restore_bind (void)
674 struct goacc_thread
*thr
= goacc_thread ();
676 thr
->dev
= thr
->saved_bound_dev
;
677 thr
->saved_bound_dev
= NULL
;
680 /* This is called from any OpenACC support function that may need to implicitly
681 initialize the libgomp runtime, either globally or from a new host thread.
682 On exit "goacc_thread" will return a valid & populated thread block. */
684 attribute_hidden
void
685 goacc_lazy_initialize (void)
687 struct goacc_thread
*thr
= goacc_thread ();
692 if (!cached_base_dev
)
693 acc_init (acc_device_default
);
695 goacc_attach_host_thread_to_device (-1);